Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TS-4994-3.0

This commit is contained in:
Hongze Cheng 2024-12-02 09:11:44 +08:00
commit 61d8fbc9d9
70 changed files with 2184 additions and 476 deletions

View File

@ -337,14 +337,14 @@ Query OK, 2 row(s) in set (0.001700s)
TDengine 提供了丰富的应用程序开发接口,其中包括 C/C++、Java、Python、Go、Node.js、C# 、RESTful 等,便于用户快速开发应用: TDengine 提供了丰富的应用程序开发接口,其中包括 C/C++、Java、Python、Go、Node.js、C# 、RESTful 等,便于用户快速开发应用:
- [Java](https://docs.taosdata.com/connector/java/) - [Java](https://docs.taosdata.com/reference/connector/java/)
- [C/C++](https://docs.taosdata.com/connector/cpp/) - [C/C++](https://docs.taosdata.com/reference/connector/cpp/)
- [Python](https://docs.taosdata.com/connector/python/) - [Python](https://docs.taosdata.com/reference/connector/python/)
- [Go](https://docs.taosdata.com/connector/go/) - [Go](https://docs.taosdata.com/reference/connector/go/)
- [Node.js](https://docs.taosdata.com/connector/node/) - [Node.js](https://docs.taosdata.com/reference/connector/node/)
- [Rust](https://docs.taosdata.com/connector/rust/) - [Rust](https://docs.taosdata.com/reference/connector/rust/)
- [C#](https://docs.taosdata.com/connector/csharp/) - [C#](https://docs.taosdata.com/reference/connector/csharp/)
- [RESTful API](https://docs.taosdata.com/connector/rest-api/) - [RESTful API](https://docs.taosdata.com/reference/connector/rest-api/)
# 成为社区贡献者 # 成为社区贡献者

View File

@ -357,14 +357,14 @@ Query OK, 2 row(s) in set (0.001700s)
TDengine provides abundant developing tools for users to develop on TDengine. Follow the links below to find your desired connectors and relevant documentation. TDengine provides abundant developing tools for users to develop on TDengine. Follow the links below to find your desired connectors and relevant documentation.
- [Java](https://docs.tdengine.com/reference/connector/java/) - [Java](https://docs.tdengine.com/reference/connectors/java/)
- [C/C++](https://docs.tdengine.com/reference/connector/cpp/) - [C/C++](https://docs.tdengine.com/reference/connectors/cpp/)
- [Python](https://docs.tdengine.com/reference/connector/python/) - [Python](https://docs.tdengine.com/reference/connectors/python/)
- [Go](https://docs.tdengine.com/reference/connector/go/) - [Go](https://docs.tdengine.com/reference/connectors/go/)
- [Node.js](https://docs.tdengine.com/reference/connector/node/) - [Node.js](https://docs.tdengine.com/reference/connectors/node/)
- [Rust](https://docs.tdengine.com/reference/connector/rust/) - [Rust](https://docs.tdengine.com/reference/connectors/rust/)
- [C#](https://docs.tdengine.com/reference/connector/csharp/) - [C#](https://docs.tdengine.com/reference/connectors/csharp/)
- [RESTful API](https://docs.tdengine.com/reference/rest-api/) - [RESTful API](https://docs.tdengine.com/reference/connectors/rest-api/)
# Contribute to TDengine # Contribute to TDengine

View File

@ -29,14 +29,23 @@ After modifying configuration file parameters, it is necessary to restart the *t
### Connection Related ### Connection Related
| Parameter Name | Parameter Description | | Parameter Name | support version | Parameter Description |
| :--------------------- | :----------------------------------------------------------- | | :--------------------- |:---------------| :----------------------------------------------------------- |
| firstEp | The endpoint of the first dnode in the cluster to connect to when taosd starts; default value: localhost:6030 | | firstEp | | The endpoint of the first dnode in the cluster to connect to when taosd starts; default value: localhost:6030 |
| secondEp | If firstEp cannot connect, attempt to connect to the second dnode's endpoint in the cluster; default value: none | | secondEp | | If firstEp cannot connect, attempt to connect to the second dnode's endpoint in the cluster; default value: none |
| fqdn | The service address that taosd listens on after startup; default value: the first hostname configured on the server | | fqdn | | The service address that taosd listens on after startup; default value: the first hostname configured on the server |
| serverPort | The port that taosd listens on after startup; default value: 6030 | | compressMsgSize | | Whether to compress RPC messages; -1: no messages are compressed; 0: all messages are compressed; N (N>0): only messages larger than N bytes are compressed; default value: -1 |
| numOfRpcSessions | The maximum number of connections a client can create; range: 100-100000; default value: 30000 | | shellActivityTimer | | The duration in seconds for the client to send heartbeats to the mnode; range: 1-120; default value: 3 |
| timeToGetAvailableConn | The maximum wait time to obtain an available connection; range: 10-50000000; unit: milliseconds; default value: 500000 | | numOfRpcSessions | | The maximum number of RPC connections supported; range: 100-100000; default value: 30000 |
| numOfRpcThreads | | The number of threads for RPC data transmission; range: 1-50, default value: half of the CPU cores |
| numOfTaskQueueThreads | | The number of threads for the client to process RPC messages, range: 4-16, default value: half of the CPU cores |
| rpcQueueMemoryAllowed | | The maximum amount of memory allowed for RPC messages received on a dnode; unit: bytes; range: 104857600-INT64_MAX; default value: 1/10 of server memory |
| resolveFQDNRetryTime | Removed in 3.x | The number of retries when FQDN resolution fails |
| timeToGetAvailableConn | Removed in 3.3.4.x | The maximum waiting time to obtain an available connection; range: 10-50000000; unit: milliseconds; default value: 500000 |
| maxShellConns | Removed in 3.x | The maximum number of connections allowed to be created |
| maxRetryWaitTime | | The maximum timeout for reconnection; default value: 10s |
| shareConnLimit | Added in 3.3.4.0 | The number of requests that a connection can share; range: 1-512; default value: 10 |
| readTimeout | Added in 3.3.4.0 | The minimum timeout for a single request; range: 64-604800; unit: seconds; default value: 900 |
### Monitoring Related ### Monitoring Related

View File

@ -39,6 +39,10 @@ The TDengine client driver provides all the APIs needed for application programm
| enableScience | Whether to enable scientific notation for floating-point numbers; 0: disable, 1: enable; default value: 1 | | enableScience | Whether to enable scientific notation for floating-point numbers; 0: disable, 1: enable; default value: 1 |
| compressMsgSize | Whether to compress RPC messages; -1: do not compress any messages; 0: compress all messages; N (N>0): compress only messages larger than N bytes; default value: -1 | | compressMsgSize | Whether to compress RPC messages; -1: do not compress any messages; 0: compress all messages; N (N>0): compress only messages larger than N bytes; default value: -1 |
| queryTableNotExistAsEmpty | Whether to return an empty result set when the queried table does not exist; false: return an error; true: return an empty result set; default value: false | | queryTableNotExistAsEmpty | Whether to return an empty result set when the queried table does not exist; false: return an error; true: return an empty result set; default value: false |
| numOfRpcThreads | The number of threads for RPC data transmission; range: 1-50, default value: half of the CPU cores |
| numOfTaskQueueThreads | The number of threads for the client to process RPC messages, range: 4-16, default value: half of the CPU cores |
| shareConnLimit | The number of requests that a connection can share; range: 1-512; default value: 10 |
| readTimeout | The minimum timeout for a single request; range: 64-604800; unit: seconds; default value: 900 |
## API ## API

View File

@ -84,6 +84,9 @@ taos -h h1.taos.com -s "use db; show tables;"
You can also control the behavior of the TDengine CLI through parameters set in the configuration file. For available configuration parameters, refer to [Client Configuration](../../components/taosc). You can also control the behavior of the TDengine CLI through parameters set in the configuration file. For available configuration parameters, refer to [Client Configuration](../../components/taosc).
## Error Codes Reference
After version 3.3.4.8 of TDengine, the TDengine CLI returned error codes in the error message. Users can search for the specific cause and solution on the error code page of the TDengine official website, see [Error Codes Table](https://docs.taosdata.com/reference/error-code)
## TDengine CLI Tips ## TDengine CLI Tips
- Use the up and down arrow keys to view previously entered commands. - Use the up and down arrow keys to view previously entered commands.

View File

@ -62,7 +62,8 @@ window_clause: {
| COUNT_WINDOW(count_val[, sliding_val]) | COUNT_WINDOW(count_val[, sliding_val])
interp_clause: interp_clause:
RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val) RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val)
| RANGE(ts_val, surrounding_time_val) FILL(fill_mod_and_val)
partition_by_clause: partition_by_clause:
PARTITION BY partition_by_expr [, partition_by_expr] ... PARTITION BY partition_by_expr [, partition_by_expr] ...
@ -256,6 +257,13 @@ The \_irowts pseudo column can only be used with the interp function to return t
SELECT _irowts, interp(current) FROM meters RANGE('2020-01-01 10:00:00', '2020-01-01 10:30:00') EVERY(1s) FILL(linear); SELECT _irowts, interp(current) FROM meters RANGE('2020-01-01 10:00:00', '2020-01-01 10:30:00') EVERY(1s) FILL(linear);
``` ```
** \_IROWTS\_ORIGIN**
Pseudo column `_irowts_origin` is used to get the original timestamp of the row used for filling. It can only be used with the INTERP query. `_irowts_origin` is not supported in stream. Only FILL PREV/NEXT/NEAR is supported. If there is not data in range, return NULL.
```sql
SELECT _irowts_origin, interp(current) FROM meters RANGE('2020-01-01 10:00:00', '2020-01-01 10:30:00') EVERY(1s) FILL(PREV);
```
## Query Objects ## Query Objects
After the FROM keyword, there can be several table (supertable) lists, or the results of subqueries. After the FROM keyword, there can be several table (supertable) lists, or the results of subqueries.

View File

@ -1869,6 +1869,9 @@ INTERP(expr [, ignore_null_values])
- INTERP can be used with the pseudo-column _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0 onwards). - INTERP can be used with the pseudo-column _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0 onwards).
- INTERP can also be used with the pseudo-column _isfilled to show whether the returned result is from the original record or produced by the interpolation algorithm (supported from version 3.0.3.0 onwards). - INTERP can also be used with the pseudo-column _isfilled to show whether the returned result is from the original record or produced by the interpolation algorithm (supported from version 3.0.3.0 onwards).
- When querying a table with a composite primary key, if there are multiple records with the same timestamp, only the data corresponding to the minimum composite primary key will participate in the calculation. - When querying a table with a composite primary key, if there are multiple records with the same timestamp, only the data corresponding to the minimum composite primary key will participate in the calculation.
- `INTERP` support NEAR fill mode. When `FILL(NEAR)` is used, the nearest value to the interpolation point is used to fill the missing value. If there are multiple values with the same distance to the interpolation point, previous row is used. NEAR fill is not supported in stream computation. For example, `SELECT _irowts,INTERP(current) FROM test.meters RANGE('2017-07-22 00:00:00','2017-07-24 12:25:00') EVERY(1h) FILL(NEAR)`(supported from version 3.3.4.9 onwards).
- Psedo column `_irowts_origin` can be used along with `INTERP` only when using NEAR/NEXT/PREV fill mode, `_irowts_origin` supported from version 3.3.4.9 onwards.
- `INTERP` RANGE clause support INTERVAL extension(supported from version 3.3.4.9 onwards), like `RANGE('2023-01-01 00:00:00', 1d)`. The second parameter is the interval length, and the unit cannot use y(year), n(month). The interval length must be an integer, with no quotes, the value can't be 0. The interval length is used to restrict the search range from the time point specified. For example, `SELECT _irowts,INTERP(current) FROM test.meters RANGE('2017-07-22 00:00:00', 1d) FILL(NEAR, 1)`. The query will return the interpolation result of the current column within the range of 1 day from the time point '2017-07-22 00:00:00'. If there is no data within the range, the specified value in FILL will be used. Only FILL PREV/NEXT/NEAR is supported in this case. It's illegal to use `EVERY` clause and NOT specify values in FILL clause in this case. None data-point range clause with INTERVAL extension is not supported currently, like `RANGE('2017-07-22 00:00:00', '2017-07-22 12:00:00', 1h)` is not supported.
### LAST ### LAST

View File

@ -250,7 +250,7 @@ flush database stream_dest_db; ---- 流计算写入数据的超级表所在的
```sql ```sql
create stream streams1 into test1.streamst as select _wstart, count(a) c1 from test.st interval(1s) ; create stream streams1 into test1.streamst as select _wstart, count(a) c1 from test.st interval(1s) ;
drop database streams1; drop stream streams1;
flush database test; flush database test;
flush database test1; flush database test1;
``` ```

View File

@ -65,6 +65,10 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T
**订阅主题及 QoS 配置** 中填写要消费的 Topic 名称和 QoS。使用如下格式设置 `{topic_name}::{qos}`(如:`my_topic::0`。MQTT 协议 5.0 支持共享订阅,可以通过多个客户端订阅同一个 Topic 实现负载均衡,使用如下格式: `$share/{group_name}/{topic_name}::{qos}`,其中,`$share` 是固定前缀,表示启用共享订阅,`group_name` 是分组名称,类似 kafka 的消费者组。 **订阅主题及 QoS 配置** 中填写要消费的 Topic 名称和 QoS。使用如下格式设置 `{topic_name}::{qos}`(如:`my_topic::0`。MQTT 协议 5.0 支持共享订阅,可以通过多个客户端订阅同一个 Topic 实现负载均衡,使用如下格式: `$share/{group_name}/{topic_name}::{qos}`,其中,`$share` 是固定前缀,表示启用共享订阅,`group_name` 是分组名称,类似 kafka 的消费者组。
**数据压缩**配置消息体压缩算法taosX 在接收到消息后,使用对应的压缩算法对消息体进行解压缩获取原始数据。可选项 none(不压缩), gzip, snappy, lz4 和 zstd默认为 none。
**字符编码**配置消息体编码格式taosX 在接收到消息后,使用对应的编码格式对消息体进行解码获取原始数据。可选项 UTF_8, GBK, GB18030, BIG5默认为 UTF_8
点击 **检查连通性** 按钮,检查数据源是否可用。 点击 **检查连通性** 按钮,检查数据源是否可用。
![mqtt-05.png](./mqtt-05.png) ![mqtt-05.png](./mqtt-05.png)

View File

@ -113,6 +113,8 @@ kcat <topic> \
**获取数据的最大时长** 中设置获取消息时等待数据不足的最长时间(以毫秒为单位),默认值为 100ms。 **获取数据的最大时长** 中设置获取消息时等待数据不足的最长时间(以毫秒为单位),默认值为 100ms。
**字符编码**配置消息体编码格式taosX 在接收到消息后,使用对应的编码格式对消息体进行解码获取原始数据。可选项 UTF_8, GBK, GB18030, BIG5默认为 UTF_8
点击 **连通性检查** 按钮,检查数据源是否可用。 点击 **连通性检查** 按钮,检查数据源是否可用。
![kafka-06.png](./kafka-06.png) ![kafka-06.png](./kafka-06.png)

Binary file not shown.

Before

Width:  |  Height:  |  Size: 48 KiB

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 104 KiB

After

Width:  |  Height:  |  Size: 53 KiB

View File

@ -27,24 +27,23 @@ taosd 命令行参数如下
### 连接相关 ### 连接相关
|参数名称|支持版本|参数含义| |参数名称|支持版本|参数含义|
|-----------------------|----------|-| |-----------------------|-------------------------|------------|
|firstEp | |taosd 启动时,主动连接的集群中首个 dnode 的 end point默认值 localhost:6030| |firstEp | |taosd 启动时,主动连接的集群中首个 dnode 的 end point默认值 localhost:6030|
|secondEp | |taosd 启动时,如果 firstEp 连接不上,尝试连接集群中第二个 dnode 的 endpoint无默认值| |secondEp | |taosd 启动时,如果 firstEp 连接不上,尝试连接集群中第二个 dnode 的 endpoint无默认值|
|fqdn | |taosd 监听的服务地址,默认为所在服务器上配置的第一个 hostname| |fqdn | |taosd 监听的服务地址,默认为所在服务器上配置的第一个 hostname|
|serverPort | |taosd 监听的端口,默认值 6030| |serverPort | |taosd 监听的端口,默认值 6030|
|compressMsgSize | |是否对 RPC 消息进行压缩;-1所有消息都不压缩0所有消息都压缩N (N>0):只有大于 N 个字节的消息才压缩;默认值 -1| |compressMsgSize | |是否对 RPC 消息进行压缩;-1所有消息都不压缩0所有消息都压缩N (N>0):只有大于 N 个字节的消息才压缩;默认值 -1|
|shellActivityTimer | |客户端向 mnode 发送心跳的时长,单位为秒,取值范围 1-120默认值 3| |shellActivityTimer | |客户端向 mnode 发送心跳的时长,单位为秒,取值范围 1-120默认值 3 |
|numOfRpcSessions | |RPC 支持的最大连接数,取值范围 100-100000默认值 30000| |numOfRpcSessions | |RPC 支持的最大连接数,取值范围 100-100000默认值 30000|
|numOfRpcThreads | |RPC 线程数目,默认值为 CPU 核数的一半| |numOfRpcThreads | |RPC 收发数据线程数目取值范围1-50,默认值为 CPU 核数的一半|
|numOfTaskQueueThreads | |dnode 处理 RPC 消息的线程数| |numOfTaskQueueThreads | |客户端处理 RPC 消息的线程数取值, 范围4-16,默认值为 CPU 核数的一半|
|statusInterval | |dnode 与 mnode 之间的心跳间隔| |rpcQueueMemoryAllowed | |dnode允许的已经收到的RPC消息占用的内存最大值单位 bytes取值范围 104857600-INT64_MAX默认值为服务器内存的 1/10 |
|rpcQueueMemoryAllowed | |dnode 允许的 rpc 消息占用的内存最大值,单位 bytes取值范围 104857600-INT64_MAX默认值 服务器内存的 1/10 | |resolveFQDNRetryTime | 3.x 之后取消 |FQDN 解析失败时的重试次数|
|resolveFQDNRetryTime | |FQDN 解析失败时的重试次数| |timeToGetAvailableConn | 3.3.4.x之后取消 |获得可用连接的最长等待时间,取值范围 10-50000000单位为毫秒默认值 500000|
|timeToGetAvailableConn | |获得可用连接的最长等待时间,取值范围 10-50000000单位为毫秒默认值 500000| |maxShellConns | 3.x 后取消 |允许创建的最大链接数|
|maxShellConns | |允许创建的最大链接数| |maxRetryWaitTime | |重连最大超时时间, 默认值是 10s|
|maxRetryWaitTime | |重连最大超时时间| |shareConnLimit |3.3.4.0 新增 |一个链接可以共享的请求的数目,取值范围 1-512默认值 10|
|shareConnLimit |3.3.4.3 后|内部参数,一个链接可以共享的查询数目,取值范围 1-256默认值 10| |readTimeout |3.3.4.0 新增 |单个请求最小超时时间,取值范围 64-604800单位为秒默认值 900|
|readTimeout |3.3.4.3 后|内部参数,最小超时时间,取值范围 64-604800单位为秒默认值 900|
### 监控相关 ### 监控相关
|参数名称|支持版本|参数含义| |参数名称|支持版本|参数含义|

View File

@ -10,17 +10,18 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API并且在
### 连接相关 ### 连接相关
|参数名称|支持版本|参数含义| |参数名称|支持版本|参数含义|
|----------------------|----------|-| |----------------------|----------|-------------|
|firstEp | |启动时,主动连接的集群中首个 dnode 的 endpoint缺省值hostname:6030若无法获取该服务器的 hostname则赋值为 localhost| |firstEp | |启动时,主动连接的集群中首个 dnode 的 endpoint缺省值hostname:6030若无法获取该服务器的 hostname则赋值为 localhost|
|secondEp | |启动时,如果 firstEp 连接不上,尝试连接集群中第二个 dnode 的 endpoint没有缺省值| |secondEp | |启动时,如果 firstEp 连接不上,尝试连接集群中第二个 dnode 的 endpoint没有缺省值|
|compressMsgSize | |是否对 RPC 消息进行压缩;-1所有消息都不压缩0所有消息都压缩N (N>0):只有大于 N 个字节的消息才压缩;缺省值 -1| |compressMsgSize | |是否对 RPC 消息进行压缩;-1所有消息都不压缩0所有消息都压缩N (N>0):只有大于 N 个字节的消息才压缩;缺省值 -1|
|shellActivityTimer | |客户端向 mnode 发送心跳的时长,单位为秒,取值范围 1-120默认值 3| |shellActivityTimer | |客户端向 mnode 发送心跳的时长,单位为秒,取值范围 1-120默认值 3|
|numOfRpcSessions | |RPC 支持的最大连接数,取值范围 100-100000缺省值 30000| |numOfRpcSessions | |RPC 支持的最大连接数,取值范围 100-100000缺省值 30000|
|numOfRpcThreads | |RPC 线程数目,默认值为 CPU 核数的一半| |numOfRpcThreads | |RPC 收发数据线程数目取值范围1-50,默认值为 CPU 核数的一半|
|timeToGetAvailableConn| |获得可用连接的最长等待时间,取值范围 10-50000000单位为毫秒缺省值 500000| |numOfTaskQueueThreads | |客户端处理 RPC消息的线程数, 范围4-16,默认值为 CPU 核数的一半|
|timeToGetAvailableConn| 3.3.4.*之后取消 |获得可用连接的最长等待时间,取值范围 10-50000000单位为毫秒缺省值 500000|
|useAdapter | |内部参数,是否使用 taosadapter影响 CSV 文件导入| |useAdapter | |内部参数,是否使用 taosadapter影响 CSV 文件导入|
|shareConnLimit |3.3.4.3 后|内部参数,一个链接可以共享的查询数目,取值范围 1-256默认值 10| |shareConnLimit |3.3.4.0 新增|内部参数,一个链接可以共享的查询数目,取值范围 1-256默认值 10|
|readTimeout |3.3.4.3 后|内部参数,最小超时时间,取值范围 64-604800单位为秒默认值 900| |readTimeout |3.3.4.0 新增|内部参数,最小超时时间,取值范围 64-604800单位为秒默认值 900|
### 查询相关 ### 查询相关
|参数名称|支持版本|参数含义| |参数名称|支持版本|参数含义|

View File

@ -90,7 +90,7 @@ taos -h h1.taos.com -s "use db; show tables;"
也可以通过配置文件中的参数设置来控制 TDengine CLI 的行为。可用配置参数请参考[客户端配置](../../components/taosc) 也可以通过配置文件中的参数设置来控制 TDengine CLI 的行为。可用配置参数请参考[客户端配置](../../components/taosc)
## 错误代码表 ## 错误代码表
在 TDengine 3.3.5.0 版本后 TDengine CLI 在返回的错误信息中包含了具体的错误代码,用户可到 TDengine 官网的错误代码详细说明页面查找具体原因及解决措施,见:[错误码参考表](../error_code/) 在 TDengine 3.3.4.8 版本后 TDengine CLI 在返回错误信息中返回了具体错误码,用户可到 TDengine 官网错误码页面查找具体原因及解决措施,见:[错误码参考表](https://docs.taosdata.com/reference/error-code/)
## TDengine CLI TAB 键补全 ## TDengine CLI TAB 键补全

View File

@ -62,7 +62,8 @@ window_clause: {
| COUNT_WINDOW(count_val[, sliding_val]) | COUNT_WINDOW(count_val[, sliding_val])
interp_clause: interp_clause:
RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val) RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val)
| RANGE(ts_val, surrounding_time_val) FILL(fill_mod_and_val)
partition_by_clause: partition_by_clause:
PARTITION BY partition_by_expr [, partition_by_expr] ... PARTITION BY partition_by_expr [, partition_by_expr] ...
@ -255,6 +256,13 @@ select _rowts, max(current) from meters;
select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear); select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear);
``` ```
**\_IROWTS\_ORIGIN**
`_irowts_origin` 伪列只能与 interp 函数一起使用,不支持在流计算中使用, 仅适用于FILL类型为PREV/NEXT/NEAR, 用于返回 interp 函数所使用的原始数据的时间戳列。若范围内无值, 则返回 NULL。
```sql
select _iorwts_origin, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(NEXT);
```
## 查询对象 ## 查询对象
FROM 关键字后面可以是若干个表(超级表)列表,也可以是子查询的结果。 FROM 关键字后面可以是若干个表(超级表)列表,也可以是子查询的结果。

View File

@ -1838,6 +1838,9 @@ ignore_null_values: {
- INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.2.0 版本以后支持)。 - INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.2.0 版本以后支持)。
- INTERP 可以与伪列 _isfilled 一起使用,显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0 版本以后支持)。 - INTERP 可以与伪列 _isfilled 一起使用,显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0 版本以后支持)。
- INTERP 对于带复合主键的表的查询,若存在相同时间戳的数据,则只有对应的复合主键最小的数据参与运算。 - INTERP 对于带复合主键的表的查询,若存在相同时间戳的数据,则只有对应的复合主键最小的数据参与运算。
- INTERP 查询支持NEAR FILL模式, 即当需要FILL时, 使用距离当前时间点最近的数据进行插值, 当前后时间戳与当前时间断面一样近时, FILL 前一行的值. 此模式在流计算中和窗口查询中不支持。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)。(3.3.4.9版本及以后支持)。
- INTERP 只有在使用FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`。`_irowts_origin`在3.3.4.9版本及以后支持。
- INTERP `RANEG`子句支持时间范围的扩展(3.3.4.9版本及以后支持), 如`RANGE('2023-01-01 00:00:00', 10s)`表示在时间点'2023-01-01 00:00:00'查找前后10s的数据进行插值, FILL PREV/NEXT/NEAR分别表示从时间点向前/向后/前后查找数据, 若时间点周围没有数据, 则使用FILL指定的值进行插值, 因此此时FILL子句必须指定值。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1). 目前仅支持时间点和时间范围的组合, 不支持时间区间和时间范围的组合, 即不支持RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h)。所指定的时间范围规则与EVERY类似, 单位不能是年或月, 值不能为0, 不能带引号。使用该扩展时, 不支持除FILL PREV/NEXT/NEAR外的其他FILL模式, 且不能指定EVERY子句。
### LAST ### LAST

View File

@ -189,6 +189,7 @@ typedef enum _mgmt_table {
#define TSDB_FILL_LINEAR 5 #define TSDB_FILL_LINEAR 5
#define TSDB_FILL_PREV 6 #define TSDB_FILL_PREV 6
#define TSDB_FILL_NEXT 7 #define TSDB_FILL_NEXT 7
#define TSDB_FILL_NEAR 8
#define TSDB_ALTER_USER_PASSWD 0x1 #define TSDB_ALTER_USER_PASSWD 0x1
#define TSDB_ALTER_USER_SUPERUSER 0x2 #define TSDB_ALTER_USER_SUPERUSER 0x2
@ -265,6 +266,7 @@ typedef enum ENodeType {
QUERY_NODE_COLUMN_OPTIONS, QUERY_NODE_COLUMN_OPTIONS,
QUERY_NODE_TSMA_OPTIONS, QUERY_NODE_TSMA_OPTIONS,
QUERY_NODE_ANOMALY_WINDOW, QUERY_NODE_ANOMALY_WINDOW,
QUERY_NODE_RANGE_AROUND,
// Statement nodes are used in parser and planner module. // Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100, QUERY_NODE_SET_OPERATOR = 100,

View File

@ -155,6 +155,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_FORECAST_LOW, FUNCTION_TYPE_FORECAST_LOW,
FUNCTION_TYPE_FORECAST_HIGH, FUNCTION_TYPE_FORECAST_HIGH,
FUNCTION_TYPE_FORECAST_ROWTS, FUNCTION_TYPE_FORECAST_ROWTS,
FUNCTION_TYPE_IROWTS_ORIGIN,
// internal function // internal function
FUNCTION_TYPE_SELECT_VALUE = 3750, FUNCTION_TYPE_SELECT_VALUE = 3750,
@ -289,6 +290,7 @@ bool fmIsPrimaryKeyFunc(int32_t funcId);
bool fmIsProcessByRowFunc(int32_t funcId); bool fmIsProcessByRowFunc(int32_t funcId);
bool fmisSelectGroupConstValueFunc(int32_t funcId); bool fmisSelectGroupConstValueFunc(int32_t funcId);
bool fmIsElapsedFunc(int32_t funcId); bool fmIsElapsedFunc(int32_t funcId);
bool fmIsRowTsOriginFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes); void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc); int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);

View File

@ -213,6 +213,8 @@ typedef struct SInterpFuncLogicNode {
EFillMode fillMode; EFillMode fillMode;
SNode* pFillValues; // SNodeListNode SNode* pFillValues; // SNodeListNode
SNode* pTimeSeries; // SColumnNode SNode* pTimeSeries; // SColumnNode
int64_t rangeInterval;
int8_t rangeIntervalUnit;
SStreamNodeOption streamNodeOption; SStreamNodeOption streamNodeOption;
} SInterpFuncLogicNode; } SInterpFuncLogicNode;
@ -528,6 +530,8 @@ typedef struct SInterpFuncPhysiNode {
SNode* pFillValues; // SNodeListNode SNode* pFillValues; // SNodeListNode
SNode* pTimeSeries; // SColumnNode SNode* pTimeSeries; // SColumnNode
SStreamNodeOption streamNodeOption; SStreamNodeOption streamNodeOption;
int64_t rangeInterval;
int8_t rangeIntervalUnit;
} SInterpFuncPhysiNode; } SInterpFuncPhysiNode;
typedef SInterpFuncPhysiNode SStreamInterpFuncPhysiNode; typedef SInterpFuncPhysiNode SStreamInterpFuncPhysiNode;

View File

@ -362,7 +362,8 @@ typedef enum EFillMode {
FILL_MODE_NULL, FILL_MODE_NULL,
FILL_MODE_NULL_F, FILL_MODE_NULL_F,
FILL_MODE_LINEAR, FILL_MODE_LINEAR,
FILL_MODE_NEXT FILL_MODE_NEXT,
FILL_MODE_NEAR,
} EFillMode; } EFillMode;
typedef enum ETimeLineMode { typedef enum ETimeLineMode {
@ -407,6 +408,11 @@ typedef struct SWindowOffsetNode {
SNode* pEndOffset; // SValueNode SNode* pEndOffset; // SValueNode
} SWindowOffsetNode; } SWindowOffsetNode;
typedef struct SRangeAroundNode {
ENodeType type;
SNode* pTimepoint;
SNode* pInterval;
} SRangeAroundNode;
typedef struct SSelectStmt { typedef struct SSelectStmt {
ENodeType type; // QUERY_NODE_SELECT_STMT ENodeType type; // QUERY_NODE_SELECT_STMT
@ -421,6 +427,7 @@ typedef struct SSelectStmt {
SNodeList* pGroupByList; // SGroupingSetNode SNodeList* pGroupByList; // SGroupingSetNode
SNode* pHaving; SNode* pHaving;
SNode* pRange; SNode* pRange;
SNode* pRangeAround;
SNode* pEvery; SNode* pEvery;
SNode* pFill; SNode* pFill;
SNodeList* pOrderByList; // SOrderByExprNode SNodeList* pOrderByList; // SOrderByExprNode

View File

@ -27,7 +27,7 @@ extern "C" {
typedef struct SStmtCallback { typedef struct SStmtCallback {
TAOS_STMT* pStmt; TAOS_STMT* pStmt;
int32_t (*getTbNameFn)(TAOS_STMT*, char**); int32_t (*getTbNameFn)(TAOS_STMT*, char**);
int32_t (*setInfoFn)(TAOS_STMT*, STableMeta*, void*, SName*, bool, SHashObj*, SHashObj*, const char*); int32_t (*setInfoFn)(TAOS_STMT*, STableMeta*, void*, SName*, bool, SHashObj*, SHashObj*, const char*, bool);
int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**); int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**);
} SStmtCallback; } SStmtCallback;
@ -147,7 +147,8 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum); int32_t colIdx, int32_t rowNum);
int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBuildStmtStbColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_STB** fields); int32_t qBuildStmtStbColFields(void* pBlock, void* boundTags, bool hasCtbName, int32_t* fieldNum,
TAOS_FIELD_STB** fields);
int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
@ -177,8 +178,8 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen); char* msgBuf, int32_t msgBufLen);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields, int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw); int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut); int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);

View File

@ -93,7 +93,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream
const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen); const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen);
SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen);
int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); int32_t recoverSession(SStreamFileState* pFileState, int64_t ckId);
void sessionWinStateClear(SStreamFileState* pFileState); void sessionWinStateClear(SStreamFileState* pFileState);
void sessionWinStateCleanup(void* pBuff); void sessionWinStateCleanup(void* pBuff);

45
packaging/smokeTest/test_smoking_selfhost.sh Normal file → Executable file
View File

@ -4,11 +4,13 @@
LOG_FILE="test_server.log" LOG_FILE="test_server.log"
SUCCESS_FILE="success.txt" SUCCESS_FILE="success.txt"
FAILED_FILE="failed.txt" FAILED_FILE="failed.txt"
REPORT_FILE="report.txt"
# Initialize/clear result files # Initialize/clear result files
> "$SUCCESS_FILE" > "$SUCCESS_FILE"
> "$FAILED_FILE" > "$FAILED_FILE"
> "$LOG_FILE" > "$LOG_FILE"
> "$REPORT_FILE"
# Switch to the target directory # Switch to the target directory
TARGET_DIR="../../tests/system-test/" TARGET_DIR="../../tests/system-test/"
@ -22,18 +24,17 @@ else
exit 1 exit 1
fi fi
# Define the Python commands to execute case list # Define the Python commands to execute
commands=( commands=(
"python3 ./test.py -f 2-query/join.py" "python3 ./test.py -f 2-query/join.py"
"python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3"
"python3 ./test.py -f 1-insert/insert_column_value.py" "python3 ./test.py -f 1-insert/insert_column_value.py"
"python3 ./test.py -f 2-query/primary_ts_base_5.py" "python3 ./test.py -f 2-query/primary_ts_base_5.py"
"python3 ./test.py -f 2-query/case_when.py" "python3 ./test.py -f 2-query/case_when.py"
"python3 ./test.py -f 2-query/partition_limit_interval.py" "python3 ./test.py -f 2-query/partition_limit_interval.py"
"python3 ./test.py -f 2-query/fill.py" "python3 ./test.py -f 2-query/fill.py"
"python3 ./test.py -f query/query_basic.py -N 3"
"python3 ./test.py -f 7-tmq/basic5.py" "python3 ./test.py -f 7-tmq/basic5.py"
"python3 ./test.py -f 8-stream/stream_basic.py" "python3 ./test.py -f 8-stream/stream_basic.py"
"python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3"
) )
# Counters # Counters
@ -45,6 +46,7 @@ fail_count=0
for cmd in "${commands[@]}" for cmd in "${commands[@]}"
do do
echo "===== Executing Command: $cmd =====" | tee -a "$LOG_FILE" echo "===== Executing Command: $cmd =====" | tee -a "$LOG_FILE"
# Execute the command and append output and errors to the log file # Execute the command and append output and errors to the log file
eval "$cmd" >> "$LOG_FILE" 2>&1 eval "$cmd" >> "$LOG_FILE" 2>&1
exit_code=$? exit_code=$?
@ -58,6 +60,7 @@ do
echo "$cmd" >> "$FAILED_FILE" echo "$cmd" >> "$FAILED_FILE"
((fail_count++)) ((fail_count++))
fi fi
echo "" | tee -a "$LOG_FILE" # Add an empty line for separation echo "" | tee -a "$LOG_FILE" # Add an empty line for separation
done done
@ -72,23 +75,31 @@ if [ $fail_count -ne 0 ]; then
echo "The following commands failed:" | tee -a "$LOG_FILE" echo "The following commands failed:" | tee -a "$LOG_FILE"
cat "$FAILED_FILE" | tee -a "$LOG_FILE" cat "$FAILED_FILE" | tee -a "$LOG_FILE"
else else
echo "All commands executed successfully." | tee -a "$LOG_FILE" echo "All commands executed successfully. Deleting log and result files..." | tee -a "$LOG_FILE"
rm -f "$LOG_FILE" "$SUCCESS_FILE" "$FAILED_FILE" "$REPORT_FILE"
echo "Log and result files deleted."
fi fi
# Optional: Generate a separate report file # Generate a separate report file if there are failed commands
echo "" > "report.txt"
echo "===== Test Report =====" >> "report.txt"
echo "Total Commands Executed: $total" >> "report.txt"
echo "Successful: $success_count" >> "report.txt"
echo "Failed: $fail_count" >> "report.txt"
if [ $fail_count -ne 0 ]; then if [ $fail_count -ne 0 ]; then
echo "" >> "report.txt" echo "" >> "$REPORT_FILE"
echo "The following commands failed:" >> "report.txt" echo "===== Test Report =====" >> "$REPORT_FILE"
cat "$FAILED_FILE" >> "report.txt" echo "Total Commands Executed: $total" >> "$REPORT_FILE"
echo "Successful: $success_count" >> "$REPORT_FILE"
echo "Failed: $fail_count" >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
echo "The following commands failed:" >> "$REPORT_FILE"
cat "$FAILED_FILE" >> "$REPORT_FILE"
else else
echo "All commands executed successfully." >> "report.txt" echo "===== Test Report =====" > "$REPORT_FILE"
echo "Total Commands Executed: $total" >> "$REPORT_FILE"
echo "Successful: $success_count" >> "$REPORT_FILE"
echo "Failed: $fail_count" >> "$REPORT_FILE"
echo "All commands executed successfully." >> "$REPORT_FILE"
fi fi
echo "Detailed logs can be found in $LOG_FILE" # Print the absolute paths of the log and result files
echo "Test report can be found in report.txt" echo "Detailed logs can be found in: $(realpath "$LOG_FILE")"
echo "Successful commands can be found in: $(realpath "$SUCCESS_FILE")"
echo "Failed commands can be found in: $(realpath "$FAILED_FILE")"
echo "Test report can be found in: $(realpath "$REPORT_FILE")"

View File

@ -97,6 +97,7 @@ else
${build_dir}/bin/${clientName} \ ${build_dir}/bin/${clientName} \
${taostools_bin_files} \ ${taostools_bin_files} \
${build_dir}/bin/${clientName}adapter \ ${build_dir}/bin/${clientName}adapter \
${build_dir}/bin/${clientName}keeper \
${build_dir}/bin/udfd \ ${build_dir}/bin/udfd \
${script_dir}/remove.sh \ ${script_dir}/remove.sh \
${script_dir}/set_core.sh \ ${script_dir}/set_core.sh \
@ -138,10 +139,16 @@ mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${
if [ -f "${compile_dir}/test/cfg/${clientName}adapter.toml" ]; then if [ -f "${compile_dir}/test/cfg/${clientName}adapter.toml" ]; then
cp ${compile_dir}/test/cfg/${clientName}adapter.toml ${install_dir}/cfg || : cp ${compile_dir}/test/cfg/${clientName}adapter.toml ${install_dir}/cfg || :
fi fi
if [ -f "${compile_dir}/test/cfg/${clientName}keeper.toml" ]; then
cp ${compile_dir}/test/cfg/${clientName}keeper.toml ${install_dir}/cfg || :
fi
if [ -f "${compile_dir}/test/cfg/${clientName}adapter.service" ]; then if [ -f "${compile_dir}/test/cfg/${clientName}adapter.service" ]; then
cp ${compile_dir}/test/cfg/${clientName}adapter.service ${install_dir}/cfg || : cp ${compile_dir}/test/cfg/${clientName}adapter.service ${install_dir}/cfg || :
fi fi
if [ -f "${compile_dir}/test/cfg/${clientName}keeper.service" ]; then
cp ${compile_dir}/test/cfg/${clientName}keeper.service ${install_dir}/cfg || :
fi
if [ -f "${cfg_dir}/${serverName}.service" ]; then if [ -f "${cfg_dir}/${serverName}.service" ]; then
cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || : cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || :
@ -422,19 +429,19 @@ if [ "$exitcode" != "0" ]; then
exit $exitcode exit $exitcode
fi fi
if [ -n "${taostools_bin_files}" ] && [ "$verMode" != "cloud" ]; then # if [ -n "${taostools_bin_files}" ] && [ "$verMode" != "cloud" ]; then
wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh" # wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh"
if [ "$osType" != "Darwin" ]; then # if [ "$osType" != "Darwin" ]; then
tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || : # tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || :
else # else
tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" || : # tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" || :
rm -rf ${taostools_install_dir} ||: # rm -rf ${taostools_install_dir} ||:
fi # fi
exitcode=$? # exitcode=$?
if [ "$exitcode" != "0" ]; then # if [ "$exitcode" != "0" ]; then
echo "tar ${taostools_pkg_name}.tar.gz error !!!" # echo "tar ${taostools_pkg_name}.tar.gz error !!!"
exit $exitcode # exit $exitcode
fi # fi
fi # fi
cd ${curr_dir} cd ${curr_dir}

View File

@ -64,12 +64,12 @@ typedef struct SStmtBindInfo {
int32_t sBindLastIdx; int32_t sBindLastIdx;
int8_t tbType; int8_t tbType;
bool tagsCached; bool tagsCached;
bool preCtbname;
void *boundTags; void *boundTags;
char tbName[TSDB_TABLE_FNAME_LEN]; char tbName[TSDB_TABLE_FNAME_LEN];
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
char stbFName[TSDB_TABLE_FNAME_LEN]; char stbFName[TSDB_TABLE_FNAME_LEN];
SName sname; SName sname;
char statbName[TSDB_TABLE_FNAME_LEN]; char statbName[TSDB_TABLE_FNAME_LEN];
} SStmtBindInfo; } SStmtBindInfo;
@ -133,7 +133,6 @@ typedef struct SStmtQueue {
uint64_t qRemainNum; uint64_t qRemainNum;
} SStmtQueue; } SStmtQueue;
typedef struct STscStmt { typedef struct STscStmt {
STscObj *taos; STscObj *taos;
SCatalog *pCatalog; SCatalog *pCatalog;
@ -204,7 +203,6 @@ extern char *gStmtStatusStr[];
} \ } \
} while (0) } while (0)
#define STMT_FLOG(param, ...) qFatal("stmt:%p " param, pStmt, __VA_ARGS__) #define STMT_FLOG(param, ...) qFatal("stmt:%p " param, pStmt, __VA_ARGS__)
#define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__) #define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__)
#define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__) #define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__)

View File

@ -260,7 +260,7 @@ int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockH
} }
int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, bool autoCreateTbl, int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, bool autoCreateTbl,
SHashObj* pVgHash, SHashObj* pBlockHash, const char* sTableName) { SHashObj* pVgHash, SHashObj* pBlockHash, const char* sTableName, bool preCtbname) {
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbName, sTableName, autoCreateTbl)); STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbName, sTableName, autoCreateTbl));

View File

@ -178,7 +178,7 @@ static int32_t stmtGetTbName(TAOS_STMT2* stmt, char** tbName) {
} }
static int32_t stmtUpdateBindInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, static int32_t stmtUpdateBindInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void* tags, SName* tbName,
const char* sTableName, bool autoCreateTbl) { const char* sTableName, bool autoCreateTbl, bool preCtbname) {
STscStmt2* pStmt = (STscStmt2*)stmt; STscStmt2* pStmt = (STscStmt2*)stmt;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
int32_t code = tNameExtractFullName(tbName, tbFName); int32_t code = tNameExtractFullName(tbName, tbFName);
@ -202,6 +202,7 @@ static int32_t stmtUpdateBindInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void
pStmt->bInfo.boundTags = tags; pStmt->bInfo.boundTags = tags;
pStmt->bInfo.tagsCached = false; pStmt->bInfo.tagsCached = false;
pStmt->bInfo.preCtbname = preCtbname;
tstrncpy(pStmt->bInfo.stbFName, sTableName, sizeof(pStmt->bInfo.stbFName)); tstrncpy(pStmt->bInfo.stbFName, sTableName, sizeof(pStmt->bInfo.stbFName));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -217,10 +218,10 @@ static int32_t stmtUpdateExecInfo(TAOS_STMT2* stmt, SHashObj* pVgHash, SHashObj*
} }
static int32_t stmtUpdateInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, bool autoCreateTbl, static int32_t stmtUpdateInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, bool autoCreateTbl,
SHashObj* pVgHash, SHashObj* pBlockHash, const char* sTableName) { SHashObj* pVgHash, SHashObj* pBlockHash, const char* sTableName, bool preCtbname) {
STscStmt2* pStmt = (STscStmt2*)stmt; STscStmt2* pStmt = (STscStmt2*)stmt;
STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbName, sTableName, autoCreateTbl)); STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbName, sTableName, autoCreateTbl, preCtbname));
STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash)); STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash));
pStmt->sql.autoCreateTbl = autoCreateTbl; pStmt->sql.autoCreateTbl = autoCreateTbl;
@ -1092,7 +1093,7 @@ static int stmtFetchStbColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIEL
} }
} }
STMT_ERR_RET(qBuildStmtStbColFields(*pDataBlock, fieldNum, fields)); STMT_ERR_RET(qBuildStmtStbColFields(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.preCtbname, fieldNum, fields));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

View File

@ -475,6 +475,7 @@ typedef struct SStreamFillSupporter {
STimeWindow winRange; STimeWindow winRange;
int32_t pkColBytes; int32_t pkColBytes;
__compar_fn_t comparePkColFn; __compar_fn_t comparePkColFn;
int32_t* pOffsetInfo;
} SStreamFillSupporter; } SStreamFillSupporter;
typedef struct SStreamScanInfo { typedef struct SStreamScanInfo {
@ -822,10 +823,11 @@ typedef struct SStreamFillOperatorInfo {
int32_t primaryTsCol; int32_t primaryTsCol;
int32_t primarySrcSlotId; int32_t primarySrcSlotId;
SStreamFillInfo* pFillInfo; SStreamFillInfo* pFillInfo;
SStreamAggSupporter* pStreamAggSup;
SArray* pCloseTs; SArray* pCloseTs;
SArray* pUpdated; SArray* pUpdated;
SGroupResInfo groupResInfo; SGroupResInfo groupResInfo;
SStreamState* pState;
SStateStore stateStore;
} SStreamFillOperatorInfo; } SStreamFillOperatorInfo;
typedef struct SStreamTimeSliceOperatorInfo { typedef struct SStreamTimeSliceOperatorInfo {
@ -884,6 +886,7 @@ typedef struct SStreamIntervalSliceOperatorInfo {
struct SOperatorInfo* pOperator; struct SOperatorInfo* pOperator;
bool hasFill; bool hasFill;
bool hasInterpoFunc; bool hasInterpoFunc;
int32_t* pOffsetInfo;
} SStreamIntervalSliceOperatorInfo; } SStreamIntervalSliceOperatorInfo;
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
@ -1059,6 +1062,7 @@ void destroyFlusedPos(void* pRes);
bool isIrowtsPseudoColumn(SExprInfo* pExprInfo); bool isIrowtsPseudoColumn(SExprInfo* pExprInfo);
bool isIsfilledPseudoColumn(SExprInfo* pExprInfo); bool isIsfilledPseudoColumn(SExprInfo* pExprInfo);
bool isInterpFunc(SExprInfo* pExprInfo); bool isInterpFunc(SExprInfo* pExprInfo);
bool isIrowtsOriginPseudoColumn(SExprInfo* pExprInfo);
int32_t encodeSSessionKey(void** buf, SSessionKey* key); int32_t encodeSSessionKey(void** buf, SSessionKey* key);
void* decodeSSessionKey(void* buf, SSessionKey* key); void* decodeSSessionKey(void* buf, SSessionKey* key);

View File

@ -150,7 +150,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo); int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo); int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo); int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);

View File

@ -90,19 +90,21 @@ int32_t saveTimeSliceWinResult(SWinKey* pKey, SSHashObj* pUpdatedMap);
int winPosCmprImpl(const void* pKey1, const void* pKey2); int winPosCmprImpl(const void* pKey1, const void* pKey2);
void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI);
SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index); SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index, int32_t* pCellOffsetInfo);
int32_t getDownstreamRes(struct SOperatorInfo* downstream, SSDataBlock** ppRes, SColumnInfo** ppPkCol); int32_t getDownstreamRes(struct SOperatorInfo* downstream, SSDataBlock** ppRes, SColumnInfo** ppPkCol);
void destroyFlusedppPos(void* ppRes); void destroyFlusedppPos(void* ppRes);
void doBuildStreamIntervalResult(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, void doBuildStreamIntervalResult(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
SGroupResInfo* pGroupResInfo); SGroupResInfo* pGroupResInfo);
void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal, void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal,
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol); int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol, int32_t* pCellOffsetInfo);
int32_t getQualifiedRowNumDesc(SExprSupp* pExprSup, SSDataBlock* pBlock, TSKEY* tsCols, int32_t rowId, bool ignoreNull); int32_t getQualifiedRowNumDesc(SExprSupp* pExprSup, SSDataBlock* pBlock, TSKEY* tsCols, int32_t rowId, bool ignoreNull);
int32_t createStreamIntervalSliceOperatorInfo(struct SOperatorInfo* downstream, SPhysiNode* pPhyNode, int32_t createStreamIntervalSliceOperatorInfo(struct SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
struct SOperatorInfo** ppOptInfo); struct SOperatorInfo** ppOptInfo);
int32_t buildAllResultKey(SStreamAggSupporter* pAggSup, TSKEY ts, SArray* pUpdated); int32_t buildAllResultKey(SStateStore* pStateStore, SStreamState* pState, TSKEY ts, SArray* pUpdated);
int32_t initOffsetInfo(int32_t** ppOffset, SSDataBlock* pRes);
TSKEY compareTs(void* pKey);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -119,7 +119,8 @@ typedef struct SStreamFillInfo {
int32_t delIndex; int32_t delIndex;
uint64_t curGroupId; uint64_t curGroupId;
bool hasNext; bool hasNext;
SResultRowData* pNonFillRow; SResultRowData* pNonFillRow;
void* pTempBuff;
} SStreamFillInfo; } SStreamFillInfo;
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);

View File

@ -2390,6 +2390,9 @@ int32_t convertFillType(int32_t mode) {
case FILL_MODE_LINEAR: case FILL_MODE_LINEAR:
type = TSDB_FILL_LINEAR; type = TSDB_FILL_LINEAR;
break; break;
case FILL_MODE_NEAR:
type = TSDB_FILL_NEAR;
break;
default: default:
type = TSDB_FILL_NONE; type = TSDB_FILL_NONE;
} }

View File

@ -614,7 +614,7 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
} else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) {
code = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo, &pOptr); code = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL == type) {
code = createStreamFillOperatorInfo(ops[0], (SStreamFillPhysiNode*)pPhyNode, pTaskInfo, &pOptr); code = createStreamFillOperatorInfo(ops[0], (SStreamFillPhysiNode*)pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) {
code = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr); code = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) {

View File

@ -100,6 +100,8 @@ void destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
taosMemoryFree(pFillSup->next.pRowVal); taosMemoryFree(pFillSup->next.pRowVal);
taosMemoryFree(pFillSup->nextNext.pRowVal); taosMemoryFree(pFillSup->nextNext.pRowVal);
taosMemoryFree(pFillSup->pOffsetInfo);
taosMemoryFree(pFillSup); taosMemoryFree(pFillSup);
} }
@ -129,6 +131,7 @@ void destroyStreamFillInfo(SStreamFillInfo* pFillInfo) {
pFillInfo->pLinearInfo = NULL; pFillInfo->pLinearInfo = NULL;
taosArrayDestroy(pFillInfo->delRanges); taosArrayDestroy(pFillInfo->delRanges);
taosMemoryFreeClear(pFillInfo->pTempBuff);
taosMemoryFree(pFillInfo); taosMemoryFree(pFillInfo);
} }
@ -148,6 +151,14 @@ static void destroyStreamFillOperatorInfo(void* param) {
clearGroupResInfo(&pInfo->groupResInfo); clearGroupResInfo(&pInfo->groupResInfo);
taosArrayDestroy(pInfo->pCloseTs); taosArrayDestroy(pInfo->pCloseTs);
if (pInfo->stateStore.streamFileStateDestroy != NULL) {
pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState);
}
if (pInfo->pState != NULL) {
taosMemoryFreeClear(pInfo->pState);
}
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
} }
@ -1157,14 +1168,19 @@ _end:
return code; return code;
} }
static void resetForceFillWindow(SResultRowData* pRowData) {
pRowData->key = INT64_MIN;
pRowData->pRowVal = NULL;
}
void doBuildForceFillResultImpl(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, void doBuildForceFillResultImpl(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup,
SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0; int32_t lino = 0;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
void* pState = pOperator->pTaskInfo->streamInfo.pState; SStreamFillOperatorInfo* pInfo = pOperator->info;
bool res = false; bool res = false;
int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
for (; pGroupResInfo->index < numOfRows; pGroupResInfo->index++) { for (; pGroupResInfo->index < numOfRows; pGroupResInfo->index++) {
SWinKey* pKey = (SWinKey*)taosArrayGet(pGroupResInfo->pRows, pGroupResInfo->index); SWinKey* pKey = (SWinKey*)taosArrayGet(pGroupResInfo->pRows, pGroupResInfo->index);
if (pBlock->info.id.groupId == 0) { if (pBlock->info.id.groupId == 0) {
@ -1172,25 +1188,30 @@ void doBuildForceFillResultImpl(SOperatorInfo* pOperator, SStreamFillSupporter*
} else if (pBlock->info.id.groupId != pKey->groupId) { } else if (pBlock->info.id.groupId != pKey->groupId) {
break; break;
} }
void* val = NULL;
int32_t len = 0; SRowBuffPos* pValPos = NULL;
int32_t winCode = pAPI->stateStore.streamStateFillGet(pOperator->pTaskInfo->streamInfo.pState, pKey, (void**)&val, &len, NULL); int32_t len = 0;
int32_t winCode = TSDB_CODE_SUCCESS;
code = pInfo->stateStore.streamStateFillGet(pInfo->pState, pKey, (void**)&pValPos, &len, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== build force fill res. key:%" PRId64 ",groupId:%" PRId64".res:%d", pKey->ts, pKey->groupId, winCode); qDebug("===stream=== build force fill res. key:%" PRId64 ",groupId:%" PRId64".res:%d", pKey->ts, pKey->groupId, winCode);
if (winCode == TSDB_CODE_SUCCESS) { if (winCode == TSDB_CODE_SUCCESS) {
pFillSup->cur.key = pKey->ts; pFillSup->cur.key = pKey->ts;
pFillSup->cur.pRowVal = val; pFillSup->cur.pRowVal = pValPos->pRowBuff;
code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res); code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
resetFillWindow(&pFillSup->cur); resetForceFillWindow(&pFillSup->cur);
releaseOutputBuf(pInfo->pState, pValPos, &pInfo->stateStore);
} else { } else {
SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, pKey); SWinKey preKey = {.ts = INT64_MIN, .groupId = pKey->groupId};
SWinKey preKey = {.ts = INT64_MIN, .groupId = pKey->groupId}; SRowBuffPos* prePos = NULL;
void* preVal = NULL; int32_t preVLen = 0;
int32_t preVLen = 0; code = pInfo->stateStore.streamStateFillGetPrev(pInfo->pState, pKey, &preKey,
winCode = pAPI->stateStore.streamStateFillGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); (void**)&prePos, &preVLen, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode == TSDB_CODE_SUCCESS) { if (winCode == TSDB_CODE_SUCCESS) {
pFillSup->cur.key = pKey->ts; pFillSup->cur.key = pKey->ts;
pFillSup->cur.pRowVal = preVal; pFillSup->cur.pRowVal = prePos->pRowBuff;
if (pFillInfo->type == TSDB_FILL_PREV) { if (pFillInfo->type == TSDB_FILL_PREV) {
code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res); code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
@ -1200,9 +1221,9 @@ void doBuildForceFillResultImpl(SOperatorInfo* pOperator, SStreamFillSupporter*
code = buildFillResult(pFillInfo->pResRow, pFillSup, pKey->ts, pBlock, &res); code = buildFillResult(pFillInfo->pResRow, pFillSup, pKey->ts, pBlock, &res);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
} }
resetFillWindow(&pFillSup->cur); resetForceFillWindow(&pFillSup->cur);
} }
pAPI->stateStore.streamStateFreeCur(pCur); releaseOutputBuf(pInfo->pState, prePos, &pInfo->stateStore);
} }
} }
@ -1247,6 +1268,45 @@ _end:
return code; return code;
} }
static void keepResultInStateBuf(SStreamFillOperatorInfo* pInfo, uint64_t groupId, SResultRowData* pRow) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SWinKey key = {.groupId = groupId, .ts = pRow->key};
int32_t curVLen = 0;
SRowBuffPos* pStatePos = NULL;
int32_t winCode = TSDB_CODE_SUCCESS;
code = pInfo->stateStore.streamStateFillAddIfNotExist(pInfo->pState, &key, (void**)&pStatePos,
&curVLen, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
memcpy(pStatePos->pRowBuff, pRow->pRowVal, pInfo->pFillSup->rowSize);
qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
}
}
int32_t keepBlockRowInStateBuf(SStreamFillOperatorInfo* pInfo, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol,
int32_t rowId, uint64_t groupId, int32_t rowSize) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
TSKEY ts = tsCol[rowId];
pFillInfo->nextRowKey = ts;
TAOS_MEMSET(pFillInfo->pTempBuff, 0, rowSize);
SResultRowData tmpNextRow = {.key = ts, .pRowVal = pFillInfo->pTempBuff};
transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow);
keepResultInStateBuf(pInfo, groupId, &tmpNextRow);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
// force window close impl // force window close impl
static int32_t doStreamForceFillImpl(SOperatorInfo* pOperator) { static int32_t doStreamForceFillImpl(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -1257,11 +1317,10 @@ static int32_t doStreamForceFillImpl(SOperatorInfo* pOperator) {
SStreamFillInfo* pFillInfo = pInfo->pFillInfo; SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
SSDataBlock* pBlock = pInfo->pSrcBlock; SSDataBlock* pBlock = pInfo->pSrcBlock;
uint64_t groupId = pBlock->info.id.groupId; uint64_t groupId = pBlock->info.id.groupId;
SStreamAggSupporter* pAggSup = pInfo->pStreamAggSup;
SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol);
TSKEY* tsCol = (TSKEY*)pTsCol->pData; TSKEY* tsCol = (TSKEY*)pTsCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++){ for (int32_t i = 0; i < pBlock->info.rows; i++){
code = keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, i, groupId, pFillSup->rowSize); code = keepBlockRowInStateBuf(pInfo, pFillInfo, pBlock, tsCol, i, groupId, pFillSup->rowSize);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
int32_t size = taosArrayGetSize(pInfo->pCloseTs); int32_t size = taosArrayGetSize(pInfo->pCloseTs);
@ -1281,7 +1340,7 @@ static int32_t doStreamForceFillImpl(SOperatorInfo* pOperator) {
} }
} }
} }
code = pAggSup->stateStore.streamStateGroupPut(pAggSup->pState, groupId, NULL, 0); code = pInfo->stateStore.streamStateGroupPut(pInfo->pState, groupId, NULL, 0);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
_end: _end:
@ -1291,13 +1350,13 @@ _end:
return code; return code;
} }
int32_t buildAllResultKey(SStreamAggSupporter* pAggSup, TSKEY ts, SArray* pUpdated) { int32_t buildAllResultKey(SStateStore* pStateStore, SStreamState* pState, TSKEY ts, SArray* pUpdated) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0; int32_t lino = 0;
int64_t groupId = 0; int64_t groupId = 0;
SStreamStateCur* pCur = pAggSup->stateStore.streamStateGroupGetCur(pAggSup->pState); SStreamStateCur* pCur = pStateStore->streamStateGroupGetCur(pState);
while (1) { while (1) {
int32_t winCode = pAggSup->stateStore.streamStateGroupGetKVByCur(pCur, &groupId, NULL, NULL); int32_t winCode = pStateStore->streamStateGroupGetKVByCur(pCur, &groupId, NULL, NULL);
if (winCode != TSDB_CODE_SUCCESS) { if (winCode != TSDB_CODE_SUCCESS) {
break; break;
} }
@ -1305,14 +1364,14 @@ int32_t buildAllResultKey(SStreamAggSupporter* pAggSup, TSKEY ts, SArray* pUpdat
void* pPushRes = taosArrayPush(pUpdated, &key); void* pPushRes = taosArrayPush(pUpdated, &key);
QUERY_CHECK_NULL(pPushRes, code, lino, _end, terrno); QUERY_CHECK_NULL(pPushRes, code, lino, _end, terrno);
pAggSup->stateStore.streamStateGroupCurNext(pCur); pStateStore->streamStateGroupCurNext(pCur);
} }
pAggSup->stateStore.streamStateFreeCur(pCur); pStateStore->streamStateFreeCur(pCur);
pCur = NULL; pCur = NULL;
_end: _end:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pAggSup->stateStore.streamStateFreeCur(pCur); pStateStore->streamStateFreeCur(pCur);
pCur = NULL; pCur = NULL;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
} }
@ -1345,7 +1404,8 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
(*ppRes) = resBlock; (*ppRes) = resBlock;
goto _end; goto _end;
} }
pInfo->pStreamAggSup->stateStore.streamStateClearExpiredState(pInfo->pStreamAggSup->pState);
pInfo->stateStore.streamStateClearExpiredState(pInfo->pState);
setStreamOperatorCompleted(pOperator); setStreamOperatorCompleted(pOperator);
(*ppRes) = NULL; (*ppRes) = NULL;
goto _end; goto _end;
@ -1372,7 +1432,11 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
pInfo->srcRowIndex = -1; pInfo->srcRowIndex = -1;
} break; } break;
case STREAM_CHECKPOINT: case STREAM_CHECKPOINT: {
pInfo->stateStore.streamStateCommit(pInfo->pState);
(*ppRes) = pBlock;
goto _end;
} break;
case STREAM_CREATE_CHILD_TABLE: { case STREAM_CREATE_CHILD_TABLE: {
(*ppRes) = pBlock; (*ppRes) = pBlock;
goto _end; goto _end;
@ -1393,7 +1457,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pCloseTs); i++) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pCloseTs); i++) {
TSKEY ts = *(TSKEY*) taosArrayGet(pInfo->pCloseTs, i); TSKEY ts = *(TSKEY*) taosArrayGet(pInfo->pCloseTs, i);
code = buildAllResultKey(pInfo->pStreamAggSup, ts, pInfo->pUpdated); code = buildAllResultKey(&pInfo->stateStore, pInfo->pState, ts, pInfo->pUpdated);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
} }
taosArrayClear(pInfo->pCloseTs); taosArrayClear(pInfo->pCloseTs);
@ -1412,7 +1476,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
if ((*ppRes) == NULL) { if ((*ppRes) == NULL) {
pInfo->pStreamAggSup->stateStore.streamStateClearExpiredState(pInfo->pStreamAggSup->pState); pInfo->stateStore.streamStateClearExpiredState(pInfo->pState);
setStreamOperatorCompleted(pOperator); setStreamOperatorCompleted(pOperator);
} }
@ -1573,10 +1637,7 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->pResRow->key = INT64_MIN; pFillInfo->pResRow->key = INT64_MIN;
pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize);
if (!pFillInfo->pResRow->pRowVal) { QUERY_CHECK_NULL(pFillInfo->pResRow->pRowVal, code, lino, _end, terrno);
code = terrno;
QUERY_CHECK_CODE(code, lino, _end);
}
for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i);
@ -1590,6 +1651,21 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pCell->type = pColData->info.type; pCell->type = pColData->info.type;
} }
int32_t numOfResCol = taosArrayGetSize(pRes->pDataBlock);
if (numOfResCol < pFillSup->numOfAllCols) {
int32_t* pTmpBuf = (int32_t*)taosMemoryRealloc(pFillSup->pOffsetInfo, pFillSup->numOfAllCols * sizeof(int32_t));
QUERY_CHECK_NULL(pTmpBuf, code, lino, _end, terrno);
pFillSup->pOffsetInfo = pTmpBuf;
SResultCellData* pCell = getResultCell(pFillInfo->pResRow, numOfResCol - 1);
int32_t preLength = pFillSup->pOffsetInfo[numOfResCol - 1] + pCell->bytes + sizeof(SResultCellData);
for (int32_t i = numOfResCol; i < pFillSup->numOfAllCols; i++) {
pFillSup->pOffsetInfo[i] = preLength;
pCell = getResultCell(pFillInfo->pResRow, i);
preLength += pCell->bytes + sizeof(SResultCellData);
}
}
pFillInfo->pNonFillRow = taosMemoryCalloc(1, sizeof(SResultRowData)); pFillInfo->pNonFillRow = taosMemoryCalloc(1, sizeof(SResultRowData));
QUERY_CHECK_NULL(pFillInfo->pNonFillRow, code, lino, _end, terrno); QUERY_CHECK_NULL(pFillInfo->pNonFillRow, code, lino, _end, terrno);
pFillInfo->pNonFillRow->key = INT64_MIN; pFillInfo->pNonFillRow->key = INT64_MIN;
@ -1607,6 +1683,7 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->delIndex = 0; pFillInfo->delIndex = 0;
pFillInfo->curGroupId = 0; pFillInfo->curGroupId = 0;
pFillInfo->hasNext = false; pFillInfo->hasNext = false;
pFillInfo->pTempBuff = taosMemoryCalloc(1, pFillSup->rowSize);
return pFillInfo; return pFillInfo;
_end: _end:
@ -1650,21 +1727,18 @@ static void setValueForFillInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo*
} }
} }
int32_t getDownStreamInfo(SOperatorInfo* downstream, int8_t* triggerType, SInterval* pInterval, SStreamAggSupporter** ppAggSup) { int32_t getDownStreamInfo(SOperatorInfo* downstream, int8_t* triggerType, SInterval* pInterval) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0; int32_t lino = 0;
if (IS_NORMAL_INTERVAL_OP(downstream)) { if (IS_NORMAL_INTERVAL_OP(downstream)) {
SStreamIntervalOperatorInfo* pInfo = downstream->info; SStreamIntervalOperatorInfo* pInfo = downstream->info;
*triggerType = pInfo->twAggSup.calTrigger; *triggerType = pInfo->twAggSup.calTrigger;
*pInterval = pInfo->interval; *pInterval = pInfo->interval;
(*ppAggSup) = NULL;
} else if (IS_CONTINUE_INTERVAL_OP(downstream)) { } else if (IS_CONTINUE_INTERVAL_OP(downstream)) {
SStreamIntervalSliceOperatorInfo* pInfo = downstream->info; SStreamIntervalSliceOperatorInfo* pInfo = downstream->info;
*triggerType = pInfo->twAggSup.calTrigger; *triggerType = pInfo->twAggSup.calTrigger;
*pInterval = pInfo->interval; *pInterval = pInfo->interval;
pInfo->hasFill = true; pInfo->hasFill = true;
(*ppAggSup) = &pInfo->streamAggSup;
pInfo->streamAggSup.stateStore.streamStateSetFillInfo(pInfo->streamAggSup.pState);
} else { } else {
code = TSDB_CODE_STREAM_INTERNAL_ERROR; code = TSDB_CODE_STREAM_INTERNAL_ERROR;
} }
@ -1677,8 +1751,31 @@ _end:
return code; return code;
} }
int32_t initFillOperatorStateBuff(SStreamFillOperatorInfo* pInfo, SStreamState* pState, SStateStore* pStore,
SReadHandle* pHandle, const char* taskIdStr, SStorageAPI* pApi) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
pInfo->stateStore = *pStore;
pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
QUERY_CHECK_NULL(pInfo->pState, code, lino, _end, terrno);
*(pInfo->pState) = *pState;
pInfo->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsCol);
code = pInfo->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->pFillSup->rowSize, 0, compareTs,
pInfo->pState, INT64_MAX, taskIdStr, pHandle->checkpointId,
STREAM_STATE_BUFF_HASH_SORT, &pInfo->pState->pFileState);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode,
SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo); QRY_PARAM_CHECK(pOptrInfo);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -1704,7 +1801,7 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi
int8_t triggerType = 0; int8_t triggerType = 0;
SInterval interval = {0}; SInterval interval = {0};
code = getDownStreamInfo(downstream, &triggerType, &interval, &pInfo->pStreamAggSup); code = getDownStreamInfo(downstream, &triggerType, &interval);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
pInfo->pFillSup = initStreamFillSup(pPhyFillNode, &interval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI, pInfo->pFillSup = initStreamFillSup(pPhyFillNode, &interval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI,
@ -1759,9 +1856,13 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi
pTaskInfo); pTaskInfo);
if (triggerType == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) { if (triggerType == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
code = initFillOperatorStateBuff(pInfo, pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.stateStore, pHandle,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI);
QUERY_CHECK_CODE(code, lino, _error);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamForceFillNext, NULL, destroyStreamFillOperatorInfo, pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamForceFillNext, NULL, destroyStreamFillOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
} else { } else {
pInfo->pState = NULL;
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFillNext, NULL, destroyStreamFillOperatorInfo, pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFillNext, NULL, destroyStreamFillOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
} }

View File

@ -74,6 +74,7 @@ void destroyStreamIntervalSliceOperatorInfo(void* param) {
blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pDelRes);
blockDataDestroy(pInfo->pCheckpointRes); blockDataDestroy(pInfo->pCheckpointRes);
taosMemoryFreeClear(pInfo->pOffsetInfo);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
@ -163,7 +164,7 @@ _end:
} }
void doStreamSliceInterpolation(SSliceRowData* pPrevWinVal, TSKEY winKey, TSKEY curTs, SSDataBlock* pDataBlock, void doStreamSliceInterpolation(SSliceRowData* pPrevWinVal, TSKEY winKey, TSKEY curTs, SSDataBlock* pDataBlock,
int32_t curRowIndex, SExprSupp* pSup, SIntervalSliceType type) { int32_t curRowIndex, SExprSupp* pSup, SIntervalSliceType type, int32_t* pOffsetInfo) {
SqlFunctionCtx* pCtx = pSup->pCtx; SqlFunctionCtx* pCtx = pSup->pCtx;
for (int32_t k = 0; k < pSup->numOfExprs; ++k) { for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) { if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
@ -175,7 +176,7 @@ void doStreamSliceInterpolation(SSliceRowData* pPrevWinVal, TSKEY winKey, TSKEY
SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, pParam->pCol->slotId); SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, pParam->pCol->slotId);
double prevVal = 0, curVal = 0, winVal = 0; double prevVal = 0, curVal = 0, winVal = 0;
SResultCellData* pCell = getSliceResultCell((SResultCellData*)pPrevWinVal->pRowVal, pParam->pCol->slotId); SResultCellData* pCell = getSliceResultCell((SResultCellData*)pPrevWinVal->pRowVal, pParam->pCol->slotId, pOffsetInfo);
GET_TYPED_DATA(prevVal, double, pCell->type, pCell->pData); GET_TYPED_DATA(prevVal, double, pCell->type, pCell->pData);
GET_TYPED_DATA(curVal, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex)); GET_TYPED_DATA(curVal, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
@ -278,7 +279,7 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput); resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
doSetElapsedEndKey(prevPoint.winKey.win.ekey, &pOperator->exprSupp); doSetElapsedEndKey(prevPoint.winKey.win.ekey, &pOperator->exprSupp);
doStreamSliceInterpolation(prevPoint.pLastRow, prevPoint.winKey.win.ekey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_END); doStreamSliceInterpolation(prevPoint.pLastRow, prevPoint.winKey.win.ekey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_END, pInfo->pOffsetInfo);
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &prevPoint.winKey.win, 1); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &prevPoint.winKey.win, 1);
code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,
0, pBlock->info.rows, numOfOutput); 0, pBlock->info.rows, numOfOutput);
@ -294,7 +295,7 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput); resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && curPoint.winKey.win.skey != curTs) { if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && curPoint.winKey.win.skey != curTs) {
doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_START); doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_START, pInfo->pOffsetInfo);
} }
forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, curWin.ekey, binarySearchForKey, NULL, forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, curWin.ekey, binarySearchForKey, NULL,
TSDB_ORDER_ASC); TSDB_ORDER_ASC);
@ -302,7 +303,7 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
if (pInfo->hasInterpoFunc && winCode != TSDB_CODE_SUCCESS) { if (pInfo->hasInterpoFunc && winCode != TSDB_CODE_SUCCESS) {
int32_t endRowId = getQualifiedRowNumDesc(pSup, pBlock, tsCols, prevEndPos, false); int32_t endRowId = getQualifiedRowNumDesc(pSup, pBlock, tsCols, prevEndPos, false);
TSKEY endRowTs = tsCols[endRowId]; TSKEY endRowTs = tsCols[endRowId];
transBlockToSliceResultRow(pBlock, endRowId, endRowTs, curPoint.pLastRow, 0, NULL, NULL); transBlockToSliceResultRow(pBlock, endRowId, endRowTs, curPoint.pLastRow, 0, NULL, NULL, pInfo->pOffsetInfo);
} }
SWinKey curKey = {.ts = curPoint.winKey.win.skey, .groupId = curPoint.winKey.groupId}; SWinKey curKey = {.ts = curPoint.winKey.win.skey, .groupId = curPoint.winKey.groupId};
if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS) { if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS) {
@ -359,9 +360,14 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
return code; return code;
} }
if (pInfo->hasFill == false) { if (pInfo->recvCkBlock) {
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState); pInfo->recvCkBlock = false;
} printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pCheckpointRes;
return code;
}
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
setStreamOperatorCompleted(pOperator); setStreamOperatorCompleted(pOperator);
(*ppRes) = NULL; (*ppRes) = NULL;
return code; return code;
@ -392,8 +398,6 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
case STREAM_CHECKPOINT: { case STREAM_CHECKPOINT: {
pInfo->recvCkBlock = true; pInfo->recvCkBlock = true;
pAggSup->stateStore.streamStateCommit(pAggSup->pState); pAggSup->stateStore.streamStateCommit(pAggSup->pState);
// doStreamIntervalSliceSaveCheckpoint(pOperator);
pInfo->recvCkBlock = true;
code = copyDataBlock(pInfo->pCheckpointRes, pBlock); code = copyDataBlock(pInfo->pCheckpointRes, pBlock);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
continue; continue;
@ -451,9 +455,13 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
if ((*ppRes) == NULL) { if ((*ppRes) == NULL) {
if (pInfo->hasFill == false) { if (pInfo->recvCkBlock) {
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState); pInfo->recvCkBlock = false;
} printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pCheckpointRes;
return code;
}
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
setStreamOperatorCompleted(pOperator); setStreamOperatorCompleted(pOperator);
} }
@ -596,6 +604,9 @@ int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiN
code = getDownstreamRes(downstream, &pDownRes, &pPkCol); code = getDownstreamRes(downstream, &pDownRes, &pPkCol);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
code = initOffsetInfo(&pInfo->pOffsetInfo, pDownRes);
QUERY_CHECK_CODE(code, lino, _error);
int32_t keyBytes = sizeof(TSKEY); int32_t keyBytes = sizeof(TSKEY);
keyBytes += blockDataGetRowSize(pDownRes) + sizeof(SResultCellData) * taosArrayGetSize(pDownRes->pDataBlock) + sizeof(bool); keyBytes += blockDataGetRowSize(pDownRes) + sizeof(SResultCellData) * taosArrayGetSize(pDownRes->pDataBlock) + sizeof(bool);
if (pPkCol) { if (pPkCol) {

View File

@ -281,8 +281,34 @@ static int32_t initTimeSliceResultBuf(SStreamFillSupporter* pFillSup, SExprSupp*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t initTimeSliceFillSup(SStreamInterpFuncPhysiNode* pPhyFillNode, SExprSupp* pExprSup, int32_t numOfExprs, SColumnInfo* pPkCol, int32_t initOffsetInfo(int32_t** ppOffset, SSDataBlock* pRes) {
SStreamFillSupporter** ppResFillSup) { int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t numOfCol = taosArrayGetSize(pRes->pDataBlock);
int32_t preLength = 0;
int32_t* pOffsetInfo = taosMemoryCalloc(numOfCol, sizeof(int32_t));
QUERY_CHECK_NULL(pOffsetInfo, code, lino, _end, lino);
for (int32_t i = 0; i < numOfCol; i++) {
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, i);
pOffsetInfo[i] = preLength;
int32_t bytes = 1;
if (pColInfo != NULL) {
bytes = pColInfo->info.bytes;
}
preLength += bytes + sizeof(SResultCellData);
}
(*ppOffset) = pOffsetInfo;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t initTimeSliceFillSup(SStreamInterpFuncPhysiNode* pPhyFillNode, SExprSupp* pExprSup, int32_t numOfExprs,
SSDataBlock* pInputRes, SColumnInfo* pPkCol, SStreamFillSupporter** ppResFillSup) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0; int32_t lino = 0;
SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter));
@ -320,6 +346,9 @@ static int32_t initTimeSliceFillSup(SStreamInterpFuncPhysiNode* pPhyFillNode, SE
pFillSup->comparePkColFn = NULL; pFillSup->comparePkColFn = NULL;
} }
code = initOffsetInfo(&pFillSup->pOffsetInfo, pInputRes);
QUERY_CHECK_CODE(code, lino, _end);
(*ppResFillSup) = pFillSup; (*ppResFillSup) = pFillSup;
_end: _end:
@ -359,17 +388,11 @@ _end:
} }
} }
SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index) { SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index, int32_t* pCellOffsetInfo) {
if (!pRowVal) { if (!pRowVal) {
return NULL; return NULL;
} }
char* pData = (char*)pRowVal; return POINTER_SHIFT(pRowVal, pCellOffsetInfo[index]);
SResultCellData* pCell = pRowVal;
for (int32_t i = 0; i < index; i++) {
pData += (pCell->bytes + sizeof(SResultCellData));
pCell = (SResultCellData*)pData;
}
return pCell;
} }
static bool isGroupKeyFunc(SExprInfo* pExprInfo) { static bool isGroupKeyFunc(SExprInfo* pExprInfo) {
@ -414,9 +437,9 @@ static int32_t fillPointResult(SStreamFillSupporter* pFillSup, SResultRowData* p
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId; int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = NULL; SResultCellData* pCell = NULL;
if (IS_FILL_CONST_VALUE(pFillSup->type) && (isGroupKeyFunc(pFillCol->pExpr) || isSelectGroupConstValueFunc(pFillCol->pExpr)) ) { if (IS_FILL_CONST_VALUE(pFillSup->type) && (isGroupKeyFunc(pFillCol->pExpr) || isSelectGroupConstValueFunc(pFillCol->pExpr)) ) {
pCell = getSliceResultCell(pNonFillRow->pRowVal, srcSlot); pCell = getSliceResultCell(pNonFillRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
} else { } else {
pCell = getSliceResultCell(pResRow->pRowVal, srcSlot); pCell = getSliceResultCell(pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
} }
code = setRowCell(pDstCol, pBlock->info.rows, pCell); code = setRowCell(pDstCol, pBlock->info.rows, pCell);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
@ -475,7 +498,7 @@ static void fillLinearRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFi
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
} else if (isInterpFunc(pFillCol->pExpr)) { } else if (isInterpFunc(pFillCol->pExpr)) {
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId; int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot); SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) {
colDataSetNULL(pDstCol, index); colDataSetNULL(pDstCol, index);
continue; continue;
@ -498,7 +521,7 @@ static void fillLinearRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFi
destroySPoint(&cur); destroySPoint(&cur);
} else { } else {
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId; int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot); SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
code = setRowCell(pDstCol, pBlock->info.rows, pCell); code = setRowCell(pDstCol, pBlock->info.rows, pCell);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
} }
@ -956,8 +979,8 @@ static void copyNonFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo
if (!isInterpFunc(pFillCol->pExpr) && !isIrowtsPseudoColumn(pFillCol->pExpr) && if (!isInterpFunc(pFillCol->pExpr) && !isIrowtsPseudoColumn(pFillCol->pExpr) &&
!isIsfilledPseudoColumn(pFillCol->pExpr)) { !isIsfilledPseudoColumn(pFillCol->pExpr)) {
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId; int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pSrcCell = getResultCell(&pFillSup->cur, srcSlot); SResultCellData* pSrcCell = getSliceResultCell(pFillSup->cur.pRowVal, srcSlot, pFillSup->pOffsetInfo);
SResultCellData* pDestCell = getResultCell(pFillInfo->pNonFillRow, srcSlot); SResultCellData* pDestCell = getSliceResultCell(pFillInfo->pNonFillRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
pDestCell->isNull = pSrcCell->isNull; pDestCell->isNull = pSrcCell->isNull;
if (!pDestCell->isNull) { if (!pDestCell->isNull) {
memcpy(pDestCell->pData, pSrcCell->pData, pSrcCell->bytes); memcpy(pDestCell->pData, pSrcCell->pData, pSrcCell->bytes);
@ -966,11 +989,11 @@ static void copyNonFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo
} }
} }
static void copyCalcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, int32_t numOfCol) { static void copyCalcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, int32_t numOfCol, int32_t* pOffsetInfo) {
for (int32_t i = 0; i < numOfCol; i++) { for (int32_t i = 0; i < numOfCol; i++) {
if (isInterpFunc(pFillCol[i].pExpr)) { if (isInterpFunc(pFillCol[i].pExpr)) {
int32_t slotId = pFillCol[i].pExpr->base.pParam[0].pCol->slotId; int32_t slotId = pFillCol[i].pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pECell = getResultCell(pEndRow, slotId); SResultCellData* pECell = getSliceResultCell(pEndRow->pRowVal, slotId, pOffsetInfo);
SPoint* pPoint = taosArrayGet(pEndPoins, slotId); SPoint* pPoint = taosArrayGet(pEndPoins, slotId);
pPoint->key = pEndRow->key; pPoint->key = pEndRow->key;
memcpy(pPoint->val, pECell->pData, pECell->bytes); memcpy(pPoint->val, pECell->pData, pECell->bytes);
@ -1112,7 +1135,7 @@ static void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo
SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd); SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd);
pFillSup->next.key = pFillSup->nextOriginKey; pFillSup->next.key = pFillSup->nextOriginKey;
copyCalcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, copyCalcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols); pFillSup->numOfAllCols, pFillSup->pOffsetInfo);
pFillSup->prev.key = pFillSup->prevOriginKey; pFillSup->prev.key = pFillSup->prevOriginKey;
pFillInfo->pResRow = &pFillSup->prev; pFillInfo->pResRow = &pFillSup->prev;
pFillInfo->pLinearInfo->hasNext = false; pFillInfo->pLinearInfo->hasNext = false;
@ -1121,7 +1144,7 @@ static void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo
pFillInfo->pos = FILL_POS_END; pFillInfo->pos = FILL_POS_END;
SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd); SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd);
copyCalcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, copyCalcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols); pFillSup->numOfAllCols, pFillSup->pOffsetInfo);
pFillSup->prev.key = pFillSup->prevOriginKey; pFillSup->prev.key = pFillSup->prevOriginKey;
pFillInfo->pResRow = &pFillSup->prev; pFillInfo->pResRow = &pFillSup->prev;
pFillInfo->pLinearInfo->hasNext = false; pFillInfo->pLinearInfo->hasNext = false;
@ -1132,7 +1155,7 @@ static void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo
SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd); SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd);
pFillSup->next.key = pFillSup->nextOriginKey; pFillSup->next.key = pFillSup->nextOriginKey;
copyCalcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, copyCalcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols); pFillSup->numOfAllCols, pFillSup->pOffsetInfo);
pFillInfo->pResRow = &pFillSup->cur; pFillInfo->pResRow = &pFillSup->cur;
pFillInfo->pLinearInfo->hasNext = false; pFillInfo->pLinearInfo->hasNext = false;
} }
@ -1253,11 +1276,11 @@ static bool needAdjustValue(SSlicePoint* pPoint, TSKEY ts, void* pPkVal, SStream
} }
void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal, void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal,
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol) { int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol, int32_t* pCellOffsetInfo) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SResultCellData* pCell = getSliceResultCell((SResultCellData*)pRowVal->pRowVal, i); SResultCellData* pCell = getSliceResultCell((SResultCellData*)pRowVal->pRowVal, i, pCellOffsetInfo);
if (!colDataIsNull_s(pColData, rowId)) { if (!colDataIsNull_s(pColData, rowId)) {
pCell->isNull = false; pCell->isNull = false;
pCell->type = pColData->info.type; pCell->type = pColData->info.type;
@ -1378,7 +1401,7 @@ static void doStreamTimeSliceImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock)
} }
right = needAdjustValue(&curPoint, tsCols[startPos], pPkVal, pFillSup, false, pFillSup->type); right = needAdjustValue(&curPoint, tsCols[startPos], pPkVal, pFillSup, false, pFillSup->type);
if (right) { if (right) {
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo); transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo, pFillSup->pOffsetInfo);
bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS; bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS;
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &curPoint.key, pInfo->pUpdatedMap, needDel, code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &curPoint.key, pInfo->pUpdatedMap, needDel,
pInfo->pDeletedMap); pInfo->pDeletedMap);
@ -1397,7 +1420,7 @@ static void doStreamTimeSliceImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock)
} }
left = needAdjustValue(&nextPoint, tsCols[leftRowId], pPkVal, pFillSup, true, pFillSup->type); left = needAdjustValue(&nextPoint, tsCols[leftRowId], pPkVal, pFillSup, true, pFillSup->type);
if (left) { if (left) {
transBlockToSliceResultRow(pBlock, leftRowId, tsCols[leftRowId], nextPoint.pLeftRow, pFillSup->rowSize, pPkVal, pPkColDataInfo); transBlockToSliceResultRow(pBlock, leftRowId, tsCols[leftRowId], nextPoint.pLeftRow, pFillSup->rowSize, pPkVal, pPkColDataInfo, pFillSup->pOffsetInfo);
bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS; bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS;
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &nextPoint.key, pInfo->pUpdatedMap, code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &nextPoint.key, pInfo->pUpdatedMap,
needDel, pInfo->pDeletedMap); needDel, pInfo->pDeletedMap);
@ -1422,7 +1445,7 @@ static void doStreamTimeSliceImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock)
} }
right = needAdjustValue(&curPoint, tsCols[startPos], pPkVal, pFillSup, false, pFillSup->type); right = needAdjustValue(&curPoint, tsCols[startPos], pPkVal, pFillSup, false, pFillSup->type);
if (right) { if (right) {
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo); transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo, pFillSup->pOffsetInfo);
bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS; bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS;
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &curPoint.key, pInfo->pUpdatedMap, needDel, code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &curPoint.key, pInfo->pUpdatedMap, needDel,
pInfo->pDeletedMap); pInfo->pDeletedMap);
@ -1886,7 +1909,7 @@ static int32_t doStreamTimeSliceNext(SOperatorInfo* pOperator, SSDataBlock** ppR
qDebug("===stream===build stream result, ts count:%d", size); qDebug("===stream===build stream result, ts count:%d", size);
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
TSKEY ts = *(TSKEY*) taosArrayGet(pInfo->pCloseTs, i); TSKEY ts = *(TSKEY*) taosArrayGet(pInfo->pCloseTs, i);
code = buildAllResultKey(&pInfo->streamAggSup, ts, pInfo->pUpdated); code = buildAllResultKey(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pState, ts, pInfo->pUpdated);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
} }
qDebug("===stream===build stream result, res count:%ld", taosArrayGetSize(pInfo->pUpdated)); qDebug("===stream===build stream result, res count:%ld", taosArrayGetSize(pInfo->pUpdated));
@ -1951,7 +1974,7 @@ static void copyFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo* p
continue; continue;
} }
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId; int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = getResultCell(pFillInfo->pResRow, srcSlot); SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
SFillColInfo* pValueCol = pFillSup->pAllColInfo + valueIndex; SFillColInfo* pValueCol = pFillSup->pAllColInfo + valueIndex;
SVariant* pVar = &(pValueCol->fillVal); SVariant* pVar = &(pValueCol->fillVal);
if (pCell->type == TSDB_DATA_TYPE_FLOAT) { if (pCell->type == TSDB_DATA_TYPE_FLOAT) {
@ -1975,7 +1998,7 @@ static void copyFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo* p
for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; SFillColInfo* pFillCol = pFillSup->pAllColInfo + i;
int32_t slotId = GET_DEST_SLOT_ID(pFillCol); int32_t slotId = GET_DEST_SLOT_ID(pFillCol);
SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, slotId, pFillSup->pOffsetInfo);
pCell->isNull = true; pCell->isNull = true;
} }
} }
@ -2095,7 +2118,7 @@ int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
pInfo->pFillSup = NULL; pInfo->pFillSup = NULL;
code = initTimeSliceFillSup(pInterpPhyNode, pExpSup, numOfExprs, pPkCol, &pInfo->pFillSup); code = initTimeSliceFillSup(pInterpPhyNode, pExpSup, numOfExprs, pDownRes, pPkCol, &pInfo->pFillSup);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
int32_t ratio = 1; int32_t ratio = 1;

View File

@ -1834,7 +1834,7 @@ int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption) {
return deleteMark; return deleteMark;
} }
static TSKEY compareTs(void* pKey) { TSKEY compareTs(void* pKey) {
SWinKey* pWinKey = (SWinKey*)pKey; SWinKey* pWinKey = (SWinKey*)pKey;
return pWinKey->ts; return pWinKey->ts;
} }

View File

@ -48,6 +48,7 @@ typedef struct STimeSliceOperatorInfo {
int32_t remainIndex; // the remaining index in the block to be processed int32_t remainIndex; // the remaining index in the block to be processed
bool hasPk; bool hasPk;
SColumn pkCol; SColumn pkCol;
int64_t rangeInterval;
} STimeSliceOperatorInfo; } STimeSliceOperatorInfo;
static void destroyTimeSliceOperatorInfo(void* param); static void destroyTimeSliceOperatorInfo(void* param);
@ -179,6 +180,11 @@ bool isIsfilledPseudoColumn(SExprInfo* pExprInfo) {
return (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_isfilled") == 0); return (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_isfilled") == 0);
} }
bool isIrowtsOriginPseudoColumn(SExprInfo* pExprInfo) {
const char* name = pExprInfo->pExpr->_function.functionName;
return (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_irowts_origin") == 0);
}
static void tRowGetKeyFromColData(int64_t ts, SColumnInfoData* pPkCol, int32_t rowIndex, SRowKey* pKey) { static void tRowGetKeyFromColData(int64_t ts, SColumnInfoData* pPkCol, int32_t rowIndex, SRowKey* pKey) {
pKey->ts = ts; pKey->ts = ts;
pKey->numOfPKs = 1; pKey->numOfPKs = 1;
@ -277,6 +283,79 @@ bool checkNullRow(SExprSupp* pExprSup, SSDataBlock* pSrcBlock, int32_t index, bo
return false; return false;
} }
static int32_t interpColSetKey(SColumnInfoData* pDst, int32_t rowNum, SGroupKeys* pKey) {
int32_t code = 0;
if (pKey->isNull == false) {
code = colDataSetVal(pDst, rowNum, pKey->pData, false);
} else {
colDataSetNULL(pDst, rowNum);
}
return code;
}
static bool interpSetFillRowWithRangeIntervalCheck(STimeSliceOperatorInfo* pSliceInfo, SArray** ppFillRow, SArray* pFillRefRow, int64_t fillRefRowTs) {
*ppFillRow = NULL;
if (pSliceInfo->rangeInterval <= 0 || llabs(fillRefRowTs - pSliceInfo->current) <= pSliceInfo->rangeInterval) {
*ppFillRow = pFillRefRow;
return true;
}
return false;
}
static bool interpDetermineNearFillRow(STimeSliceOperatorInfo* pSliceInfo, SArray** ppNearRow) {
if (!pSliceInfo->isPrevRowSet && !pSliceInfo->isNextRowSet) {
*ppNearRow = NULL;
return false;
}
SGroupKeys *pPrevTsKey = NULL, *pNextTsKey = NULL;
int64_t* pPrevTs = NULL, *pNextTs = NULL;
if (pSliceInfo->isPrevRowSet) {
pPrevTsKey = taosArrayGet(pSliceInfo->pPrevRow, pSliceInfo->tsCol.slotId);
pPrevTs = (int64_t*)pPrevTsKey->pData;
}
if (pSliceInfo->isNextRowSet) {
pNextTsKey = taosArrayGet(pSliceInfo->pNextRow, pSliceInfo->tsCol.slotId);
pNextTs = (int64_t*)pNextTsKey->pData;
}
if (!pPrevTsKey) {
*ppNearRow = pSliceInfo->pNextRow;
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppNearRow, pSliceInfo->pNextRow, *pNextTs);
} else if (!pNextTsKey) {
*ppNearRow = pSliceInfo->pPrevRow;
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppNearRow, pSliceInfo->pPrevRow, *pPrevTs);
} else {
if (llabs(pSliceInfo->current - *pPrevTs) <= llabs(*pNextTs - pSliceInfo->current)) {
// take prev if euqal
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppNearRow, pSliceInfo->pPrevRow, *pPrevTs);
} else {
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppNearRow, pSliceInfo->pNextRow, *pNextTs);
}
}
return true;
}
static bool interpDetermineFillRefRow(STimeSliceOperatorInfo* pSliceInfo, SArray** ppOutRow) {
bool needFill = false;
if (pSliceInfo->fillType == TSDB_FILL_PREV) {
if (pSliceInfo->isPrevRowSet) {
SGroupKeys* pTsCol = taosArrayGet(pSliceInfo->pPrevRow, pSliceInfo->tsCol.slotId);
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppOutRow, pSliceInfo->pPrevRow, *(int64_t*)pTsCol->pData);
needFill = true;
}
} else if (pSliceInfo->fillType == TSDB_FILL_NEXT) {
if (pSliceInfo->isNextRowSet) {
SGroupKeys* pTsCol = taosArrayGet(pSliceInfo->pNextRow, pSliceInfo->tsCol.slotId);
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppOutRow, pSliceInfo->pNextRow, *(int64_t*)pTsCol->pData);
needFill = true;
}
} else if (pSliceInfo->fillType == TSDB_FILL_NEAR) {
needFill = interpDetermineNearFillRow(pSliceInfo, ppOutRow);
} else {
needFill = true;
}
return needFill;
}
static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock,
SSDataBlock* pSrcBlock, int32_t index, bool beforeTs, SExecTaskInfo* pTaskInfo) { SSDataBlock* pSrcBlock, int32_t index, bool beforeTs, SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -290,6 +369,8 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
int32_t fillColIndex = 0; int32_t fillColIndex = 0;
int32_t groupKeyIndex = 0; int32_t groupKeyIndex = 0;
bool hasInterp = true; bool hasInterp = true;
SArray* pFillRefRow = NULL;
bool needFill = interpDetermineFillRefRow(pSliceInfo, &pFillRefRow);
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
@ -305,7 +386,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
code = colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); code = colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
continue; continue;
} else if (!isInterpFunc(pExprInfo)) { } else if (!isInterpFunc(pExprInfo) && !isIrowtsOriginPseudoColumn(pExprInfo)) {
if (isGroupKeyFunc(pExprInfo) || isSelectGroupConstValueFunc(pExprInfo)) { if (isGroupKeyFunc(pExprInfo) || isSelectGroupConstValueFunc(pExprInfo)) {
if (pSrcBlock != NULL) { if (pSrcBlock != NULL) {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
@ -344,7 +425,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
continue; continue;
} }
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; int32_t srcSlot = isIrowtsOriginPseudoColumn(pExprInfo) ? pSliceInfo->tsCol.slotId : pExprInfo->base.pParam[0].pCol->slotId;
switch (pSliceInfo->fillType) { switch (pSliceInfo->fillType) {
case TSDB_FILL_NULL: case TSDB_FILL_NULL:
case TSDB_FILL_NULL_F: { case TSDB_FILL_NULL_F: {
@ -352,6 +433,25 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
break; break;
} }
case TSDB_FILL_PREV:
case TSDB_FILL_NEAR:
case TSDB_FILL_NEXT: {
if (!needFill) {
hasInterp = false;
break;
}
if (pFillRefRow) {
code = interpColSetKey(pDst, rows, taosArrayGet(pFillRefRow, srcSlot));
QUERY_CHECK_CODE(code, lino, _end);
break;
}
// no fillRefRow, fall through to fill specified values
if (srcSlot == pSliceInfo->tsCol.slotId) {
// if is _irowts_origin, there is no value to fill, just set to null
colDataSetNULL(pDst, rows);
break;
}
}
case TSDB_FILL_SET_VALUE: case TSDB_FILL_SET_VALUE:
case TSDB_FILL_SET_VALUE_F: { case TSDB_FILL_SET_VALUE_F: {
SVariant* pVar = &pSliceInfo->pFillColInfo[fillColIndex].fillVal; SVariant* pVar = &pSliceInfo->pFillColInfo[fillColIndex].fillVal;
@ -444,38 +544,6 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
taosMemoryFree(current.val); taosMemoryFree(current.val);
break; break;
} }
case TSDB_FILL_PREV: {
if (!pSliceInfo->isPrevRowSet) {
hasInterp = false;
break;
}
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
if (pkey->isNull == false) {
code = colDataSetVal(pDst, rows, pkey->pData, false);
QUERY_CHECK_CODE(code, lino, _end);
} else {
colDataSetNULL(pDst, rows);
}
break;
}
case TSDB_FILL_NEXT: {
if (!pSliceInfo->isNextRowSet) {
hasInterp = false;
break;
}
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
if (pkey->isNull == false) {
code = colDataSetVal(pDst, rows, pkey->pData, false);
QUERY_CHECK_CODE(code, lino, _end);
} else {
colDataSetNULL(pDst, rows);
}
break;
}
case TSDB_FILL_NONE: case TSDB_FILL_NONE:
default: default:
break; break;
@ -507,7 +575,7 @@ static int32_t addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSu
int32_t dstSlot = pExprInfo->base.resSchema.slotId; int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (isIrowtsPseudoColumn(pExprInfo)) { if (isIrowtsPseudoColumn(pExprInfo) || isIrowtsOriginPseudoColumn(pExprInfo)) {
code = colDataSetVal(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); code = colDataSetVal(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
} else if (isIsfilledPseudoColumn(pExprInfo)) { } else if (isIsfilledPseudoColumn(pExprInfo)) {
@ -1233,6 +1301,7 @@ int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyN
pInfo->pNextGroupRes = NULL; pInfo->pNextGroupRes = NULL;
pInfo->pRemainRes = NULL; pInfo->pRemainRes = NULL;
pInfo->remainIndex = 0; pInfo->remainIndex = 0;
pInfo->rangeInterval = pInterpPhyNode->rangeInterval;
if (pInfo->hasPk) { if (pInfo->hasPk) {
pInfo->prevKey.numOfPKs = 1; pInfo->prevKey.numOfPKs = 1;

View File

@ -3094,7 +3094,7 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf
qptCtx.result.code = createFillOperatorInfo(NULL, (SFillPhysiNode*)pNode, pTaskInfo, ppOperaotr); qptCtx.result.code = createFillOperatorInfo(NULL, (SFillPhysiNode*)pNode, pTaskInfo, ppOperaotr);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
qptCtx.result.code = createStreamFillOperatorInfo(NULL, (SStreamFillPhysiNode*)pNode, pTaskInfo, ppOperaotr); qptCtx.result.code = createStreamFillOperatorInfo(NULL, (SStreamFillPhysiNode*)pNode, pTaskInfo, pReadHandle, ppOperaotr);
break; break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
qptCtx.result.code = createSessionAggOperatorInfo(NULL, (SSessionWinodwPhysiNode*)pNode, pTaskInfo, ppOperaotr); qptCtx.result.code = createSessionAggOperatorInfo(NULL, (SSessionWinodwPhysiNode*)pNode, pTaskInfo, ppOperaotr);

View File

@ -5596,6 +5596,20 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = NULL, .sprocessFunc = NULL,
.finalizeFunc = NULL .finalizeFunc = NULL
}, },
{
.name = "_irowts_origin",
.type = FUNCTION_TYPE_IROWTS_ORIGIN,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.parameters = {.minParamNum = 0,
.maxParamNum = 0,
.paramInfoPattern = 0,
.outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_TIMESTAMP_TYPE}},
.translateFunc = translateTimePseudoColumn,
.getEnvFunc = getTimePseudoFuncEnv,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
}; };
// clang-format on // clang-format on

View File

@ -703,3 +703,10 @@ bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId) {
bool fmIsCountLikeFunc(int32_t funcId) { bool fmIsCountLikeFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_COUNT_LIKE_FUNC); return isSpecificClassifyFunc(funcId, FUNC_MGT_COUNT_LIKE_FUNC);
} }
bool fmIsRowTsOriginFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNCTION_TYPE_IROWTS_ORIGIN == funcMgtBuiltins[funcId].type;
}

View File

@ -684,6 +684,8 @@ static int32_t logicInterpFuncCopy(const SInterpFuncLogicNode* pSrc, SInterpFunc
CLONE_NODE_FIELD(pFillValues); CLONE_NODE_FIELD(pFillValues);
CLONE_NODE_FIELD(pTimeSeries); CLONE_NODE_FIELD(pTimeSeries);
COPY_OBJECT_FIELD(streamNodeOption, sizeof(SStreamNodeOption)); COPY_OBJECT_FIELD(streamNodeOption, sizeof(SStreamNodeOption));
COPY_SCALAR_FIELD(rangeInterval);
COPY_SCALAR_FIELD(rangeIntervalUnit);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

View File

@ -1300,6 +1300,8 @@ static const char* jkInterpFuncLogicPlanFillMode = "fillMode";
static const char* jkInterpFuncLogicPlanFillValues = "FillValues"; static const char* jkInterpFuncLogicPlanFillValues = "FillValues";
static const char* jkInterpFuncLogicPlanTimeSeries = "TimeSeries"; static const char* jkInterpFuncLogicPlanTimeSeries = "TimeSeries";
static const char* jkInterpFuncLogicPlanStreamNodeOption = "StreamNodeOption"; static const char* jkInterpFuncLogicPlanStreamNodeOption = "StreamNodeOption";
static const char* jkInterpFuncLogicPlanRangeInterval = "RangeInterval";
static const char* jkInterpFuncLogicPlanRangeIntervalUnit = "RangeIntervalUnit";
static int32_t logicInterpFuncNodeToJson(const void* pObj, SJson* pJson) { static int32_t logicInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
const SInterpFuncLogicNode* pNode = (const SInterpFuncLogicNode*)pObj; const SInterpFuncLogicNode* pNode = (const SInterpFuncLogicNode*)pObj;
@ -1335,6 +1337,12 @@ static int32_t logicInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); code = tjsonAddObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncLogicPlanRangeInterval, pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncLogicPlanRangeIntervalUnit, pNode->rangeIntervalUnit);
}
return code; return code;
} }
@ -1373,6 +1381,12 @@ static int32_t jsonToLogicInterpFuncNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); code = tjsonToObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncLogicPlanRangeInterval, &pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkInterpFuncLogicPlanRangeIntervalUnit, &pNode->rangeIntervalUnit);
}
return code; return code;
} }
@ -3314,6 +3328,8 @@ static const char* jkInterpFuncPhysiPlanFillMode = "FillMode";
static const char* jkInterpFuncPhysiPlanFillValues = "FillValues"; static const char* jkInterpFuncPhysiPlanFillValues = "FillValues";
static const char* jkInterpFuncPhysiPlanTimeSeries = "TimeSeries"; static const char* jkInterpFuncPhysiPlanTimeSeries = "TimeSeries";
static const char* jkInterpFuncPhysiPlanStreamNodeOption = "StreamNodeOption"; static const char* jkInterpFuncPhysiPlanStreamNodeOption = "StreamNodeOption";
static const char* jkInterpFuncPhysiPlanRangeInterval = "RangeInterval";
static const char* jkInterpFuncPhysiPlanRangeIntervalUnit = "RangeIntervalUnit";
static int32_t physiInterpFuncNodeToJson(const void* pObj, SJson* pJson) { static int32_t physiInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
const SInterpFuncPhysiNode* pNode = (const SInterpFuncPhysiNode*)pObj; const SInterpFuncPhysiNode* pNode = (const SInterpFuncPhysiNode*)pObj;
@ -3352,6 +3368,12 @@ static int32_t physiInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption); code = tjsonAddObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanRangeInterval, pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanRangeIntervalUnit, pNode->rangeIntervalUnit);
}
return code; return code;
} }
@ -3393,6 +3415,12 @@ static int32_t jsonToPhysiInterpFuncNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption); code = tjsonToObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncPhysiPlanRangeInterval, &pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkInterpFuncPhysiPlanRangeIntervalUnit, &pNode->rangeIntervalUnit);
}
return code; return code;
} }

View File

@ -3746,7 +3746,9 @@ enum {
PHY_INERP_FUNC_CODE_INTERVAL_UNIT, PHY_INERP_FUNC_CODE_INTERVAL_UNIT,
PHY_INERP_FUNC_CODE_FILL_MODE, PHY_INERP_FUNC_CODE_FILL_MODE,
PHY_INERP_FUNC_CODE_FILL_VALUES, PHY_INERP_FUNC_CODE_FILL_VALUES,
PHY_INERP_FUNC_CODE_TIME_SERIES PHY_INERP_FUNC_CODE_TIME_SERIES,
PHY_INTERP_FUNC_CODE_RANGE_INTERVAL,
PHY_INTERP_FUNC_CODE_RANGE_INTERVAL_UNIT,
}; };
static int32_t physiInterpFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { static int32_t physiInterpFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -3777,6 +3779,12 @@ static int32_t physiInterpFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder)
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_TIME_SERIES, nodeToMsg, pNode->pTimeSeries); code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_TIME_SERIES, nodeToMsg, pNode->pTimeSeries);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_INTERP_FUNC_CODE_RANGE_INTERVAL, pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI8(pEncoder, PHY_INTERP_FUNC_CODE_RANGE_INTERVAL_UNIT, pNode->rangeIntervalUnit);
}
return code; return code;
} }
@ -3815,6 +3823,12 @@ static int32_t msgToPhysiInterpFuncNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_INERP_FUNC_CODE_TIME_SERIES: case PHY_INERP_FUNC_CODE_TIME_SERIES:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTimeSeries); code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTimeSeries);
break; break;
case PHY_INTERP_FUNC_CODE_RANGE_INTERVAL:
code = tlvDecodeI64(pTlv, &pNode->rangeInterval);
break;
case PHY_INTERP_FUNC_CODE_RANGE_INTERVAL_UNIT:
code = tlvDecodeI8(pTlv, &pNode->rangeIntervalUnit);
break;
default: default:
break; break;
} }

View File

@ -470,6 +470,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:
code = makeNode(type, sizeof(SSetOperator), &pNode); code = makeNode(type, sizeof(SSetOperator), &pNode);
break; break;
case QUERY_NODE_RANGE_AROUND:
code = makeNode(type, sizeof(SRangeAroundNode), &pNode); break;
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
code = makeNode(type, sizeof(SSelectStmt), &pNode); code = makeNode(type, sizeof(SSelectStmt), &pNode);
break; break;
@ -1248,6 +1250,12 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pWin->pEndOffset); nodesDestroyNode(pWin->pEndOffset);
break; break;
} }
case QUERY_NODE_RANGE_AROUND: {
SRangeAroundNode* pAround = (SRangeAroundNode*)pNode;
nodesDestroyNode(pAround->pInterval);
nodesDestroyNode(pAround->pTimepoint);
break;
}
case QUERY_NODE_SET_OPERATOR: { case QUERY_NODE_SET_OPERATOR: {
SSetOperator* pStmt = (SSetOperator*)pNode; SSetOperator* pStmt = (SSetOperator*)pNode;
nodesDestroyList(pStmt->pProjectionList); nodesDestroyList(pStmt->pProjectionList);
@ -1269,6 +1277,7 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyList(pStmt->pGroupByList); nodesDestroyList(pStmt->pGroupByList);
nodesDestroyNode(pStmt->pHaving); nodesDestroyNode(pStmt->pHaving);
nodesDestroyNode(pStmt->pRange); nodesDestroyNode(pStmt->pRange);
nodesDestroyNode(pStmt->pRangeAround);
nodesDestroyNode(pStmt->pEvery); nodesDestroyNode(pStmt->pEvery);
nodesDestroyNode(pStmt->pFill); nodesDestroyNode(pStmt->pFill);
nodesDestroyList(pStmt->pOrderByList); nodesDestroyList(pStmt->pOrderByList);

View File

@ -163,6 +163,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValue
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd); SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint); SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
SNode* createInterpTimeAround(SAstCreateContext* pCxt, SNode* pTimepoint, SNode* pInterval);
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
SNode* createAlterSingleTagColumnNode(SAstCreateContext* pCtx, SToken* token, SNode* pVal); SNode* createAlterSingleTagColumnNode(SAstCreateContext* pCtx, SToken* token, SNode* pVal);

View File

@ -1210,6 +1210,7 @@ pseudo_column(A) ::= QTAGS(B).
pseudo_column(A) ::= FLOW(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= FLOW(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= FHIGH(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= FHIGH(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= FROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= FROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= IROWTS_ORIGIN(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
@ -1455,7 +1456,7 @@ jlimit_clause_opt(A) ::= JLIMIT NK_INTEGER(B).
query_specification(A) ::= query_specification(A) ::=
SELECT hint_list(M) set_quantifier_opt(B) tag_mode_opt(N) select_list(C) from_clause_opt(D) SELECT hint_list(M) set_quantifier_opt(B) tag_mode_opt(N) select_list(C) from_clause_opt(D)
where_clause_opt(E) partition_by_clause_opt(F) range_opt(J) every_opt(K) where_clause_opt(E) partition_by_clause_opt(F) range_opt(J) every_opt(K)
fill_opt(L) twindow_clause_opt(G) group_by_clause_opt(H) having_clause_opt(I). { interp_fill_opt(L) twindow_clause_opt(G) group_by_clause_opt(H) having_clause_opt(I). {
A = createSelectStmt(pCxt, B, C, D, M); A = createSelectStmt(pCxt, B, C, D, M);
A = setSelectStmtTagMode(pCxt, A, N); A = setSelectStmtTagMode(pCxt, A, N);
A = addWhereClause(pCxt, A, E); A = addWhereClause(pCxt, A, E);
@ -1540,19 +1541,41 @@ interval_sliding_duration_literal(A) ::= NK_VARIABLE(B).
interval_sliding_duration_literal(A) ::= NK_STRING(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } interval_sliding_duration_literal(A) ::= NK_STRING(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
interval_sliding_duration_literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); } interval_sliding_duration_literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
interp_fill_opt(A) ::= . { A = NULL; }
interp_fill_opt(A) ::= fill_value(B). { A = B; }
interp_fill_opt(A) ::=
FILL NK_LP fill_position_mode_extension(B) NK_COMMA expression_list(C) NK_RP. { A = createFillNode(pCxt, B, createNodeListNode(pCxt, C)); }
interp_fill_opt(A) ::= FILL NK_LP interp_fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); }
fill_opt(A) ::= . { A = NULL; } fill_opt(A) ::= . { A = NULL; }
fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); }
fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } fill_opt(A) ::= fill_value(B). { A = B; }
fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); }
fill_value(A) ::= FILL NK_LP VALUE NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); }
fill_value(A) ::= FILL NK_LP VALUE_F NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); }
%type fill_mode { EFillMode } %type fill_mode { EFillMode }
%destructor fill_mode { } %destructor fill_mode { }
fill_mode(A) ::= NONE. { A = FILL_MODE_NONE; } fill_mode(A) ::= NONE. { A = FILL_MODE_NONE; }
fill_mode(A) ::= PREV. { A = FILL_MODE_PREV; }
fill_mode(A) ::= NULL. { A = FILL_MODE_NULL; } fill_mode(A) ::= NULL. { A = FILL_MODE_NULL; }
fill_mode(A) ::= NULL_F. { A = FILL_MODE_NULL_F; } fill_mode(A) ::= NULL_F. { A = FILL_MODE_NULL_F; }
fill_mode(A) ::= LINEAR. { A = FILL_MODE_LINEAR; } fill_mode(A) ::= LINEAR. { A = FILL_MODE_LINEAR; }
fill_mode(A) ::= NEXT. { A = FILL_MODE_NEXT; } fill_mode(A) ::= fill_position_mode(B). { A = B; }
%type fill_position_mode { EFillMode }
%destructor fill_position_mode { }
fill_position_mode(A) ::= PREV. { A = FILL_MODE_PREV; }
fill_position_mode(A) ::= NEXT. { A = FILL_MODE_NEXT; }
%type fill_position_mode_extension { EFillMode }
%destructor fill_position_mode_extension { }
fill_position_mode_extension(A) ::= fill_position_mode(B). { A = B; }
fill_position_mode_extension(A) ::= NEAR. { A = FILL_MODE_NEAR; }
%type interp_fill_mode { EFillMode }
%destructor interp_fill_mode { }
interp_fill_mode(A) ::= fill_mode(B). { A = B; }
interp_fill_mode(A) ::= NEAR. { A = FILL_MODE_NEAR; }
%type group_by_clause_opt { SNodeList* } %type group_by_clause_opt { SNodeList* }
%destructor group_by_clause_opt { nodesDestroyList($$); } %destructor group_by_clause_opt { nodesDestroyList($$); }

View File

@ -1460,6 +1460,9 @@ _err:
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd) { SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
if (pEnd && nodeType(pEnd) == QUERY_NODE_VALUE && ((SValueNode*)pEnd)->flag & VALUE_FLAG_IS_DURATION) {
return createInterpTimeAround(pCxt, pStart, pEnd);
}
return createBetweenAnd(pCxt, createPrimaryKeyCol(pCxt, NULL), pStart, pEnd); return createBetweenAnd(pCxt, createPrimaryKeyCol(pCxt, NULL), pStart, pEnd);
_err: _err:
nodesDestroyNode(pStart); nodesDestroyNode(pStart);
@ -1475,6 +1478,19 @@ _err:
return NULL; return NULL;
} }
SNode* createInterpTimeAround(SAstCreateContext* pCxt, SNode* pTimepoint, SNode* pInterval) {
CHECK_PARSER_STATUS(pCxt);
SRangeAroundNode* pAround = NULL;
pCxt->errCode = nodesMakeNode(QUERY_NODE_RANGE_AROUND, (SNode**)&pAround);
CHECK_PARSER_STATUS(pCxt);
pAround->pTimepoint = createInterpTimePoint(pCxt, pTimepoint);
pAround->pInterval = pInterval;
CHECK_PARSER_STATUS(pCxt);
return (SNode*)pAround;
_err:
return NULL;
}
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen) { SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SWhenThenNode* pWhenThen = NULL; SWhenThenNode* pWhenThen = NULL;
@ -1632,8 +1648,15 @@ _err:
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange) { SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SSelectStmt* pSelect = (SSelectStmt*)pStmt;
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
((SSelectStmt*)pStmt)->pRange = pRange; if (pRange && nodeType(pRange) == QUERY_NODE_RANGE_AROUND) {
pSelect->pRangeAround = pRange;
SRangeAroundNode* pAround = (SRangeAroundNode*)pRange;
TSWAP(pSelect->pRange, pAround->pTimepoint);
} else {
pSelect->pRange = pRange;
}
} }
return pStmt; return pStmt;
_err: _err:

View File

@ -31,6 +31,7 @@ typedef struct SInsertParseContext {
bool needTableTagVal; bool needTableTagVal;
bool needRequest; // whether or not request server bool needRequest; // whether or not request server
bool isStmtBind; // whether is stmt bind bool isStmtBind; // whether is stmt bind
bool preCtbname;
} SInsertParseContext; } SInsertParseContext;
typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param);
@ -1831,6 +1832,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
if (TK_NK_QUESTION == pToken->type) { if (TK_NK_QUESTION == pToken->type) {
pCxt->isStmtBind = true; pCxt->isStmtBind = true;
if (pCols->pColIndex[i] == tbnameIdx) { if (pCols->pColIndex[i] == tbnameIdx) {
pCxt->preCtbname = false;
*bFoundTbName = true; *bFoundTbName = true;
} }
if (NULL == pCxt->pComCxt->pStmtCb) { if (NULL == pCxt->pComCxt->pStmtCb) {
@ -2571,10 +2573,11 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif
} }
int32_t code = (*pCxt->pComCxt->pStmtCb->getTbNameFn)(pCxt->pComCxt->pStmtCb->pStmt, &tbName); int32_t code = (*pCxt->pComCxt->pStmtCb->getTbNameFn)(pCxt->pComCxt->pStmtCb->pStmt, &tbName);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; pCxt->preCtbname = true;
} else {
pTbName->z = tbName;
pTbName->n = strlen(tbName);
} }
pTbName->z = tbName;
pTbName->n = strlen(tbName);
} }
if (pCxt->isStmtBind) { if (pCxt->isStmtBind) {
@ -2599,7 +2602,7 @@ static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt)
SStmtCallback* pStmtCb = pCxt->pComCxt->pStmtCb; SStmtCallback* pStmtCb = pCxt->pComCxt->pStmtCb;
int32_t code = (*pStmtCb->setInfoFn)(pStmtCb->pStmt, pStmt->pTableMeta, tags, &pStmt->targetTableName, int32_t code = (*pStmtCb->setInfoFn)(pStmtCb->pStmt, pStmt->pTableMeta, tags, &pStmt->targetTableName,
pStmt->usingTableProcessing, pStmt->pVgroupsHashObj, pStmt->pTableBlockHashObj, pStmt->usingTableProcessing, pStmt->pVgroupsHashObj, pStmt->pTableBlockHashObj,
pStmt->usingTableName.tname); pStmt->usingTableName.tname, pCxt->preCtbname);
memset(&pCxt->tags, 0, sizeof(pCxt->tags)); memset(&pCxt->tags, 0, sizeof(pCxt->tags));
pStmt->pVgroupsHashObj = NULL; pStmt->pVgroupsHashObj = NULL;

View File

@ -939,39 +939,72 @@ int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSc
} }
int32_t buildStbBoundFields(SBoundColInfo boundColsInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_STB** fields, int32_t buildStbBoundFields(SBoundColInfo boundColsInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_STB** fields,
STableMeta* pMeta) { STableMeta* pMeta, void* boundTags, bool preCtbname) {
SBoundColInfo* tags = (SBoundColInfo*)boundTags;
int32_t numOfBound = boundColsInfo.numOfBound + tags->numOfBound + (preCtbname ? 1 : 0);
int32_t idx = 0;
if (fields != NULL) { if (fields != NULL) {
*fields = taosMemoryCalloc(boundColsInfo.numOfBound, sizeof(TAOS_FIELD_STB)); *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_STB));
if (NULL == *fields) { if (NULL == *fields) {
return terrno; return terrno;
} }
SSchema* schema = &pSchema[boundColsInfo.pColIndex[0]]; if (preCtbname && numOfBound != boundColsInfo.numOfBound) {
if (TSDB_DATA_TYPE_TIMESTAMP == schema->type) { (*fields)[idx].field_type = TAOS_FIELD_TBNAME;
(*fields)[0].precision = pMeta->tableInfo.precision; tstrncpy((*fields)[idx].name, "tbname", sizeof((*fields)[idx].name));
(*fields)[idx].type = TSDB_DATA_TYPE_VARCHAR;
(*fields)[idx].bytes = TSDB_TABLE_FNAME_LEN;
idx++;
} }
for (int32_t i = 0; i < boundColsInfo.numOfBound; ++i) { if (tags->numOfBound > 0) {
int16_t idx = boundColsInfo.pColIndex[i]; SSchema* pSchema = getTableTagSchema(pMeta);
if (idx == pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type) {
(*fields)[i].field_type = TAOS_FIELD_TBNAME; (*fields)[0].precision = pMeta->tableInfo.precision;
tstrncpy((*fields)[i].name, "tbname", sizeof((*fields)[i].name));
continue;
} else if (idx < pMeta->tableInfo.numOfColumns) {
(*fields)[i].field_type = TAOS_FIELD_COL;
} else {
(*fields)[i].field_type = TAOS_FIELD_TAG;
} }
schema = &pSchema[idx]; for (int32_t i = 0; i < tags->numOfBound; ++i) {
tstrncpy((*fields)[i].name, schema->name, sizeof((*fields)[i].name)); (*fields)[idx].field_type = TAOS_FIELD_TAG;
(*fields)[i].type = schema->type;
(*fields)[i].bytes = schema->bytes; SSchema* schema = &pSchema[tags->pColIndex[i]];
tstrncpy((*fields)[idx].name, schema->name, sizeof((*fields)[i].name));
(*fields)[idx].type = schema->type;
(*fields)[idx].bytes = schema->bytes;
idx++;
}
}
if (boundColsInfo.numOfBound > 0) {
SSchema* schema = &pSchema[boundColsInfo.pColIndex[0]];
if (TSDB_DATA_TYPE_TIMESTAMP == schema->type) {
(*fields)[0].precision = pMeta->tableInfo.precision;
}
for (int32_t i = 0; i < boundColsInfo.numOfBound; ++i) {
int16_t idxCol = boundColsInfo.pColIndex[i];
if (idxCol == pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags) {
(*fields)[idx].field_type = TAOS_FIELD_TBNAME;
tstrncpy((*fields)[i].name, "tbname", sizeof((*fields)[idx].name));
continue;
} else if (idxCol < pMeta->tableInfo.numOfColumns) {
(*fields)[idx].field_type = TAOS_FIELD_COL;
} else {
(*fields)[idx].field_type = TAOS_FIELD_TAG;
}
schema = &pSchema[idxCol];
tstrncpy((*fields)[idx].name, schema->name, sizeof((*fields)[idx].name));
(*fields)[idx].type = schema->type;
(*fields)[idx].bytes = schema->bytes;
idx++;
}
} }
} }
*fieldNum = boundColsInfo.numOfBound; *fieldNum = numOfBound;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1018,7 +1051,8 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qBuildStmtStbColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_STB** fields) { int32_t qBuildStmtStbColFields(void* pBlock, void* boundTags, bool preCtbname, int32_t* fieldNum,
TAOS_FIELD_STB** fields) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
if (pDataBlock->boundColsInfo.numOfBound <= 0) { if (pDataBlock->boundColsInfo.numOfBound <= 0) {
@ -1030,7 +1064,8 @@ int32_t qBuildStmtStbColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_STB**
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
CHECK_CODE(buildStbBoundFields(pDataBlock->boundColsInfo, pSchema, fieldNum, fields, pDataBlock->pMeta)); CHECK_CODE(buildStbBoundFields(pDataBlock->boundColsInfo, pSchema, fieldNum, fields, pDataBlock->pMeta, boundTags,
preCtbname));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

View File

@ -176,6 +176,7 @@ static SKeyword keywordTable[] = {
{"NORMAL", TK_NORMAL}, {"NORMAL", TK_NORMAL},
{"NCHAR", TK_NCHAR}, {"NCHAR", TK_NCHAR},
{"NEXT", TK_NEXT}, {"NEXT", TK_NEXT},
{"NEAR", TK_NEAR},
{"NMATCH", TK_NMATCH}, {"NMATCH", TK_NMATCH},
{"NONE", TK_NONE}, {"NONE", TK_NONE},
{"NOT", TK_NOT}, {"NOT", TK_NOT},
@ -326,6 +327,7 @@ static SKeyword keywordTable[] = {
{"WRITE", TK_WRITE}, {"WRITE", TK_WRITE},
{"_C0", TK_ROWTS}, {"_C0", TK_ROWTS},
{"_IROWTS", TK_IROWTS}, {"_IROWTS", TK_IROWTS},
{"_IROWTS_ORIGIN", TK_IROWTS_ORIGIN},
{"_ISFILLED", TK_ISFILLED}, {"_ISFILLED", TK_ISFILLED},
{"_QDURATION", TK_QDURATION}, {"_QDURATION", TK_QDURATION},
{"_QEND", TK_QEND}, {"_QEND", TK_QEND},

View File

@ -1267,7 +1267,8 @@ bool isPrimaryKeyImpl(SNode* pExpr) {
FUNCTION_TYPE_LAST_ROW == pFunc->funcType || FUNCTION_TYPE_TIMETRUNCATE == pFunc->funcType) { FUNCTION_TYPE_LAST_ROW == pFunc->funcType || FUNCTION_TYPE_TIMETRUNCATE == pFunc->funcType) {
return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0)); return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0));
} else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType ||
FUNCTION_TYPE_IROWTS == pFunc->funcType || FUNCTION_TYPE_FORECAST_ROWTS == pFunc->funcType) { FUNCTION_TYPE_IROWTS == pFunc->funcType || FUNCTION_TYPE_IROWTS_ORIGIN == pFunc->funcType ||
FUNCTION_TYPE_FORECAST_ROWTS == pFunc->funcType) {
return true; return true;
} }
} else if (QUERY_NODE_OPERATOR == nodeType(pExpr)) { } else if (QUERY_NODE_OPERATOR == nodeType(pExpr)) {
@ -5397,14 +5398,11 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList
return code; return code;
} }
static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) { static int32_t doCheckFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) {
if (FILL_MODE_VALUE != pFill->mode && FILL_MODE_VALUE_F != pFill->mode) {
return TSDB_CODE_SUCCESS;
}
int32_t fillNo = 0; int32_t fillNo = 0;
SNodeListNode* pFillValues = (SNodeListNode*)pFill->pValues; SNodeListNode* pFillValues = (SNodeListNode*)pFill->pValues;
SNode* pProject = NULL; SNode* pProject = NULL;
if (!pFillValues) return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch");
FOREACH(pProject, pProjectionList) { FOREACH(pProject, pProjectionList) {
if (needFill(pProject)) { if (needFill(pProject)) {
if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) { if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) {
@ -5424,6 +5422,14 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) {
if (FILL_MODE_VALUE != pFill->mode && FILL_MODE_VALUE_F != pFill->mode) {
return TSDB_CODE_SUCCESS;
}
return doCheckFillValues(pCxt, pFill, pProjectionList);
return TSDB_CODE_SUCCESS;
}
static int32_t translateFillValues(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translateFillValues(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) || if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) ||
NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) { NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) {
@ -6194,6 +6200,31 @@ static int32_t translateInterpEvery(STranslateContext* pCxt, SNode** pEvery) {
return code; return code;
} }
static EDealRes hasRowTsOriginFuncWalkNode(SNode* pNode, void* ctx) {
bool *hasRowTsOriginFunc = ctx;
if (nodeType(pNode) == QUERY_NODE_FUNCTION) {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
if (fmIsRowTsOriginFunc(pFunc->funcId)) {
*hasRowTsOriginFunc = true;
return DEAL_RES_END;
}
}
return DEAL_RES_CONTINUE;
}
static int32_t checkInterpForStream(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (pCxt->createStream) {
SFillNode* pFill = (SFillNode*)pSelect->pFill;
if (pFill->mode == FILL_MODE_NEAR) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "FILL NEAR is not supported by stream");
}
if (pSelect->pRangeAround) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "RANGE with around is not supported by stream");
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -6209,13 +6240,60 @@ static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkFill(pCxt, (SFillNode*)pSelect->pFill, (SValueNode*)pSelect->pEvery, true); code = checkFill(pCxt, (SFillNode*)pSelect->pFill, (SValueNode*)pSelect->pEvery, true);
} }
bool hasRowTsOriginFunc = false;
nodesWalkExprs(pSelect->pProjectionList, hasRowTsOriginFuncWalkNode, &hasRowTsOriginFunc);
if (hasRowTsOriginFunc && pCxt->createStream) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"_irowts_origin is not supported by stream");
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkFillValues(pCxt, (SFillNode*)pSelect->pFill, pSelect->pProjectionList); SFillNode* pFill = (SFillNode*)pSelect->pFill;
if (pSelect->pRangeAround) {
if (pFill->mode != FILL_MODE_PREV && pFill->mode != FILL_MODE_NEXT && pFill->mode != FILL_MODE_NEAR) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE,
"Range with interval can only used with fill PREV/NEXT/NEAR");
}
if (TSDB_CODE_SUCCESS == code)
code = doCheckFillValues(pCxt, pFill, pSelect->pProjectionList);
} else {
if (FILL_MODE_PREV == pFill->mode || FILL_MODE_NEXT == pFill->mode || FILL_MODE_NEAR == pFill->mode) {
if (pFill->pValues) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Can't specify fill values");
}
} else {
if (hasRowTsOriginFunc) return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, "_irowts_origin can only be used with FILL PREV/NEXT/NEAR");
}
code = checkFillValues(pCxt, pFill, pSelect->pProjectionList);
}
} }
return code; return code;
} }
static int32_t translateInterpAround(STranslateContext* pCxt, SSelectStmt* pSelect) {
int32_t code = 0;
if (pSelect->pRangeAround) {
SRangeAroundNode* pAround = (SRangeAroundNode*)pSelect->pRangeAround;
code = translateExpr(pCxt, &pAround->pInterval);
if (TSDB_CODE_SUCCESS == code) {
if (nodeType(pAround->pInterval) == QUERY_NODE_VALUE) {
SValueNode* pVal = (SValueNode*)pAround->pInterval;
if (pVal->datum.i == 0) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, "Range interval cannot be 0");
}
int8_t unit = pVal->unit;
if (unit == TIME_UNIT_YEAR || unit == TIME_UNIT_MONTH) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE,
"Unsupported time unit in RANGE clause");
}
} else {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, "Invalid range interval");
}
}
}
return code;
}
static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (!pSelect->hasInterpFunc) { if (!pSelect->hasInterpFunc) {
if (NULL != pSelect->pRange || NULL != pSelect->pEvery || NULL != pSelect->pFill) { if (NULL != pSelect->pRange || NULL != pSelect->pEvery || NULL != pSelect->pFill) {
@ -6236,6 +6314,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
} }
} }
int32_t code = 0;
if (pCxt->createStream) { if (pCxt->createStream) {
if (NULL != pSelect->pRange) { if (NULL != pSelect->pRange) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
@ -6247,23 +6326,40 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
"Missing EVERY clause or FILL clause"); "Missing EVERY clause or FILL clause");
} }
} else { } else {
if (NULL == pSelect->pRange || NULL == pSelect->pEvery || NULL == pSelect->pFill) { if (!pSelect->pRangeAround) {
if (pSelect->pRange != NULL && QUERY_NODE_OPERATOR == nodeType(pSelect->pRange) && pSelect->pEvery == NULL) { if (NULL == pSelect->pRange || NULL == pSelect->pEvery || NULL == pSelect->pFill) {
// single point interp every can be omitted if (pSelect->pRange != NULL && QUERY_NODE_OPERATOR == nodeType(pSelect->pRange) && pSelect->pEvery == NULL) {
} else { // single point interp every can be omitted
} else {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE,
"Missing RANGE clause, EVERY clause or FILL clause");
}
}
} else {
if (pSelect->pEvery) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE,
"Missing RANGE clause, EVERY clause or FILL clause"); "Range clause with around interval can't be used with EVERY clause");
}
if (!pSelect->pFill) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE,
"Missing FILL clause");
} }
} }
} }
int32_t code = translateExpr(pCxt, &pSelect->pRange); code = translateExpr(pCxt, &pSelect->pRange);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = translateInterpEvery(pCxt, &pSelect->pEvery); code = translateInterpEvery(pCxt, &pSelect->pEvery);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = translateInterpFill(pCxt, pSelect); code = translateInterpFill(pCxt, pSelect);
} }
if (TSDB_CODE_SUCCESS == code) {
code = translateInterpAround(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkInterpForStream(pCxt, pSelect);
}
return code; return code;
} }

View File

@ -973,6 +973,16 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
pInterpFunc->precision = pSelect->precision; pInterpFunc->precision = pSelect->precision;
} }
if (TSDB_CODE_SUCCESS == code && pSelect->pRangeAround) {
SNode* pRangeInterval = ((SRangeAroundNode*)pSelect->pRangeAround)->pInterval;
if (!pRangeInterval || nodeType(pRangeInterval) != QUERY_NODE_VALUE) {
code = TSDB_CODE_PAR_INTERNAL_ERROR;
} else {
pInterpFunc->rangeInterval = ((SValueNode*)pRangeInterval)->datum.i;
pInterpFunc->rangeIntervalUnit = ((SValueNode*)pRangeInterval)->unit;
}
}
// set the output // set the output
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExprs(pInterpFunc->pFuncs, &pInterpFunc->node.pTargets); code = createColumnByRewriteExprs(pInterpFunc->pFuncs, &pInterpFunc->node.pTargets);

View File

@ -1959,6 +1959,8 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh
pInterpFunc->intervalUnit = pFuncLogicNode->intervalUnit; pInterpFunc->intervalUnit = pFuncLogicNode->intervalUnit;
pInterpFunc->precision = pFuncLogicNode->node.precision; pInterpFunc->precision = pFuncLogicNode->node.precision;
pInterpFunc->pFillValues = NULL; pInterpFunc->pFillValues = NULL;
pInterpFunc->rangeInterval = pFuncLogicNode->rangeInterval;
pInterpFunc->rangeIntervalUnit = pFuncLogicNode->rangeIntervalUnit;
code = nodesCloneNode(pFuncLogicNode->pFillValues, &pInterpFunc->pFillValues); code = nodesCloneNode(pFuncLogicNode->pFillValues, &pInterpFunc->pFillValues);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
code = code; code = code;

View File

@ -4130,7 +4130,7 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const
SStreamStateCur* streamStateFillSeekToLast_rocksdb(SStreamState* pState) { SStreamStateCur* streamStateFillSeekToLast_rocksdb(SStreamState* pState) {
SWinKey key = {.groupId = UINT64_MAX, .ts = INT64_MAX}; SWinKey key = {.groupId = UINT64_MAX, .ts = INT64_MAX};
return streamStateFillSeekKeyNext_rocksdb(pState, &key); return streamStateFillSeekKeyPrev_rocksdb(pState, &key);
} }
#ifdef BUILD_NO_CALL #ifdef BUILD_NO_CALL

View File

@ -14,6 +14,7 @@
*/ */
#include "streamInt.h" #include "streamInt.h"
#include "ttime.h"
static int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) { static int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) {
*pSubmit = NULL; *pSubmit = NULL;
@ -307,13 +308,17 @@ void streamFreeQitem(SStreamQueueItem* data) {
} }
} }
int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t trigger, SInterval* pInterval, STimeWindow* pLatestWindow, const char* id) { int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t interval, SInterval* pInterval,
STimeWindow* pLatestWindow, const char* id) {
QRY_PARAM_CHECK(pTrigger); QRY_PARAM_CHECK(pTrigger);
int64_t ts = INT64_MIN;
SStreamTrigger* p = NULL; SStreamTrigger* p = NULL;
int64_t ts = taosGetTimestamp(pInterval->precision);
int64_t skey = pLatestWindow->skey + interval;
int32_t code = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0, (void**)&p); int32_t code = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0, (void**)&p);
if (code) { if (code) {
stError("s-task:%s failed to create force_window trigger, code:%s", id, tstrerror(code));
return code; return code;
} }
@ -324,26 +329,10 @@ int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t trigge
return terrno; return terrno;
} }
// let's calculate the previous time window p->pBlock->info.window.skey = skey;
SInterval interval = {.interval = trigger, p->pBlock->info.window.ekey = TMAX(ts, skey + interval);
.sliding = trigger,
.intervalUnit = pInterval->intervalUnit,
.slidingUnit = pInterval->slidingUnit};
ts = taosGetTimestampMs();
if (pLatestWindow->skey == INT64_MIN) {
STimeWindow window = getAlignQueryTimeWindow(&interval, ts - trigger);
p->pBlock->info.window.skey = window.skey;
p->pBlock->info.window.ekey = TMAX(ts, window.ekey);
} else {
int64_t skey = pLatestWindow->skey + trigger;
p->pBlock->info.window.skey = skey;
p->pBlock->info.window.ekey = TMAX(ts, skey + trigger);
}
p->pBlock->info.type = STREAM_GET_RESULT; p->pBlock->info.type = STREAM_GET_RESULT;
stDebug("s-task:%s force_window_close trigger block generated, window range:%" PRId64 "-%" PRId64, id, stDebug("s-task:%s force_window_close trigger block generated, window range:%" PRId64 "-%" PRId64, id,
p->pBlock->info.window.skey, p->pBlock->info.window.ekey); p->pBlock->info.window.skey, p->pBlock->info.window.ekey);

View File

@ -17,6 +17,9 @@
#include "streamInt.h" #include "streamInt.h"
#include "ttimer.h" #include "ttimer.h"
#define TRIGGER_RECHECK_INTERVAL (5 * 1000)
#define INITIAL_TRIGGER_INTERVAL (120 * 1000)
static void streamTaskResumeHelper(void* param, void* tmrId); static void streamTaskResumeHelper(void* param, void* tmrId);
static void streamTaskSchedHelper(void* param, void* tmrId); static void streamTaskSchedHelper(void* param, void* tmrId);
@ -24,7 +27,7 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
int64_t delay = 0; int64_t delay = 0;
int32_t code = 0; int32_t code = 0;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int64_t* pTaskRefId = NULL; int64_t* pTaskRefId = NULL;
if (pTask->info.fillHistory == 1) { if (pTask->info.fillHistory == 1) {
return; return;
@ -32,8 +35,8 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
// dynamic set the trigger & triggerParam for STREAM_TRIGGER_FORCE_WINDOW_CLOSE // dynamic set the trigger & triggerParam for STREAM_TRIGGER_FORCE_WINDOW_CLOSE
if ((pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) && (pTask->info.taskLevel == TASK_LEVEL__SOURCE)) { if ((pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) && (pTask->info.taskLevel == TASK_LEVEL__SOURCE)) {
int64_t waterMark = 0; int64_t waterMark = 0;
SInterval interval = {0}; SInterval interval = {0};
STimeWindow lastTimeWindow = {0}; STimeWindow lastTimeWindow = {0};
code = qGetStreamIntervalExecInfo(pTask->exec.pExecutor, &waterMark, &interval, &lastTimeWindow); code = qGetStreamIntervalExecInfo(pTask->exec.pExecutor, &waterMark, &interval, &lastTimeWindow);
if (code) { if (code) {
@ -41,19 +44,37 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
return; return;
} }
pTask->status.latestForceWindow = lastTimeWindow;
pTask->info.delaySchedParam = interval.sliding; pTask->info.delaySchedParam = interval.sliding;
pTask->info.watermark = waterMark; pTask->info.watermark = waterMark;
pTask->info.interval = interval; pTask->info.interval = interval;
// calculate the first start timestamp // calculate the first start timestamp
int64_t now = taosGetTimestamp(interval.precision); int64_t now = taosGetTimestamp(interval.precision);
STimeWindow curWin = getAlignQueryTimeWindow(&pTask->info.interval, now); STimeWindow curWin = getAlignQueryTimeWindow(&pTask->info.interval, now);
delay = (curWin.ekey + 1) - now + waterMark;
if (lastTimeWindow.skey == INT64_MIN) { // start from now, not the exec task timestamp after delay
pTask->status.latestForceWindow.skey = curWin.skey - pTask->info.interval.interval;
pTask->status.latestForceWindow.ekey = now;
delay = (curWin.ekey + 1) - now + waterMark;
delay = convertTimePrecision(delay, interval.precision, TSDB_TIME_PRECISION_MILLI);
} else {
pTask->status.latestForceWindow = lastTimeWindow;
// It's the current calculated time window
int64_t calEkey = lastTimeWindow.skey + pTask->info.interval.interval * 2;
if (calEkey + waterMark < now) { // unfinished time window existed
delay = INITIAL_TRIGGER_INTERVAL; // wait for 2min to start to calculate
} else {
delay = (curWin.ekey + 1) - now + waterMark;
delay = convertTimePrecision(delay, interval.precision, TSDB_TIME_PRECISION_MILLI);
}
}
stInfo("s-task:%s extract interval info from executor, wm:%" PRId64 " interval:%" PRId64 " unit:%c sliding:%" PRId64 stInfo("s-task:%s extract interval info from executor, wm:%" PRId64 " interval:%" PRId64 " unit:%c sliding:%" PRId64
" unit:%c, initial start after:%" PRId64, " unit:%c, initial start after:%" PRId64 "ms last_win:%" PRId64 "-%" PRId64,
id, waterMark, interval.interval, interval.intervalUnit, interval.sliding, interval.slidingUnit, delay); id, waterMark, interval.interval, interval.intervalUnit, interval.sliding, interval.slidingUnit, delay,
pTask->status.latestForceWindow.skey, pTask->status.latestForceWindow.ekey);
} else { } else {
delay = pTask->info.delaySchedParam; delay = pTask->info.delaySchedParam;
if (delay == 0) { if (delay == 0) {
@ -66,8 +87,8 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
stDebug("s-task:%s refId:%" PRId64 " enable the scheduler trigger, delay:%" PRId64, pTask->id.idStr, stDebug("s-task:%s refId:%" PRId64 " enable the scheduler trigger, delay:%" PRId64, pTask->id.idStr,
pTask->id.refId, delay); pTask->id.refId, delay);
streamTmrStart(streamTaskSchedHelper, (int32_t)delay, pTaskRefId, streamTimer, streamTmrStart(streamTaskSchedHelper, (int32_t)delay, pTaskRefId, streamTimer, &pTask->schedInfo.pDelayTimer,
&pTask->schedInfo.pDelayTimer, pTask->pMeta->vgId, "sched-tmr"); pTask->pMeta->vgId, "sched-tmr");
pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE; pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE;
} }
} }
@ -184,7 +205,62 @@ void streamTaskResumeHelper(void* param, void* tmrId) {
streamTaskFreeRefId(param); streamTaskFreeRefId(param);
} }
static int32_t doCreateForceWindowTrigger(SStreamTask* pTask, int32_t* pNextTrigger) {
int32_t num = 0;
int32_t code = 0;
const char* id = pTask->id.idStr;
int8_t precision = pTask->info.interval.precision;
SStreamTrigger* pTrigger = NULL;
while (1) {
code = streamCreateForcewindowTrigger(&pTrigger, pTask->info.delaySchedParam, &pTask->info.interval,
&pTask->status.latestForceWindow, id);
if (code != 0) {
*pNextTrigger = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI);
stError("s-task:%s failed to prepare force window close trigger, code:%s, try again in %dms", id,
tstrerror(code), *pNextTrigger);
return code;
}
// in the force window close model, status trigger does not matter. So we do not set the trigger model
code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to put retrieve aggRes block into q, code:%s", pTask->id.idStr, tstrerror(code));
return code;
}
num += 1;
// check whether the time window gaps exist or not
int64_t now = taosGetTimestamp(precision);
int64_t ekey = pTrigger->pBlock->info.window.skey + pTask->info.interval.interval;
// there are gaps, needs to be filled
STimeWindow w = pTrigger->pBlock->info.window;
w.ekey = w.skey + pTask->info.interval.interval;
if (w.skey <= pTask->status.latestForceWindow.skey) {
stFatal("s-task:%s invalid new time window in force_window_close trigger model, skey:%" PRId64
" should be greater than latestForceWindow skey:%" PRId64,
pTask->id.idStr, w.skey, pTask->status.latestForceWindow.skey);
}
pTask->status.latestForceWindow = w;
if (ekey + pTask->info.watermark + pTask->info.interval.interval > now) {
int64_t prev = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI);
*pNextTrigger = ekey + pTask->info.watermark + pTask->info.interval.interval - now;
*pNextTrigger = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI);
stDebug("s-task:%s generate %d time window(s), trigger delay adjust from %" PRId64 " to %d", id, num, prev,
*pNextTrigger);
return code;
} else {
stDebug("s-task:%s gap exist for force_window_close, current force_window_skey:%" PRId64, id, w.skey);
}
}
}
void streamTaskSchedHelper(void* param, void* tmrId) { void streamTaskSchedHelper(void* param, void* tmrId) {
int32_t code = 0;
int64_t taskRefId = *(int64_t*)param; int64_t taskRefId = *(int64_t*)param;
SStreamTask* pTask = taosAcquireRef(streamTaskRefPool, taskRefId); SStreamTask* pTask = taosAcquireRef(streamTaskRefPool, taskRefId);
if (pTask == NULL) { if (pTask == NULL) {
@ -193,15 +269,20 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
return; return;
} }
stDebug("s-task:%s acquire task, refId:%"PRId64, pTask->id.idStr, pTask->id.refId); stDebug("s-task:%s acquire task, refId:%" PRId64, pTask->id.idStr, pTask->id.refId);
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam; int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam;
int32_t vgId = pTask->pMeta->vgId; int32_t vgId = pTask->pMeta->vgId;
int32_t code = 0;
int8_t status = atomic_load_8(&pTask->schedInfo.status); int8_t status = atomic_load_8(&pTask->schedInfo.status);
stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
int32_t next = convertTimePrecision(nextTrigger, pTask->info.interval.precision, TSDB_TIME_PRECISION_MILLI);
stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, next);
} else {
stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
}
if (streamTaskShouldStop(pTask)) { if (streamTaskShouldStop(pTask)) {
stDebug("s-task:%s should stop, jump out of schedTimer", id); stDebug("s-task:%s should stop, jump out of schedTimer", id);
@ -211,62 +292,30 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
} }
if (streamTaskShouldPause(pTask)) { if (streamTaskShouldPause(pTask)) {
stDebug("s-task:%s is paused, recheck in %.2fs", id, nextTrigger/1000.0); stDebug("s-task:%s is paused, recheck in %.2fs", id, TRIGGER_RECHECK_INTERVAL / 1000.0);
streamTmrStart(streamTaskSchedHelper, nextTrigger, param, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, streamTmrStart(streamTaskSchedHelper, TRIGGER_RECHECK_INTERVAL, param, streamTimer, &pTask->schedInfo.pDelayTimer,
"sched-run-tmr"); vgId, "sched-run-tmr");
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
} }
if (streamTaskShouldPause(pTask)) { if (pTask->status.downstreamReady == 0) {
stDebug("s-task:%s is paused, check in nextTrigger:%ds", id, nextTrigger/1000); stDebug("s-task:%s downstream not ready, recheck in %.2fs", id, TRIGGER_RECHECK_INTERVAL / 1000.0);
streamTmrStart(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, streamTmrStart(streamTaskSchedHelper, TRIGGER_RECHECK_INTERVAL, param, streamTimer, &pTask->schedInfo.pDelayTimer,
"sched-run-tmr"); vgId, "sched-run-tmr");
streamMetaReleaseTask(pTask->pMeta, pTask);
return;
} }
if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) { if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) {
stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, nextTrigger); nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 10 seec
stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, TRIGGER_RECHECK_INTERVAL);
} else { } else {
if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && pTask->info.taskLevel == TASK_LEVEL__SOURCE) { if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
SStreamTrigger* pTrigger = NULL; code = doCreateForceWindowTrigger(pTask, &nextTrigger);
if (code != TSDB_CODE_SUCCESS) {
while (1) { goto _end;
code = streamCreateForcewindowTrigger(&pTrigger, pTask->info.delaySchedParam, &pTask->info.interval,
&pTask->status.latestForceWindow, id);
if (code != 0) {
stError("s-task:%s failed to prepare force window close trigger, code:%s, try again in %dms", id,
tstrerror(code), nextTrigger);
goto _end;
}
// in the force window close model, status trigger does not matter. So we do not set the trigger model
code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to put retrieve aggRes block into q, code:%s", pTask->id.idStr, tstrerror(code));
goto _end;
}
// check whether the time window gaps exist or not
int64_t now = taosGetTimestamp(pTask->info.interval.precision);
int64_t intervalEndTs = pTrigger->pBlock->info.window.skey + pTask->info.interval.interval;
// there are gaps, needs to be filled
STimeWindow w = pTrigger->pBlock->info.window;
w.ekey = w.skey + pTask->info.interval.interval;
if (w.skey <= pTask->status.latestForceWindow.skey) {
stFatal("s-task:%s invalid new time window in force_window_close model, skey:%" PRId64
" should be greater than latestForceWindow skey:%" PRId64,
pTask->id.idStr, w.skey, pTask->status.latestForceWindow.skey);
}
pTask->status.latestForceWindow = w;
if (intervalEndTs + pTask->info.watermark + pTask->info.interval.interval > now) {
break;
} else {
stDebug("s-task:%s gap exist for force_window_close, current force_window_skey:%" PRId64, id, w.skey);
}
} }
} else if (status == TASK_TRIGGER_STATUS__MAY_ACTIVE) { } else if (status == TASK_TRIGGER_STATUS__MAY_ACTIVE) {
SStreamTrigger* pTrigger = NULL; SStreamTrigger* pTrigger = NULL;
code = streamCreateSinkResTrigger(&pTrigger); code = streamCreateSinkResTrigger(&pTrigger);

View File

@ -23,6 +23,33 @@
#define NUM_OF_CACHE_WIN 64 #define NUM_OF_CACHE_WIN 64
#define MAX_NUM_OF_CACHE_WIN 128 #define MAX_NUM_OF_CACHE_WIN 128
int32_t recoverSearchBuff(SStreamFileState* pFileState, SArray* pWinStates, uint64_t groupId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SWinKey start = {.groupId = groupId, .ts = INT64_MAX};
void* pState = getStateFileStore(pFileState);
SStreamStateCur* pCur = streamStateFillSeekKeyPrev_rocksdb(pState, &start);
for (int32_t i = 0; i < NUM_OF_CACHE_WIN; i++) {
SWinKey tmpKey = {.groupId = groupId};
int32_t tmpRes = streamStateFillGetGroupKVByCur_rocksdb(pCur, &tmpKey, NULL, 0);
if (tmpRes != TSDB_CODE_SUCCESS) {
break;
}
void* tmp = taosArrayPush(pWinStates, &tmpKey);
QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
streamStateCurPrev_rocksdb(pCur);
}
taosArraySort(pWinStates, winKeyCmprImpl);
streamStateFreeCur(pCur);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t getHashSortRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, void** pVal, int32_t* pVLen, int32_t getHashSortRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode) { int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -38,22 +65,8 @@ int32_t getHashSortRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, vo
// recover // recover
if (taosArrayGetSize(pWinStates) == 0 && needClearDiskBuff(pFileState)) { if (taosArrayGetSize(pWinStates) == 0 && needClearDiskBuff(pFileState)) {
TSKEY ts = getFlushMark(pFileState); code = recoverSearchBuff(pFileState, pWinStates, pKey->groupId);
SWinKey start = {.groupId = pKey->groupId, .ts = INT64_MAX}; QUERY_CHECK_CODE(code, lino, _end);
void* pState = getStateFileStore(pFileState);
SStreamStateCur* pCur = streamStateFillSeekKeyPrev_rocksdb(pState, &start);
for (int32_t i = 0; i < NUM_OF_CACHE_WIN; i++) {
SWinKey tmpKey = {.groupId = pKey->groupId};
int32_t tmpRes = streamStateFillGetGroupKVByCur_rocksdb(pCur, &tmpKey, NULL, 0);
if (tmpRes != TSDB_CODE_SUCCESS) {
break;
}
void* tmp = taosArrayPush(pWinStates, &tmpKey);
QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
streamStateCurPrev_rocksdb(pCur);
}
taosArraySort(pWinStates, winKeyCmprImpl);
streamStateFreeCur(pCur);
} }
code = addSearchItem(pFileState, pWinStates, pKey); code = addSearchItem(pFileState, pWinStates, pKey);
@ -203,29 +216,17 @@ int32_t getHashSortPrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SW
SArray* pWinStates = NULL; SArray* pWinStates = NULL;
SSHashObj* pSearchBuff = getSearchBuff(pFileState); SSHashObj* pSearchBuff = getSearchBuff(pFileState);
void* pState = getStateFileStore(pFileState); void* pState = getStateFileStore(pFileState);
void** ppBuff = (void**) tSimpleHashGet(pSearchBuff, &pKey->groupId, sizeof(uint64_t)); // void** ppBuff = (void**) tSimpleHashGet(pSearchBuff, &pKey->groupId, sizeof(uint64_t));
if (ppBuff) {
pWinStates = (SArray*)(*ppBuff); code = addArrayBuffIfNotExist(pSearchBuff, pKey->groupId, &pWinStates);
} else { QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== search buff is empty.group id:%" PRId64, pKey->groupId);
SStreamStateCur* pCur = streamStateFillSeekKeyPrev_rocksdb(pState, pKey); // recover
void* tmpVal = NULL; if (taosArrayGetSize(pWinStates) == 0 && needClearDiskBuff(pFileState)) {
int32_t len = 0; code = recoverSearchBuff(pFileState, pWinStates, pKey->groupId);
(*pWinCode) = streamStateFillGetGroupKVByCur_rocksdb(pCur, pResKey, (const void**)&tmpVal, &len); QUERY_CHECK_CODE(code, lino, _end);
if ((*pWinCode) == TSDB_CODE_SUCCESS) {
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
QUERY_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pRowBuff, tmpVal, len);
taosMemoryFreeClear(tmpVal);
*pVLen = getRowStateRowSize(pFileState);
(*ppVal) = pNewPos;
}
streamStateFreeCur(pCur);
return code;
} }
int32_t size = taosArrayGetSize(pWinStates); int32_t size = taosArrayGetSize(pWinStates);
int32_t index = binarySearch(pWinStates, size, pKey, fillStateKeyCompare); int32_t index = binarySearch(pWinStates, size, pKey, fillStateKeyCompare);
if (index >= 0) { if (index >= 0) {

View File

@ -259,7 +259,7 @@ int32_t streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize,
if (type == STREAM_STATE_BUFF_HASH || type == STREAM_STATE_BUFF_HASH_SEARCH) { if (type == STREAM_STATE_BUFF_HASH || type == STREAM_STATE_BUFF_HASH_SEARCH) {
code = recoverSnapshot(pFileState, checkpointId); code = recoverSnapshot(pFileState, checkpointId);
} else if (type == STREAM_STATE_BUFF_SORT) { } else if (type == STREAM_STATE_BUFF_SORT) {
code = recoverSesssion(pFileState, checkpointId); code = recoverSession(pFileState, checkpointId);
} else if (type == STREAM_STATE_BUFF_HASH_SORT) { } else if (type == STREAM_STATE_BUFF_HASH_SORT) {
code = recoverFillSnapshot(pFileState, checkpointId); code = recoverFillSnapshot(pFileState, checkpointId);
} }
@ -914,7 +914,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
return code; return code;
} }
int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { int32_t recoverSession(SStreamFileState* pFileState, int64_t ckId) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0; int32_t lino = 0;
int32_t winRes = TSDB_CODE_SUCCESS; int32_t winRes = TSDB_CODE_SUCCESS;
@ -991,6 +991,7 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) {
} }
winCode = streamStateGetKVByCur_rocksdb(getStateFileStore(pFileState), pCur, pNewPos->pKey, (const void**)&pVal, &vlen); winCode = streamStateGetKVByCur_rocksdb(getStateFileStore(pFileState), pCur, pNewPos->pKey, (const void**)&pVal, &vlen);
qDebug("===stream=== get state by cur winres:%d. %s", winCode, __func__);
if (winCode != TSDB_CODE_SUCCESS || pFileState->getTs(pNewPos->pKey) < pFileState->flushMark) { if (winCode != TSDB_CODE_SUCCESS || pFileState->getTs(pNewPos->pKey) < pFileState->flushMark) {
destroyRowBuffPos(pNewPos); destroyRowBuffPos(pNewPos);
SListNode* pNode = tdListPopTail(pFileState->usedBuffs); SListNode* pNode = tdListPopTail(pFileState->usedBuffs);
@ -1007,6 +1008,7 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) {
memcpy(pNewPos->pRowBuff, pVal, vlen); memcpy(pNewPos->pRowBuff, pVal, vlen);
taosMemoryFreeClear(pVal); taosMemoryFreeClear(pVal);
pNewPos->beFlushed = true; pNewPos->beFlushed = true;
qDebug("===stream=== read checkpoint state from disc. %s", __func__);
code = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); code = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyRowBuffPos(pNewPos); destroyRowBuffPos(pNewPos);
@ -1077,6 +1079,7 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) {
int32_t vlen = 0; int32_t vlen = 0;
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
winRes = streamStateFillGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen); winRes = streamStateFillGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen);
qDebug("===stream=== get state by cur winres:%d. %s", winRes, __func__);
if (winRes != TSDB_CODE_SUCCESS || isFlushedState(pFileState, pFileState->getTs(pNewPos->pKey), 0)) { if (winRes != TSDB_CODE_SUCCESS || isFlushedState(pFileState, pFileState->getTs(pNewPos->pKey), 0)) {
destroyRowBuffPos(pNewPos); destroyRowBuffPos(pNewPos);
SListNode* pNode = tdListPopTail(pFileState->usedBuffs); SListNode* pNode = tdListPopTail(pFileState->usedBuffs);
@ -1085,9 +1088,17 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) {
break; break;
} }
if (vlen != pFileState->rowSize) {
qError("row size mismatch, expect:%d, actual:%d", pFileState->rowSize, vlen);
code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
taosMemoryFreeClear(pVal);
QUERY_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pRowBuff, pVal, vlen); memcpy(pNewPos->pRowBuff, pVal, vlen);
taosMemoryFreeClear(pVal); taosMemoryFreeClear(pVal);
pNewPos->beFlushed = true; pNewPos->beFlushed = true;
qDebug("===stream=== read checkpoint state from disc. %s", __func__);
winRes = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES); winRes = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES);
if (winRes != TSDB_CODE_SUCCESS) { if (winRes != TSDB_CODE_SUCCESS) {
destroyRowBuffPos(pNewPos); destroyRowBuffPos(pNewPos);
@ -1232,11 +1243,6 @@ void clearExpiredState(SStreamFileState* pFileState) {
int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey); int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey);
qTrace("clear expired file, ts:%" PRId64 ". %s at line %d res:%d", pKey->ts, __func__, __LINE__, code_file); qTrace("clear expired file, ts:%" PRId64 ". %s at line %d res:%d", pKey->ts, __func__, __LINE__, code_file);
} }
if (pFileState->hasFillCatch == false) {
int32_t code_file = streamStateFillDel_rocksdb(pFileState->pFileStore, pKey);
qTrace("force clear expired file, ts:%" PRId64 ". %s at line %d res %d", pKey->ts, __func__, __LINE__, code_file);
}
} }
taosArrayRemoveBatch(pWinStates, 0, size - 1, NULL); taosArrayRemoveBatch(pWinStates, 0, size - 1, NULL);
} }

View File

@ -159,6 +159,11 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery2.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py

View File

@ -0,0 +1,19 @@
#Column Define
#caseID,rerunTimes,Run with Sanitizer,casePath,caseCommand
#NA,NA,y or n,script,./test.sh -f tsim/user/basic.sim
# system test
#
#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_multi_agg.py
#,,n,system-test,python3 ./test.py -f 8-stream/stream_basic.py
# army-test
#,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2
#tsim test
#,,y,script,./test.sh -f tsim/query/timeline.sim
#docs-examples test
#,,n,docs-examples-test,bash c.sh

34
tests/script/api/stmt2-get-fields.c Normal file → Executable file
View File

@ -21,7 +21,7 @@ void getFields(TAOS *taos, const char *sql) {
if (code != 0) { if (code != 0) {
printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt)); printf("failed get col,ErrCode: 0x%x, ErrMessage: %s.\n", code, taos_stmt2_error(stmt));
} else { } else {
printf("col nums:%d\n", fieldNum); printf("bind nums:%d\n", fieldNum);
for (int i = 0; i < fieldNum; i++) { for (int i = 0; i < fieldNum; i++) {
printf("field[%d]: %s, data_type:%d, field_type:%d\n", i, pFields[i].name, pFields[i].type, printf("field[%d]: %s, data_type:%d, field_type:%d\n", i, pFields[i].name, pFields[i].type,
pFields[i].field_type); pFields[i].field_type);
@ -76,7 +76,7 @@ void do_stmt(TAOS *taos) {
// case 4 : INSERT INTO db.? using db.stb TAGS(?,?) VALUES(?,?) // case 4 : INSERT INTO db.? using db.stb TAGS(?,?) VALUES(?,?)
// not support this clause // not support this clause
sql = "insert into db.? using db.stb tags(?, ?) values(?,?)"; sql = "insert into db.? using db.stb tags(?, ?) values(?,?)";
printf("case 4 (not support): %s\n", sql); printf("case 4 : %s\n", sql);
getFields(taos, sql); getFields(taos, sql);
// case 5 : INSERT INTO db.stb(t1,t2,ts,b) values(?,?,?,?) // case 5 : INSERT INTO db.stb(t1,t2,ts,b) values(?,?,?,?)
@ -114,6 +114,36 @@ void do_stmt(TAOS *taos) {
sql = "insert into db.ntb(nts,ni) values(?,?)"; sql = "insert into db.ntb(nts,ni) values(?,?)";
printf("case 10 : %s\n", sql); printf("case 10 : %s\n", sql);
getFields(taos, sql); getFields(taos, sql);
// case 11 : insert into db.? values(?,?)
// normal table must have tbnam
sql = "insert into db.? values(?,?)";
printf("case 11 (normal table must have tbname): %s\n", sql);
getFields(taos, sql);
// case 12 : insert into db.? using db.stb(t2,t1) tags(?, ?) (b,ts)values(?,?)
// disordered
sql = "insert into db.? using db.stb(t2,t1) tags(?, ?) (b,ts)values(?,?)";
printf("case 12 : %s\n", sql);
getFields(taos, sql);
// case 13 : insert into db.? using db.stb tags(?, ?) values(?,?)
// no field name
sql = "insert into db.? using db.stb tags(?, ?) values(?,?)";
printf("case 13 : %s\n", sql);
getFields(taos, sql);
// case 14 : insert into db.? using db.stb tags(?, ?) values(?,?)
// less para
sql = "insert into db.? using db.stb (t2)tags(?) (ts)values(?)";
printf("case 14 : %s\n", sql);
getFields(taos, sql);
// case 15 : insert into db.? using db.stb tags(?, ?) values(?,?)
// less para
sql = "insert into db.d0 (ts)values(?)";
printf("case 15 : %s\n", sql);
getFields(taos, sql);
} }
int main() { int main() {

View File

@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start
sleep 50 sleep 50
sql connect sql connect
print =============== create database print ========================================== create database
sql create database test vgroups 2; sql create database test vgroups 2;
sql select * from information_schema.ins_databases sql select * from information_schema.ins_databases
if $rows != 3 then if $rows != 3 then
@ -18,7 +18,47 @@ sql create stable st(ts timestamp, a int) tags(t int);
sql create table tu1 using st tags(1); sql create table tu1 using st tags(1);
sql create stream stream1 trigger force_window_close into str_dst as select _wstart, count(*) from st partition by tbname interval(5s); sql create stream stream1 trigger force_window_close into str_dst as select _wstart, count(*) from st partition by tbname interval(5s);
run tsim/stream/checkTaskStatus.sim
sql insert into tu1 values(now, 1);
sleep 5500
sql pause stream stream1
$loop_count = 0
loop1:
sleep 500
$loop_count = $loop_count + 1
if $loop_count == 20 then
goto end_loop1
endi
sql insert into tu1 values(now, 1);
goto loop1
end_loop1:
sql resume stream stream1
sleep 5000
sql select sum(`count(*)`) from (select * from str_dst)
if $data00 != 20 then
print expect 20, actual: $data00
return -1
endi
sql drop database test
print ===================================== micro precision db test
print ============ create db
sql create database test vgroups 2 precision 'us';
sql use test
sql create stable st(ts timestamp, a int) tags(t int);
sql create table tu1 using st tags(1);
sql create stream stream1 trigger force_window_close into str_dst as select _wstart, count(*) from st partition by tbname interval(5s);
run tsim/stream/checkTaskStatus.sim run tsim/stream/checkTaskStatus.sim
sql insert into tu1 values(now, 1); sql insert into tu1 values(now, 1);
@ -41,10 +81,58 @@ goto loop0
end_loop: end_loop:
sql resume stream stream1 sql resume stream stream1
sql select * from str_dst sleep 5000
if $rows != 3 then sql select sum(`count(*)`) from (select * from str_dst)
print expect 3, actual: $rows
if $data00 != 20 then
print expect 20, actual: $data00
return -1
endi
sql drop stream stream1
sql drop table str_dst
print ============================= too long watermark test
sql drop table tu1;
sql create table tu1 using st tags(1);
sql create stream stream2 trigger force_window_close watermark 30s into str_dst as select _wstart, count(*), now() from st partition by tbname interval(5s);
run tsim/stream/checkTaskStatus.sim
$loop_count = 0
loop2:
sleep 500
$loop_count = $loop_count + 1
if $loop_count == 20 then
goto end_loop3
endi
sql insert into tu1 values(now, 1);
goto loop2
end_loop3:
sql select count(*) from str_dst
print =================rows: $data00
if $data00 != 0 then
print expect 0, actual $data00
return -1
endi
sleep 35000
sql select sum(`count(*)`) from (select * from str_dst)
if $data00 != 19 then
print expect 19, actual: $data00
return -1
endi
sql select round(timediff(`now()`, `_wstart`)/1000000) from str_dst;
if $data00 != 35.000000000 then
print expect 35.000000000 , actual $data00
return -1
endi endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -1,67 +0,0 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c checkpointInterval -v 60
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 4;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int)tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams1 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt1 as select _wstart, count(a) from st partition by tbname interval(2s);
sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(a) from st interval(2s);
run tsim/stream/checkTaskStatus.sim
sleep 70000
print restart taosd 01 ......
system sh/stop_dnodes.sh
system sh/exec.sh -n dnode1 -s start
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(now + 3000a,1,1,1);
$loop_count = 0
loop0:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print select * from streamt1;
sql select * from streamt1;
print $data00 $data01 $data02
if $rows == 0 then
goto loop0
endi
print select * from streamt2;
sql select * from streamt2;
print $data00 $data01 $data02
if $rows == 0 then
goto loop0
endi
print end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -304,6 +304,253 @@ if $rows != 1 then
return -1 return -1
endi endi
print step3
print =============== create database
sql create database test4 vgroups 4;
sql use test4;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1234567890t1 using st tags(1,1,1);
sql create table t1234567890t2 using st tags(2,2,2);
sql create stable streamt9(ts timestamp,a varchar(10),b tinyint,c tinyint) tags(ta varchar(3),cc int,tc int);
sql create stable streamt10(ts timestamp,a varchar(10),b tinyint,c tinyint) tags(ta varchar(3),cc int,tc int);
sql create stable streamt11(ts timestamp,a varchar(10),b tinyint,c tinyint) tags(ta varchar(3),cc int,tc int);
sql create stream streams9 trigger FORCE_WINDOW_CLOSE IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt9 TAGS(cc,ta) SUBTABLE(concat(concat("tbn-", tbname), "_1")) as select _irowts, interp(a), _isfilled as a1, interp(b) from st partition by tbname as ta, b as cc every(2s) fill(value, 100000,200000);
sql create stream streams10 trigger FORCE_WINDOW_CLOSE IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt10 TAGS(cc,ta) SUBTABLE(concat(concat("tbn-", tbname), "_2")) as select _wstart, twa(a), sum(b),max(c) from st partition by tbname as ta, b as cc interval(2s) fill(NULL);
sql create stream streams11 trigger FORCE_WINDOW_CLOSE IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt11 TAGS(cc,ta) SUBTABLE(concat(concat("tbn-", tbname), "_3")) as select _wstart, count(a),avg(c),min(b) from st partition by tbname as ta, b as cc interval(2s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1234567890t1 values(now + 3s,100000,1,1);
$loop_count = 0
loop9:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print 2 sql select cc,ta, * from streamt9;
sql select cc,ta, * from streamt9;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
# row 0
if $rows < 2 then
print ======rows=$rows
goto loop9
endi
if $data00 != 1 then
return -1
endi
if $data01 != @t12@ then
return -1
endi
if $data03 != @100000@ then
return -1
endi
if $data04 != 1 then
return -1
endi
if $data05 != 64 then
return -1
endi
print 3 sql select * from information_schema.ins_tables where stable_name = "streamt9";
sql select * from information_schema.ins_tables where stable_name = "streamt9";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
print 4 sql select * from information_schema.ins_tables where stable_name = "streamt9" and table_name like "tbn-t1234567890t1_1%";
sql select * from information_schema.ins_tables where stable_name = "streamt9" and table_name like "tbn-t1234567890t1_1%";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
$loop_count = 0
loop10:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print 2 sql select cc,ta, * from streamt10;
sql select cc,ta, * from streamt10;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
# row 0
if $rows < 2 then
print ======rows=$rows
goto loop10
endi
if $data00 != 1 then
return -1
endi
if $data01 != @t12@ then
return -1
endi
if $data03 != @100000.000@ then
return -1
endi
if $data04 != 1 then
return -1
endi
if $data05 != 1 then
return -1
endi
print 3 sql select * from information_schema.ins_tables where stable_name = "streamt10";
sql select * from information_schema.ins_tables where stable_name = "streamt10";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
print 4 sql select * from information_schema.ins_tables where stable_name = "streamt10" and table_name like "tbn-t1234567890t1_2%";
sql select * from information_schema.ins_tables where stable_name = "streamt10" and table_name like "tbn-t1234567890t1_2%";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
$loop_count = 0
loop11:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print 2 sql select cc,ta,* from streamt11;
sql select cc,ta,* from streamt11;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
# row 0
if $rows < 1 then
print ======rows=$rows
goto loop11
endi
if $data00 != 1 then
return -1
endi
if $data01 != @t12@ then
return -1
endi
if $data03 != @1@ then
return -1
endi
if $data04 != 1 then
return -1
endi
if $data05 != 1 then
return -1
endi
print 3 sql select * from information_schema.ins_tables where stable_name = "streamt11";
sql select * from information_schema.ins_tables where stable_name = "streamt11";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
print 4 sql select * from information_schema.ins_tables where stable_name = "streamt11" and table_name like "tbn-t1234567890t1_3%";
sql select * from information_schema.ins_tables where stable_name = "streamt11" and table_name like "tbn-t1234567890t1_3%";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
print end print end
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -0,0 +1,180 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c checkpointInterval -v 60
system sh/cfg.sh -n dnode1 -c ratioOfVnodeStreamThreads -v 4
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int)tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams1 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt1 as select _wstart, count(a),max(b) from st partition by tbname interval(5s);
sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(a), max(b) from st interval(5s);
sql create stream streams3 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt3 as select _wstart, count(a), twa(b) from st partition by tbname interval(5s) fill(prev);
sql create stream streams4 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt4 as select _irowts, interp(a), interp(b) from st partition by tbname every(5s) fill(prev);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(now + 3000a,1,1,1);
$loop_count = 0
loop0:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print select * from streamt3;
sql select * from streamt3;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows == 0 then
goto loop0
endi
print select * from streamt4;
sql select * from streamt4;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows == 0 then
goto loop0
endi
sleep 70000
$loop_count = 0
loop0_1:
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print sql select * from information_schema.ins_stream_tasks where checkpoint_time is null;
sql select * from information_schema.ins_stream_tasks where checkpoint_time is null;
sleep 10000
if $rows > 0 then
print wait checkpoint.rows = $rows
goto loop0_1
endi
print restart taosd 01 ......
system sh/stop_dnodes.sh
system sh/exec.sh -n dnode1 -s start
run tsim/stream/checkTaskStatus.sim
print select * from streamt3;
sql select * from streamt3;
$streamt3_rows = $rows
print =====streamt3_rows=$streamt3_rows
print select * from streamt4;
sql select * from streamt4;
$streamt4_rows = $rows
print =====streamt4_rows=$streamt4_rows
$loop_count = 0
loop1:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print select * from streamt3;
sql select * from streamt3;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows <= $streamt3_rows then
print =====rows=$rows
print =====streamt3_rows=$streamt3_rows
goto loop1
endi
print select * from streamt4;
sql select * from streamt4;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows <= $streamt4_rows then
print =====rows=$rows
print =====streamt4_rows=$streamt4_rows
goto loop1
endi
sql insert into t1 values(now + 3000a,10,10,10);
$loop_count = 0
loop2:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print select * from streamt1 order by 1;
sql select * from streamt1 order by 1;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $data12 != 10 then
goto loop2
endi
print end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -0,0 +1,522 @@
import queue
from random import randrange
import time
import threading
import secrets
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
from datetime import timezone
from tzlocal import get_localzone
# from tmqCommon import *
ROUND: int = 500
class TDTestCase:
updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'debugFlag': 143}
check_failed: bool = False
def __init__(self):
self.vgroups = 4
self.ctbNum = 10
self.rowsPerTbl = 10000
self.duraion = '1h'
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), False)
def create_database(self, tsql, dbName, dropFlag=1, vgroups=2, replica=1, duration: str = '1d'):
if dropFlag == 1:
tsql.execute("drop database if exists %s" % (dbName))
tsql.execute("create database if not exists %s vgroups %d replica %d duration %s" % (
dbName, vgroups, replica, duration))
tdLog.debug("complete to create database %s" % (dbName))
return
def create_stable(self, tsql, paraDict):
colString = tdCom.gen_column_type_str(
colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"])
tagString = tdCom.gen_tag_type_str(
tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"])
sqlString = f"create table if not exists %s.%s (%s) tags (%s)" % (
paraDict["dbName"], paraDict["stbName"], colString, tagString)
tdLog.debug("%s" % (sqlString))
tsql.execute(sqlString)
return
def create_ctable(self, tsql=None, dbName='dbx', stbName='stb', ctbPrefix='ctb', ctbNum=1, ctbStartIdx=0):
for i in range(ctbNum):
sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % (dbName, ctbPrefix, i+ctbStartIdx, dbName, stbName, (i+ctbStartIdx) % 5, i+ctbStartIdx + random.randint(
1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100))
tsql.execute(sqlString)
tdLog.debug("complete to create %d child tables by %s.%s" %
(ctbNum, dbName, stbName))
return
def init_normal_tb(self, tsql, db_name: str, tb_name: str, rows: int, start_ts: int, ts_step: int):
sql = 'CREATE TABLE %s.%s (ts timestamp, c1 INT, c2 INT, c3 INT, c4 double, c5 VARCHAR(255))' % (
db_name, tb_name)
tsql.execute(sql)
sql = 'INSERT INTO %s.%s values' % (db_name, tb_name)
for j in range(rows):
sql += f'(%d, %d,%d,%d,{random.random()},"varchar_%d"),' % (start_ts + j * ts_step + randrange(500), j %
10 + randrange(200), j % 10, j % 10, j % 10 + randrange(100))
tsql.execute(sql)
def insert_data(self, tsql, dbName, ctbPrefix, ctbNum, rowsPerTbl, batchNum, startTs, tsStep):
tdLog.debug("start to insert data ............")
tsql.execute("use %s" % dbName)
pre_insert = "insert into "
sql = pre_insert
for i in range(ctbNum):
rowsBatched = 0
sql += " %s.%s%d values " % (dbName, ctbPrefix, i)
for j in range(rowsPerTbl):
if (i < ctbNum/2):
sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') " % (startTs + j*tsStep + randrange(
500), j % 10 + randrange(100), j % 10 + randrange(200), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
else:
sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') " % (
startTs + j*tsStep + randrange(500), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
rowsBatched += 1
if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
tsql.execute(sql)
rowsBatched = 0
if j < rowsPerTbl - 1:
sql = "insert into %s.%s%d values " % (dbName, ctbPrefix, i)
else:
sql = "insert into "
if sql != pre_insert:
tsql.execute(sql)
tdLog.debug("insert data ............ [OK]")
return
def init_data(self, db: str = 'test', ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500):
tdLog.printNoPrefix(
"======== prepare test env include database, stable, ctables, and insert data: ")
paraDict = {'dbName': db,
'dropFlag': 1,
'vgroups': 4,
'stbName': 'meters',
'colPrefix': 'c',
'tagPrefix': 't',
'colSchema': [{'type': 'INT', 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'FLOAT', 'count': 1}, {'type': 'DOUBLE', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'tinyint', 'count': 1}, {'type': 'bool', 'count': 1}, {'type': 'binary', 'len': 10, 'count': 1}, {'type': 'nchar', 'len': 10, 'count': 1}],
'tagSchema': [{'type': 'INT', 'count': 1}, {'type': 'nchar', 'len': 20, 'count': 1}, {'type': 'binary', 'len': 20, 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'DOUBLE', 'count': 1}],
'ctbPrefix': 't',
'ctbStartIdx': 0,
'ctbNum': ctb_num,
'rowsPerTbl': rows_per_ctb,
'batchNum': 3000,
'startTs': start_ts,
'tsStep': ts_step}
paraDict['vgroups'] = self.vgroups
paraDict['ctbNum'] = ctb_num
paraDict['rowsPerTbl'] = rows_per_ctb
tdLog.info("create database")
self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"],
vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion)
tdLog.info("create stb")
self.create_stable(tsql=tdSql, paraDict=paraDict)
tdLog.info("create child tables")
self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"],
stbName=paraDict["stbName"], ctbPrefix=paraDict["ctbPrefix"],
ctbNum=paraDict["ctbNum"], ctbStartIdx=paraDict["ctbStartIdx"])
self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],
ctbPrefix=paraDict["ctbPrefix"], ctbNum=paraDict["ctbNum"],
rowsPerTbl=paraDict["rowsPerTbl"], batchNum=paraDict["batchNum"],
startTs=paraDict["startTs"], tsStep=paraDict["tsStep"])
self.init_normal_tb(tdSql, paraDict['dbName'], 'norm_tb',
paraDict['rowsPerTbl'], paraDict['startTs'], paraDict['tsStep'])
def run(self):
self.init_data()
self.test_interp_extension()
def datetime_add_tz(self, dt):
if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:
return dt.replace(tzinfo=get_localzone())
return dt
def binary_search_ts(self, select_results, ts):
mid = 0
try:
found: bool = False
start = 0
end = len(select_results) - 1
while start <= end:
mid = (start + end) // 2
if self.datetime_add_tz(select_results[mid][0]) == ts:
found = True
return mid
elif self.datetime_add_tz(select_results[mid][0]) < ts:
start = mid + 1
else:
end = mid - 1
if not found:
tdLog.exit(f"cannot find ts in select results {ts} {select_results}")
return start
except Exception as e:
tdLog.debug(f"{select_results[mid][0]}, {ts}, {len(select_results)}, {select_results[mid]}")
self.check_failed = True
tdLog.exit(f"binary_search_ts error: {e}")
def distance(self, ts1, ts2):
return abs(self.datetime_add_tz(ts1) - self.datetime_add_tz(ts2))
## TODO pass last position to avoid search from the beginning
def is_nearest(self, select_results, irowts_origin, irowts):
if len(select_results) <= 1:
return True
try:
#tdLog.debug(f"check is_nearest for: {irowts_origin} {irowts}")
idx = self.binary_search_ts(select_results, irowts_origin)
if idx == 0:
#tdLog.debug(f"prev row: null,cur row: {select_results[idx]}, next row: {select_results[idx + 1]}")
res = self.distance(irowts, select_results[idx][0]) <= self.distance(irowts, select_results[idx + 1][0])
if not res:
tdLog.debug(f"prev row: null,cur row: {select_results[idx]}, next row: {select_results[idx + 1]}, irowts_origin: {irowts_origin}, irowts: {irowts}")
return res
if idx == len(select_results) - 1:
#tdLog.debug(f"prev row: {select_results[idx - 1]},cur row: {select_results[idx]}, next row: null")
res = self.distance(irowts, select_results[idx][0]) <= self.distance(irowts, select_results[idx - 1][0])
if not res:
tdLog.debug(f"prev row: {select_results[idx - 1]},cur row: {select_results[idx]}, next row: null, irowts_origin: {irowts_origin}, irowts: {irowts}")
return res
#tdLog.debug(f"prev row: {select_results[idx - 1]},cur row: {select_results[idx]}, next row: {select_results[idx + 1]}")
res = self.distance(irowts, select_results[idx][0]) <= self.distance(irowts, select_results[idx - 1][0]) and self.distance(irowts, select_results[idx][0]) <= self.distance(irowts, select_results[idx + 1][0])
if not res:
tdLog.debug(f"prev row: {select_results[idx - 1]},cur row: {select_results[idx]}, next row: {select_results[idx + 1]}, irowts_origin: {irowts_origin}, irowts: {irowts}")
return res
except Exception as e:
self.check_failed = True
tdLog.exit(f"is_nearest error: {e}")
## interp_results: _irowts_origin, _irowts, ..., _isfilled
## select_all_results must be sorted by ts in ascending order
def check_result_for_near(self, interp_results, select_all_results, sql, sql_select_all):
#tdLog.debug(f"check_result_for_near for sql: {sql}, sql_select_all{sql_select_all}")
for row in interp_results:
if row[0].tzinfo is None or row[0].tzinfo.utcoffset(row[0]) is None:
irowts_origin = row[0].replace(tzinfo=get_localzone())
irowts = row[1].replace(tzinfo=get_localzone())
else:
irowts_origin = row[0]
irowts = row[1]
if not self.is_nearest(select_all_results, irowts_origin, irowts):
self.check_failed = True
tdLog.exit(f"interp result is not the nearest for row: {row}, {sql}")
def query_routine(self, sql_queue: queue.Queue, output_queue: queue.Queue):
try:
tdcom = TDCom()
cli = tdcom.newTdSql()
while True:
item = sql_queue.get()
if item is None or self.check_failed:
output_queue.put(None)
break
(sql, sql_select_all, _) = item
cli.query(sql, queryTimes=1)
interp_results = cli.queryResult
if sql_select_all is not None:
cli.query(sql_select_all, queryTimes=1)
output_queue.put((sql, interp_results, cli.queryResult, sql_select_all))
cli.close()
except Exception as e:
self.check_failed = True
tdLog.exit(f"query_routine error: {e}")
def interp_check_near_routine(self, select_all_results, output_queue: queue.Queue):
try:
while True:
item = output_queue.get()
if item is None:
break
(sql, interp_results, all_results, sql_select_all) = item
if all_results is not None:
self.check_result_for_near(interp_results, all_results, sql, sql_select_all)
else:
self.check_result_for_near(interp_results, select_all_results, sql, None)
except Exception as e:
self.check_failed = True
tdLog.exit(f"interp_check_near_routine error: {e}")
def create_qt_threads(self, sql_queue: queue.Queue, output_queue: queue.Queue, num: int):
qts = []
for _ in range(0, num):
qt = threading.Thread(target=self.query_routine, args=(sql_queue, output_queue))
qt.start()
qts.append(qt)
return qts
def wait_qt_threads(self, qts: list):
for qt in qts:
qt.join()
### first(ts) | last(ts)
### 2018-09-17 09:00:00.047 | 2018-09-17 10:23:19.863
def test_interp_fill_extension_near(self):
sql = f"select last(ts), c1, c2 from test.t0"
tdSql.query(sql, queryTimes=1)
lastRow = tdSql.queryResult[0]
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(near)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(61)
for i in range(0, 61):
tdSql.checkData(i, 0, lastRow[0])
tdSql.checkData(i, 2, lastRow[1])
tdSql.checkData(i, 3, lastRow[2])
tdSql.checkData(i, 4, True)
sql = f"select ts, c1, c2 from test.t0 where ts between '2018-09-17 08:59:59' and '2018-09-17 09:00:06' order by ts asc"
tdSql.query(sql, queryTimes=1)
select_all_results = tdSql.queryResult
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 range('2018-09-17 09:00:00', '2018-09-17 09:00:05') every(1s) fill(near)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(6)
self.check_result_for_near(tdSql.queryResult, select_all_results, sql, None)
start = 1537146000000
end = 1537151000000
tdSql.query("select ts, c1, c2 from test.t0 order by ts asc", queryTimes=1)
select_all_results = tdSql.queryResult
qt_threads_num = 4
sql_queue = queue.Queue()
output_queue = queue.Queue()
qts = self.create_qt_threads(sql_queue, output_queue, qt_threads_num)
ct = threading.Thread(target=self.interp_check_near_routine, args=(select_all_results, output_queue))
ct.start()
for i in range(0, ROUND):
range_start = random.randint(start, end)
range_end = random.randint(range_start, end)
every = random.randint(1, 15)
#tdLog.debug(f"range_start: {range_start}, range_end: {range_end}")
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 range({range_start}, {range_end}) every({every}s) fill(near)"
sql_queue.put((sql, None, None))
### no prev only, no next only, no prev and no next, have prev and have next
for i in range(0, ROUND):
range_point = random.randint(start, end)
## all data points are can be filled by near
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 range({range_point}, 1h) fill(near, 1, 2)"
sql_queue.put((sql, None, None))
for i in range(0, ROUND):
range_start = random.randint(start, end)
range_end = random.randint(range_start, end)
range_where_start = random.randint(start, end)
range_where_end = random.randint(range_where_start, end)
every = random.randint(1, 15)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 where ts between {range_where_start} and {range_where_end} range({range_start}, {range_end}) every({every}s) fill(near)"
tdSql.query(f'select to_char(cast({range_where_start} as timestamp), \'YYYY-MM-DD HH24:MI:SS.MS\'), to_char(cast({range_where_end} as timestamp), \'YYYY-MM-DD HH24:MI:SS.MS\')', queryTimes=1)
where_start_str = tdSql.queryResult[0][0]
where_end_str = tdSql.queryResult[0][1]
sql_select_all = f"select ts, c1, c2 from test.t0 where ts between '{where_start_str}' and '{where_end_str}' order by ts asc"
sql_queue.put((sql, sql_select_all, None))
for i in range(0, ROUND):
range_start = random.randint(start, end)
range_end = random.randint(range_start, end)
range_where_start = random.randint(start, end)
range_where_end = random.randint(range_where_start, end)
range_point = random.randint(start, end)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 where ts between {range_where_start} and {range_where_end} range({range_point}, 1h) fill(near, 1, 2)"
tdSql.query(f'select to_char(cast({range_where_start} as timestamp), \'YYYY-MM-DD HH24:MI:SS.MS\'), to_char(cast({range_where_end} as timestamp), \'YYYY-MM-DD HH24:MI:SS.MS\')', queryTimes=1)
where_start_str = tdSql.queryResult[0][0]
where_end_str = tdSql.queryResult[0][1]
sql_select_all = f"select ts, c1, c2 from test.t0 where ts between '{where_start_str}' and '{where_end_str}' order by ts asc"
sql_queue.put((sql, sql_select_all, None))
for i in range(0, qt_threads_num):
sql_queue.put(None)
self.wait_qt_threads(qts)
ct.join()
if self.check_failed:
tdLog.exit("interp check near failed")
def test_interp_extension_irowts_origin(self):
sql = f"select _irowts, _irowts_origin, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(near)"
tdSql.query(sql, queryTimes=1)
sql = f"select _irowts, _irowts_origin, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(NULL)"
tdSql.error(sql, -2147473833)
sql = f"select _irowts, _irowts_origin, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(linear)"
tdSql.error(sql, -2147473833)
sql = f"select _irowts, _irowts_origin, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(NULL_F)"
tdSql.error(sql, -2147473833)
def test_interp_fill_extension(self):
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(near, 0, 0)"
tdSql.query(sql, queryTimes=1)
### must specify value
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(near)"
tdSql.error(sql, -2147473915)
### num of fill value mismatch
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(near, 1)"
tdSql.error(sql, -2147473915)
### range with around interval cannot specify two timepoints, currently not supported
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:02:00', 1h) fill(near, 1, 1)"
tdSql.error(sql, -2147473920) ## syntax error
### NULL/linear cannot specify other values
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:02:00') fill(NULL, 1, 1)"
tdSql.error(sql, -2147473920)
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:02:00') fill(linear, 1, 1)"
tdSql.error(sql, -2147473920) ## syntax error
### cannot have every clause with range around
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) every(1s) fill(prev, 1, 1)"
tdSql.error(sql, -2147473827) ## TSDB_CODE_PAR_INVALID_INTERP_CLAUSE
### cannot specify near/prev/next values when using range
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(near, 1, 1)"
tdSql.error(sql, -2147473915) ## cannot specify values
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00') every(1s) fill(near, 1, 1)"
tdSql.error(sql, -2147473915) ## cannot specify values
### when range around interval is set, only prev/next/near is supported
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(NULL, 1, 1)"
tdSql.error(sql, -2147473920)
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(NULL)"
tdSql.error(sql, -2147473861) ## TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(linear, 1, 1)"
tdSql.error(sql, -2147473920)
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(linear)"
tdSql.error(sql, -2147473861) ## TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE
### range interval cannot be 0
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 0h) fill(near, 1, 1)"
tdSql.error(sql, -2147473861) ## TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1y) fill(near, 1, 1)"
tdSql.error(sql, -2147473915) ## TSDB_CODE_PAR_WRONG_VALUE_TYPE
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1n) fill(near, 1, 1)"
tdSql.error(sql, -2147473915) ## TSDB_CODE_PAR_WRONG_VALUE_TYPE
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters where ts between '2020-02-01 00:00:00' and '2020-02-01 00:00:00' range('2020-02-01 00:00:00', 1h) fill(near, 1, 1)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(0)
### first(ts) | last(ts)
### 2018-09-17 09:00:00.047 | 2018-09-17 10:23:19.863
sql = "select to_char(first(ts), 'YYYY-MM-DD HH24:MI:SS.MS') from test.meters"
tdSql.query(sql, queryTimes=1)
first_ts = tdSql.queryResult[0][0]
sql = "select to_char(last(ts), 'YYYY-MM-DD HH24:MI:SS.MS') from test.meters"
tdSql.query(sql, queryTimes=1)
last_ts = tdSql.queryResult[0][0]
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1d) fill(near, 1, 2)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, '2020-02-01 00:00:00.000')
tdSql.checkData(0, 2, 1)
tdSql.checkData(0, 3, 2)
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2018-09-18 10:25:00', 1d) fill(prev, 3, 4)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, '2018-09-18 10:25:00.000')
tdSql.checkData(0, 2, 3)
tdSql.checkData(0, 3, 4)
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2018-09-16 08:25:00', 1d) fill(next, 5, 6)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, '2018-09-16 08:25:00.000')
tdSql.checkData(0, 2, 5)
tdSql.checkData(0, 3, 6)
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2018-09-16 09:00:01', 1d) fill(next, 1, 2)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, first_ts)
tdSql.checkData(0, 1, '2018-09-16 09:00:01')
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2018-09-18 10:23:19', 1d) fill(prev, 1, 2)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, last_ts)
tdSql.checkData(0, 1, '2018-09-18 10:23:19')
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('{last_ts}', 1a) fill(next, 1, 2)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, last_ts)
tdSql.checkData(0, 1, last_ts)
tdSql.checkData(0, 4, False)
def test_interval_fill_extension(self):
## not allowed
sql = f"select count(*) from test.meters interval(1s) fill(near)"
tdSql.error(sql, -2147473920) ## syntax error
sql = f"select count(*) from test.meters interval(1s) fill(prev, 1)"
tdSql.error(sql, -2147473920) ## syntax error
sql = f"select count(*) from test.meters interval(1s) fill(next, 1)"
tdSql.error(sql, -2147473920) ## syntax error
sql = f"select _irowts_origin, count(*) from test.meters where ts between '2018-09-17 08:59:59' and '2018-09-17 09:00:06' interval(1s) fill(next)"
tdSql.error(sql, -2147473918) ## invalid column name _irowts_origin
def test_interp_fill_extension_stream(self):
## near is not supported
sql = f"create stream s1 trigger force_window_close into test.s_res_tb as select _irowts, interp(c1), interp(c2)from test.meters partition by tbname every(1s) fill(near);"
tdSql.error(sql, -2147473851) ## TSDB_CODE_PAR_INVALID_STREAM_QUERY
## _irowts_origin is not support
sql = f"create stream s1 trigger force_window_close into test.s_res_tb as select _irowts_origin, interp(c1), interp(c2)from test.meters partition by tbname every(1s) fill(prev);"
tdSql.error(sql, -2147473851) ## TSDB_CODE_PAR_INVALID_STREAM_QUERY
sql = f"create stream s1 trigger force_window_close into test.s_res_tb as select _irowts, interp(c1), interp(c2)from test.meters partition by tbname every(1s) fill(next, 1, 1);"
tdSql.error(sql, -2147473915) ## cannot specify values
def test_interp_extension(self):
self.test_interp_fill_extension_near()
self.test_interp_extension_irowts_origin()
self.test_interp_fill_extension()
self.test_interval_fill_extension()
self.test_interp_fill_extension_stream()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
event = threading.Event()
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

View File

@ -93,6 +93,18 @@ class TDTestCase:
tdSql.error( tdSql.error(
f"create stream itp_force_error_1 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 0 into itp_force_error_1 as select _irowts,tbname,_isfilled,interp(c11,1) from {self.stb_name} partition by tbname every(5s) fill(prev) ;" f"create stream itp_force_error_1 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 0 into itp_force_error_1 as select _irowts,tbname,_isfilled,interp(c11,1) from {self.stb_name} partition by tbname every(5s) fill(prev) ;"
) )
tdSql.error(
f"create stream itp_1d_next_error_1 trigger force_window_close FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 1 into itp_1d_next_error_t1 as select _irowts,tbname,_isfilled,interp(current) from {self.stb_name} where groupid=100 partition by every(5s) fill(next) ;"
)
tdSql.error(
f"create stream itp_1d_next_error_1 trigger at_once FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 1 into itp_1d_next_error_t1 as select _irowts,tbname,_isfilled,interp(current) from {self.stb_name} where groupid=100 partition by every(5s) fill(next) ;"
)
tdSql.error(
f"create stream itp_1d_next_error_1 trigger window_close FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 1 into itp_1d_next_error_t1 as select _irowts,tbname,_isfilled,interp(current) from {self.stb_name} where groupid=100 partition by every(5s) fill(next) ;"
)
tdSql.error(
f"create stream itp_1d_next_error_1 trigger max_delay 5s FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 1 into itp_1d_next_error_t1 as select _irowts,tbname,_isfilled,interp(current) from {self.stb_name} where groupid=100 partition by every(5s) fill(next) ;"
)
# function name : interp # function name : interp
trigger_mode = "force_window_close" trigger_mode = "force_window_close"

View File

@ -24,6 +24,7 @@ import time
import traceback import traceback
import os import os
from os import path from os import path
import psutil
class TDTestCase: class TDTestCase:
@ -117,6 +118,69 @@ class TDTestCase:
if not tdSql.getData(2, 0).startswith('new-t3_stb_'): if not tdSql.getData(2, 0).startswith('new-t3_stb_'):
tdLog.exit("error6") tdLog.exit("error6")
def caseDropStream(self):
tdLog.info(f"start caseDropStream")
sql = "drop database if exists d1;"
tdSql.query(sql)
sql = "drop database if exists db;"
tdSql.query(sql)
sql ="show streams;"
tdSql.query(sql)
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
sql ="select * from information_schema.ins_stream_tasks;"
tdSql.query(sql)
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
self.taosBenchmark(" -d db -t 2 -v 2 -n 1000000 -y")
# create stream
tdSql.execute("use db;")
tdSql.execute("create stream stream4 fill_history 1 into sta4 as select _wstart, sum(current),avg(current),last(current),min(voltage),first(voltage),last(phase),max(phase),count(phase), _wend, _wduration from meters partition by tbname, ts interval(10a);", show=True)
time.sleep(10)
sql ="select * from information_schema.ins_stream_tasks where status == 'ready';"
tdSql.query(sql, show=True)
tdSql.check_rows_loop(4, sql, loopCount=100, waitTime=0.5)
pl = psutil.pids()
for pid in pl:
try:
if psutil.Process(pid).name() == 'taosd':
taosdPid = pid
break
except psutil.NoSuchProcess:
pass
tdLog.info("taosd pid:{}".format(taosdPid))
p = psutil.Process(taosdPid)
cpuInfo = p.cpu_percent(interval=5)
tdLog.info("taosd cpu:{}".format(cpuInfo))
tdSql.execute("drop stream stream4;", show=True)
sql ="show streams;"
tdSql.query(sql, show=True)
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
sql ="select * from information_schema.ins_stream_tasks;"
tdSql.query(sql, show=True)
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
for i in range(10):
cpuInfo = p.cpu_percent(interval=5)
tdLog.info("taosd cpu:{}".format(cpuInfo))
if cpuInfo < 10:
return
else:
time.sleep(1)
continue
cpuInfo = p.cpu_percent(interval=5)
tdLog.info("taosd cpu:{}".format(cpuInfo))
if cpuInfo > 10:
tdLog.exit("drop stream failed, stream tasks are still running")
# run # run
def run(self): def run(self):
self.case1() self.case1()
@ -145,6 +209,9 @@ class TDTestCase:
tdSql.query(sql) tdSql.query(sql)
tdSql.checkRows(0) tdSql.checkRows(0)
self.caseDropStream()
# stop # stop
def stop(self): def stop(self):
tdSql.close() tdSql.close()