Merge branch '3.0' into enh/TD-30933-3.0

This commit is contained in:
kailixu 2024-11-30 20:36:20 +08:00
commit 0303480f01
59 changed files with 2070 additions and 436 deletions

View File

@ -337,14 +337,14 @@ Query OK, 2 row(s) in set (0.001700s)
TDengine 提供了丰富的应用程序开发接口,其中包括 C/C++、Java、Python、Go、Node.js、C# 、RESTful 等,便于用户快速开发应用:
- [Java](https://docs.taosdata.com/connector/java/)
- [C/C++](https://docs.taosdata.com/connector/cpp/)
- [Python](https://docs.taosdata.com/connector/python/)
- [Go](https://docs.taosdata.com/connector/go/)
- [Node.js](https://docs.taosdata.com/connector/node/)
- [Rust](https://docs.taosdata.com/connector/rust/)
- [C#](https://docs.taosdata.com/connector/csharp/)
- [RESTful API](https://docs.taosdata.com/connector/rest-api/)
- [Java](https://docs.taosdata.com/reference/connector/java/)
- [C/C++](https://docs.taosdata.com/reference/connector/cpp/)
- [Python](https://docs.taosdata.com/reference/connector/python/)
- [Go](https://docs.taosdata.com/reference/connector/go/)
- [Node.js](https://docs.taosdata.com/reference/connector/node/)
- [Rust](https://docs.taosdata.com/reference/connector/rust/)
- [C#](https://docs.taosdata.com/reference/connector/csharp/)
- [RESTful API](https://docs.taosdata.com/reference/connector/rest-api/)
# 成为社区贡献者

View File

@ -357,14 +357,14 @@ Query OK, 2 row(s) in set (0.001700s)
TDengine provides abundant developing tools for users to develop on TDengine. Follow the links below to find your desired connectors and relevant documentation.
- [Java](https://docs.tdengine.com/reference/connector/java/)
- [C/C++](https://docs.tdengine.com/reference/connector/cpp/)
- [Python](https://docs.tdengine.com/reference/connector/python/)
- [Go](https://docs.tdengine.com/reference/connector/go/)
- [Node.js](https://docs.tdengine.com/reference/connector/node/)
- [Rust](https://docs.tdengine.com/reference/connector/rust/)
- [C#](https://docs.tdengine.com/reference/connector/csharp/)
- [RESTful API](https://docs.tdengine.com/reference/rest-api/)
- [Java](https://docs.tdengine.com/reference/connectors/java/)
- [C/C++](https://docs.tdengine.com/reference/connectors/cpp/)
- [Python](https://docs.tdengine.com/reference/connectors/python/)
- [Go](https://docs.tdengine.com/reference/connectors/go/)
- [Node.js](https://docs.tdengine.com/reference/connectors/node/)
- [Rust](https://docs.tdengine.com/reference/connectors/rust/)
- [C#](https://docs.tdengine.com/reference/connectors/csharp/)
- [RESTful API](https://docs.tdengine.com/reference/connectors/rest-api/)
# Contribute to TDengine

View File

@ -29,14 +29,23 @@ After modifying configuration file parameters, it is necessary to restart the *t
### Connection Related
| Parameter Name | Parameter Description |
| :--------------------- | :----------------------------------------------------------- |
| firstEp | The endpoint of the first dnode in the cluster to connect to when taosd starts; default value: localhost:6030 |
| secondEp | If firstEp cannot connect, attempt to connect to the second dnode's endpoint in the cluster; default value: none |
| fqdn | The service address that taosd listens on after startup; default value: the first hostname configured on the server |
| serverPort | The port that taosd listens on after startup; default value: 6030 |
| numOfRpcSessions | The maximum number of connections a client can create; range: 100-100000; default value: 30000 |
| timeToGetAvailableConn | The maximum wait time to obtain an available connection; range: 10-50000000; unit: milliseconds; default value: 500000 |
| Parameter Name | support version | Parameter Description |
| :--------------------- |:---------------| :----------------------------------------------------------- |
| firstEp | | The endpoint of the first dnode in the cluster to connect to when taosd starts; default value: localhost:6030 |
| secondEp | | If firstEp cannot connect, attempt to connect to the second dnode's endpoint in the cluster; default value: none |
| fqdn | | The service address that taosd listens on after startup; default value: the first hostname configured on the server |
| compressMsgSize | | Whether to compress RPC messages; -1: no messages are compressed; 0: all messages are compressed; N (N>0): only messages larger than N bytes are compressed; default value: -1 |
| shellActivityTimer | | The duration in seconds for the client to send heartbeats to the mnode; range: 1-120; default value: 3 |
| numOfRpcSessions | | The maximum number of RPC connections supported; range: 100-100000; default value: 30000 |
| numOfRpcThreads | | The number of threads for RPC data transmission; range: 1-50, default value: half of the CPU cores |
| numOfTaskQueueThreads | | The number of threads for the client to process RPC messages, range: 4-16, default value: half of the CPU cores |
| rpcQueueMemoryAllowed | | The maximum amount of memory allowed for RPC messages received on a dnode; unit: bytes; range: 104857600-INT64_MAX; default value: 1/10 of server memory |
| resolveFQDNRetryTime | Removed in 3.x | The number of retries when FQDN resolution fails |
| timeToGetAvailableConn | Removed in 3.3.4.x | The maximum waiting time to obtain an available connection; range: 10-50000000; unit: milliseconds; default value: 500000 |
| maxShellConns | Removed in 3.x | The maximum number of connections allowed to be created |
| maxRetryWaitTime | | The maximum timeout for reconnection; default value: 10s |
| shareConnLimit | Added in 3.3.4.0 | The number of requests that a connection can share; range: 1-512; default value: 10 |
| readTimeout | Added in 3.3.4.0 | The minimum timeout for a single request; range: 64-604800; unit: seconds; default value: 900 |
### Monitoring Related

View File

@ -39,6 +39,10 @@ The TDengine client driver provides all the APIs needed for application programm
| enableScience | Whether to enable scientific notation for floating-point numbers; 0: disable, 1: enable; default value: 1 |
| compressMsgSize | Whether to compress RPC messages; -1: do not compress any messages; 0: compress all messages; N (N>0): compress only messages larger than N bytes; default value: -1 |
| queryTableNotExistAsEmpty | Whether to return an empty result set when the queried table does not exist; false: return an error; true: return an empty result set; default value: false |
| numOfRpcThreads | The number of threads for RPC data transmission; range: 1-50, default value: half of the CPU cores |
| numOfTaskQueueThreads | The number of threads for the client to process RPC messages, range: 4-16, default value: half of the CPU cores |
| shareConnLimit | The number of requests that a connection can share; range: 1-512; default value: 10 |
| readTimeout | The minimum timeout for a single request; range: 64-604800; unit: seconds; default value: 900 |
## API

View File

@ -84,6 +84,9 @@ taos -h h1.taos.com -s "use db; show tables;"
You can also control the behavior of the TDengine CLI through parameters set in the configuration file. For available configuration parameters, refer to [Client Configuration](../../components/taosc).
## Error Codes Reference
After version 3.3.4.8 of TDengine, the TDengine CLI returned error codes in the error message. Users can search for the specific cause and solution on the error code page of the TDengine official website, see [Error Codes Table](https://docs.taosdata.com/reference/error-code)
## TDengine CLI Tips
- Use the up and down arrow keys to view previously entered commands.

View File

@ -62,7 +62,8 @@ window_clause: {
| COUNT_WINDOW(count_val[, sliding_val])
interp_clause:
RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val)
RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val)
| RANGE(ts_val, surrounding_time_val) FILL(fill_mod_and_val)
partition_by_clause:
PARTITION BY partition_by_expr [, partition_by_expr] ...
@ -256,6 +257,13 @@ The \_irowts pseudo column can only be used with the interp function to return t
SELECT _irowts, interp(current) FROM meters RANGE('2020-01-01 10:00:00', '2020-01-01 10:30:00') EVERY(1s) FILL(linear);
```
** \_IROWTS\_ORIGIN**
Pseudo column `_irowts_origin` is used to get the original timestamp of the row used for filling. It can only be used with the INTERP query. `_irowts_origin` is not supported in stream. Only FILL PREV/NEXT/NEAR is supported. If there is not data in range, return NULL.
```sql
SELECT _irowts_origin, interp(current) FROM meters RANGE('2020-01-01 10:00:00', '2020-01-01 10:30:00') EVERY(1s) FILL(PREV);
```
## Query Objects
After the FROM keyword, there can be several table (supertable) lists, or the results of subqueries.

View File

@ -1869,6 +1869,9 @@ INTERP(expr [, ignore_null_values])
- INTERP can be used with the pseudo-column _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0 onwards).
- INTERP can also be used with the pseudo-column _isfilled to show whether the returned result is from the original record or produced by the interpolation algorithm (supported from version 3.0.3.0 onwards).
- When querying a table with a composite primary key, if there are multiple records with the same timestamp, only the data corresponding to the minimum composite primary key will participate in the calculation.
- `INTERP` support NEAR fill mode. When `FILL(NEAR)` is used, the nearest value to the interpolation point is used to fill the missing value. If there are multiple values with the same distance to the interpolation point, previous row is used. NEAR fill is not supported in stream computation. For example, `SELECT _irowts,INTERP(current) FROM test.meters RANGE('2017-07-22 00:00:00','2017-07-24 12:25:00') EVERY(1h) FILL(NEAR)`(supported from version 3.3.4.9 onwards).
- Psedo column `_irowts_origin` can be used along with `INTERP` only when using NEAR/NEXT/PREV fill mode, `_irowts_origin` supported from version 3.3.4.9 onwards.
- `INTERP` RANGE clause support INTERVAL extension(supported from version 3.3.4.9 onwards), like `RANGE('2023-01-01 00:00:00', 1d)`. The second parameter is the interval length, and the unit cannot use y(year), n(month). The interval length must be an integer, with no quotes, the value can't be 0. The interval length is used to restrict the search range from the time point specified. For example, `SELECT _irowts,INTERP(current) FROM test.meters RANGE('2017-07-22 00:00:00', 1d) FILL(NEAR, 1)`. The query will return the interpolation result of the current column within the range of 1 day from the time point '2017-07-22 00:00:00'. If there is no data within the range, the specified value in FILL will be used. Only FILL PREV/NEXT/NEAR is supported in this case. It's illegal to use `EVERY` clause and NOT specify values in FILL clause in this case. None data-point range clause with INTERVAL extension is not supported currently, like `RANGE('2017-07-22 00:00:00', '2017-07-22 12:00:00', 1h)` is not supported.
### LAST

View File

@ -250,7 +250,7 @@ flush database stream_dest_db; ---- 流计算写入数据的超级表所在的
```sql
create stream streams1 into test1.streamst as select _wstart, count(a) c1 from test.st interval(1s) ;
drop database streams1;
drop stream streams1;
flush database test;
flush database test1;
```

View File

@ -27,24 +27,23 @@ taosd 命令行参数如下
### 连接相关
|参数名称|支持版本|参数含义|
|-----------------------|----------|-|
|firstEp | |taosd 启动时,主动连接的集群中首个 dnode 的 end point默认值 localhost:6030|
|secondEp | |taosd 启动时,如果 firstEp 连接不上,尝试连接集群中第二个 dnode 的 endpoint无默认值|
|fqdn | |taosd 监听的服务地址,默认为所在服务器上配置的第一个 hostname|
|serverPort | |taosd 监听的端口,默认值 6030|
|compressMsgSize | |是否对 RPC 消息进行压缩;-1所有消息都不压缩0所有消息都压缩N (N>0):只有大于 N 个字节的消息才压缩;默认值 -1|
|shellActivityTimer | |客户端向 mnode 发送心跳的时长,单位为秒,取值范围 1-120默认值 3|
|numOfRpcSessions | |RPC 支持的最大连接数,取值范围 100-100000默认值 30000|
|numOfRpcThreads | |RPC 线程数目,默认值为 CPU 核数的一半|
|numOfTaskQueueThreads | |dnode 处理 RPC 消息的线程数|
|statusInterval | |dnode 与 mnode 之间的心跳间隔|
|rpcQueueMemoryAllowed | |dnode 允许的 rpc 消息占用的内存最大值,单位 bytes取值范围 104857600-INT64_MAX默认值 服务器内存的 1/10 |
|resolveFQDNRetryTime | |FQDN 解析失败时的重试次数|
|timeToGetAvailableConn | |获得可用连接的最长等待时间,取值范围 10-50000000单位为毫秒默认值 500000|
|maxShellConns | |允许创建的最大链接数|
|maxRetryWaitTime | |重连最大超时时间|
|shareConnLimit |3.3.4.3 后|内部参数,一个链接可以共享的查询数目,取值范围 1-256默认值 10|
|readTimeout |3.3.4.3 后|内部参数,最小超时时间,取值范围 64-604800单位为秒默认值 900|
|-----------------------|-------------------------|------------|
|firstEp | |taosd 启动时,主动连接的集群中首个 dnode 的 end point默认值 localhost:6030|
|secondEp | |taosd 启动时,如果 firstEp 连接不上,尝试连接集群中第二个 dnode 的 endpoint无默认值|
|fqdn | |taosd 监听的服务地址,默认为所在服务器上配置的第一个 hostname|
|serverPort | |taosd 监听的端口,默认值 6030|
|compressMsgSize | |是否对 RPC 消息进行压缩;-1所有消息都不压缩0所有消息都压缩N (N>0):只有大于 N 个字节的消息才压缩;默认值 -1|
|shellActivityTimer | |客户端向 mnode 发送心跳的时长,单位为秒,取值范围 1-120默认值 3 |
|numOfRpcSessions | |RPC 支持的最大连接数,取值范围 100-100000默认值 30000|
|numOfRpcThreads | |RPC 收发数据线程数目取值范围1-50,默认值为 CPU 核数的一半|
|numOfTaskQueueThreads | |客户端处理 RPC 消息的线程数取值, 范围4-16,默认值为 CPU 核数的一半|
|rpcQueueMemoryAllowed | |dnode允许的已经收到的RPC消息占用的内存最大值单位 bytes取值范围 104857600-INT64_MAX默认值为服务器内存的 1/10 |
|resolveFQDNRetryTime | 3.x 之后取消 |FQDN 解析失败时的重试次数|
|timeToGetAvailableConn | 3.3.4.x之后取消 |获得可用连接的最长等待时间,取值范围 10-50000000单位为毫秒默认值 500000|
|maxShellConns | 3.x 后取消 |允许创建的最大链接数|
|maxRetryWaitTime | |重连最大超时时间, 默认值是 10s|
|shareConnLimit |3.3.4.0 新增 |一个链接可以共享的请求的数目,取值范围 1-512默认值 10|
|readTimeout |3.3.4.0 新增 |单个请求最小超时时间,取值范围 64-604800单位为秒默认值 900|
### 监控相关
|参数名称|支持版本|参数含义|

View File

@ -10,17 +10,18 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API并且在
### 连接相关
|参数名称|支持版本|参数含义|
|----------------------|----------|-|
|firstEp | |启动时,主动连接的集群中首个 dnode 的 endpoint缺省值hostname:6030若无法获取该服务器的 hostname则赋值为 localhost|
|secondEp | |启动时,如果 firstEp 连接不上,尝试连接集群中第二个 dnode 的 endpoint没有缺省值|
|compressMsgSize | |是否对 RPC 消息进行压缩;-1所有消息都不压缩0所有消息都压缩N (N>0):只有大于 N 个字节的消息才压缩;缺省值 -1|
|shellActivityTimer | |客户端向 mnode 发送心跳的时长,单位为秒,取值范围 1-120默认值 3|
|numOfRpcSessions | |RPC 支持的最大连接数,取值范围 100-100000缺省值 30000|
|numOfRpcThreads | |RPC 线程数目,默认值为 CPU 核数的一半|
|timeToGetAvailableConn| |获得可用连接的最长等待时间,取值范围 10-50000000单位为毫秒缺省值 500000|
|----------------------|----------|-------------|
|firstEp | |启动时,主动连接的集群中首个 dnode 的 endpoint缺省值hostname:6030若无法获取该服务器的 hostname则赋值为 localhost|
|secondEp | |启动时,如果 firstEp 连接不上,尝试连接集群中第二个 dnode 的 endpoint没有缺省值|
|compressMsgSize | |是否对 RPC 消息进行压缩;-1所有消息都不压缩0所有消息都压缩N (N>0):只有大于 N 个字节的消息才压缩;缺省值 -1|
|shellActivityTimer | |客户端向 mnode 发送心跳的时长,单位为秒,取值范围 1-120默认值 3|
|numOfRpcSessions | |RPC 支持的最大连接数,取值范围 100-100000缺省值 30000|
|numOfRpcThreads | |RPC 收发数据线程数目取值范围1-50,默认值为 CPU 核数的一半|
|numOfTaskQueueThreads | |客户端处理 RPC消息的线程数, 范围4-16,默认值为 CPU 核数的一半|
|timeToGetAvailableConn| 3.3.4.*之后取消 |获得可用连接的最长等待时间,取值范围 10-50000000单位为毫秒缺省值 500000|
|useAdapter | |内部参数,是否使用 taosadapter影响 CSV 文件导入|
|shareConnLimit |3.3.4.3 后|内部参数,一个链接可以共享的查询数目,取值范围 1-256默认值 10|
|readTimeout |3.3.4.3 后|内部参数,最小超时时间,取值范围 64-604800单位为秒默认值 900|
|shareConnLimit |3.3.4.0 新增|内部参数,一个链接可以共享的查询数目,取值范围 1-256默认值 10|
|readTimeout |3.3.4.0 新增|内部参数,最小超时时间,取值范围 64-604800单位为秒默认值 900|
### 查询相关
|参数名称|支持版本|参数含义|

View File

@ -90,7 +90,7 @@ taos -h h1.taos.com -s "use db; show tables;"
也可以通过配置文件中的参数设置来控制 TDengine CLI 的行为。可用配置参数请参考[客户端配置](../../components/taosc)
## 错误代码表
在 TDengine 3.3.5.0 版本后 TDengine CLI 在返回的错误信息中包含了具体的错误代码,用户可到 TDengine 官网的错误代码详细说明页面查找具体原因及解决措施,见:[错误码参考表](../error_code/)
在 TDengine 3.3.4.8 版本后 TDengine CLI 在返回错误信息中返回了具体错误码,用户可到 TDengine 官网错误码页面查找具体原因及解决措施,见:[错误码参考表](https://docs.taosdata.com/reference/error-code/)
## TDengine CLI TAB 键补全

View File

@ -62,7 +62,8 @@ window_clause: {
| COUNT_WINDOW(count_val[, sliding_val])
interp_clause:
RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val)
RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val)
| RANGE(ts_val, surrounding_time_val) FILL(fill_mod_and_val)
partition_by_clause:
PARTITION BY partition_by_expr [, partition_by_expr] ...
@ -255,6 +256,13 @@ select _rowts, max(current) from meters;
select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear);
```
**\_IROWTS\_ORIGIN**
`_irowts_origin` 伪列只能与 interp 函数一起使用,不支持在流计算中使用, 仅适用于FILL类型为PREV/NEXT/NEAR, 用于返回 interp 函数所使用的原始数据的时间戳列。若范围内无值, 则返回 NULL。
```sql
select _iorwts_origin, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(NEXT);
```
## 查询对象
FROM 关键字后面可以是若干个表(超级表)列表,也可以是子查询的结果。

View File

@ -1838,6 +1838,9 @@ ignore_null_values: {
- INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.2.0 版本以后支持)。
- INTERP 可以与伪列 _isfilled 一起使用,显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0 版本以后支持)。
- INTERP 对于带复合主键的表的查询,若存在相同时间戳的数据,则只有对应的复合主键最小的数据参与运算。
- INTERP 查询支持NEAR FILL模式, 即当需要FILL时, 使用距离当前时间点最近的数据进行插值, 当前后时间戳与当前时间断面一样近时, FILL 前一行的值. 此模式在流计算中和窗口查询中不支持。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)。(3.3.4.9版本及以后支持)。
- INTERP 只有在使用FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`。`_irowts_origin`在3.3.4.9版本及以后支持。
- INTERP `RANEG`子句支持时间范围的扩展(3.3.4.9版本及以后支持), 如`RANGE('2023-01-01 00:00:00', 10s)`表示在时间点'2023-01-01 00:00:00'查找前后10s的数据进行插值, FILL PREV/NEXT/NEAR分别表示从时间点向前/向后/前后查找数据, 若时间点周围没有数据, 则使用FILL指定的值进行插值, 因此此时FILL子句必须指定值。例如: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1). 目前仅支持时间点和时间范围的组合, 不支持时间区间和时间范围的组合, 即不支持RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h)。所指定的时间范围规则与EVERY类似, 单位不能是年或月, 值不能为0, 不能带引号。使用该扩展时, 不支持除FILL PREV/NEXT/NEAR外的其他FILL模式, 且不能指定EVERY子句。
### LAST

View File

@ -188,6 +188,7 @@ typedef enum _mgmt_table {
#define TSDB_FILL_LINEAR 5
#define TSDB_FILL_PREV 6
#define TSDB_FILL_NEXT 7
#define TSDB_FILL_NEAR 8
#define TSDB_ALTER_USER_PASSWD 0x1
#define TSDB_ALTER_USER_SUPERUSER 0x2
@ -264,6 +265,7 @@ typedef enum ENodeType {
QUERY_NODE_COLUMN_OPTIONS,
QUERY_NODE_TSMA_OPTIONS,
QUERY_NODE_ANOMALY_WINDOW,
QUERY_NODE_RANGE_AROUND,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100,

View File

@ -155,6 +155,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_FORECAST_LOW,
FUNCTION_TYPE_FORECAST_HIGH,
FUNCTION_TYPE_FORECAST_ROWTS,
FUNCTION_TYPE_IROWTS_ORIGIN,
// internal function
FUNCTION_TYPE_SELECT_VALUE = 3750,
@ -289,6 +290,7 @@ bool fmIsPrimaryKeyFunc(int32_t funcId);
bool fmIsProcessByRowFunc(int32_t funcId);
bool fmisSelectGroupConstValueFunc(int32_t funcId);
bool fmIsElapsedFunc(int32_t funcId);
bool fmIsRowTsOriginFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);

View File

@ -213,6 +213,8 @@ typedef struct SInterpFuncLogicNode {
EFillMode fillMode;
SNode* pFillValues; // SNodeListNode
SNode* pTimeSeries; // SColumnNode
int64_t rangeInterval;
int8_t rangeIntervalUnit;
SStreamNodeOption streamNodeOption;
} SInterpFuncLogicNode;
@ -528,6 +530,8 @@ typedef struct SInterpFuncPhysiNode {
SNode* pFillValues; // SNodeListNode
SNode* pTimeSeries; // SColumnNode
SStreamNodeOption streamNodeOption;
int64_t rangeInterval;
int8_t rangeIntervalUnit;
} SInterpFuncPhysiNode;
typedef SInterpFuncPhysiNode SStreamInterpFuncPhysiNode;

View File

@ -362,7 +362,8 @@ typedef enum EFillMode {
FILL_MODE_NULL,
FILL_MODE_NULL_F,
FILL_MODE_LINEAR,
FILL_MODE_NEXT
FILL_MODE_NEXT,
FILL_MODE_NEAR,
} EFillMode;
typedef enum ETimeLineMode {
@ -407,6 +408,11 @@ typedef struct SWindowOffsetNode {
SNode* pEndOffset; // SValueNode
} SWindowOffsetNode;
typedef struct SRangeAroundNode {
ENodeType type;
SNode* pTimepoint;
SNode* pInterval;
} SRangeAroundNode;
typedef struct SSelectStmt {
ENodeType type; // QUERY_NODE_SELECT_STMT
@ -421,6 +427,7 @@ typedef struct SSelectStmt {
SNodeList* pGroupByList; // SGroupingSetNode
SNode* pHaving;
SNode* pRange;
SNode* pRangeAround;
SNode* pEvery;
SNode* pFill;
SNodeList* pOrderByList; // SOrderByExprNode

View File

@ -93,7 +93,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream
const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen);
SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen);
int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId);
int32_t recoverSession(SStreamFileState* pFileState, int64_t ckId);
void sessionWinStateClear(SStreamFileState* pFileState);
void sessionWinStateCleanup(void* pBuff);

45
packaging/smokeTest/test_smoking_selfhost.sh Normal file → Executable file
View File

@ -4,11 +4,13 @@
LOG_FILE="test_server.log"
SUCCESS_FILE="success.txt"
FAILED_FILE="failed.txt"
REPORT_FILE="report.txt"
# Initialize/clear result files
> "$SUCCESS_FILE"
> "$FAILED_FILE"
> "$LOG_FILE"
> "$REPORT_FILE"
# Switch to the target directory
TARGET_DIR="../../tests/system-test/"
@ -22,18 +24,17 @@ else
exit 1
fi
# Define the Python commands to execute case list
# Define the Python commands to execute
commands=(
"python3 ./test.py -f 2-query/join.py"
"python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3"
"python3 ./test.py -f 1-insert/insert_column_value.py"
"python3 ./test.py -f 2-query/primary_ts_base_5.py"
"python3 ./test.py -f 2-query/case_when.py"
"python3 ./test.py -f 2-query/partition_limit_interval.py"
"python3 ./test.py -f 2-query/fill.py"
"python3 ./test.py -f query/query_basic.py -N 3"
"python3 ./test.py -f 7-tmq/basic5.py"
"python3 ./test.py -f 8-stream/stream_basic.py"
"python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3"
)
# Counters
@ -45,6 +46,7 @@ fail_count=0
for cmd in "${commands[@]}"
do
echo "===== Executing Command: $cmd =====" | tee -a "$LOG_FILE"
# Execute the command and append output and errors to the log file
eval "$cmd" >> "$LOG_FILE" 2>&1
exit_code=$?
@ -58,6 +60,7 @@ do
echo "$cmd" >> "$FAILED_FILE"
((fail_count++))
fi
echo "" | tee -a "$LOG_FILE" # Add an empty line for separation
done
@ -72,23 +75,31 @@ if [ $fail_count -ne 0 ]; then
echo "The following commands failed:" | tee -a "$LOG_FILE"
cat "$FAILED_FILE" | tee -a "$LOG_FILE"
else
echo "All commands executed successfully." | tee -a "$LOG_FILE"
echo "All commands executed successfully. Deleting log and result files..." | tee -a "$LOG_FILE"
rm -f "$LOG_FILE" "$SUCCESS_FILE" "$FAILED_FILE" "$REPORT_FILE"
echo "Log and result files deleted."
fi
# Optional: Generate a separate report file
echo "" > "report.txt"
echo "===== Test Report =====" >> "report.txt"
echo "Total Commands Executed: $total" >> "report.txt"
echo "Successful: $success_count" >> "report.txt"
echo "Failed: $fail_count" >> "report.txt"
# Generate a separate report file if there are failed commands
if [ $fail_count -ne 0 ]; then
echo "" >> "report.txt"
echo "The following commands failed:" >> "report.txt"
cat "$FAILED_FILE" >> "report.txt"
echo "" >> "$REPORT_FILE"
echo "===== Test Report =====" >> "$REPORT_FILE"
echo "Total Commands Executed: $total" >> "$REPORT_FILE"
echo "Successful: $success_count" >> "$REPORT_FILE"
echo "Failed: $fail_count" >> "$REPORT_FILE"
echo "" >> "$REPORT_FILE"
echo "The following commands failed:" >> "$REPORT_FILE"
cat "$FAILED_FILE" >> "$REPORT_FILE"
else
echo "All commands executed successfully." >> "report.txt"
echo "===== Test Report =====" > "$REPORT_FILE"
echo "Total Commands Executed: $total" >> "$REPORT_FILE"
echo "Successful: $success_count" >> "$REPORT_FILE"
echo "Failed: $fail_count" >> "$REPORT_FILE"
echo "All commands executed successfully." >> "$REPORT_FILE"
fi
echo "Detailed logs can be found in $LOG_FILE"
echo "Test report can be found in report.txt"
# Print the absolute paths of the log and result files
echo "Detailed logs can be found in: $(realpath "$LOG_FILE")"
echo "Successful commands can be found in: $(realpath "$SUCCESS_FILE")"
echo "Failed commands can be found in: $(realpath "$FAILED_FILE")"
echo "Test report can be found in: $(realpath "$REPORT_FILE")"

View File

@ -97,6 +97,7 @@ else
${build_dir}/bin/${clientName} \
${taostools_bin_files} \
${build_dir}/bin/${clientName}adapter \
${build_dir}/bin/${clientName}keeper \
${build_dir}/bin/udfd \
${script_dir}/remove.sh \
${script_dir}/set_core.sh \
@ -138,10 +139,16 @@ mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${
if [ -f "${compile_dir}/test/cfg/${clientName}adapter.toml" ]; then
cp ${compile_dir}/test/cfg/${clientName}adapter.toml ${install_dir}/cfg || :
fi
if [ -f "${compile_dir}/test/cfg/${clientName}keeper.toml" ]; then
cp ${compile_dir}/test/cfg/${clientName}keeper.toml ${install_dir}/cfg || :
fi
if [ -f "${compile_dir}/test/cfg/${clientName}adapter.service" ]; then
cp ${compile_dir}/test/cfg/${clientName}adapter.service ${install_dir}/cfg || :
fi
if [ -f "${compile_dir}/test/cfg/${clientName}keeper.service" ]; then
cp ${compile_dir}/test/cfg/${clientName}keeper.service ${install_dir}/cfg || :
fi
if [ -f "${cfg_dir}/${serverName}.service" ]; then
cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || :
@ -422,19 +429,19 @@ if [ "$exitcode" != "0" ]; then
exit $exitcode
fi
if [ -n "${taostools_bin_files}" ] && [ "$verMode" != "cloud" ]; then
wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh"
if [ "$osType" != "Darwin" ]; then
tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || :
else
tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" || :
rm -rf ${taostools_install_dir} ||:
fi
exitcode=$?
if [ "$exitcode" != "0" ]; then
echo "tar ${taostools_pkg_name}.tar.gz error !!!"
exit $exitcode
fi
fi
# if [ -n "${taostools_bin_files}" ] && [ "$verMode" != "cloud" ]; then
# wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh"
# if [ "$osType" != "Darwin" ]; then
# tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || :
# else
# tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" || :
# rm -rf ${taostools_install_dir} ||:
# fi
# exitcode=$?
# if [ "$exitcode" != "0" ]; then
# echo "tar ${taostools_pkg_name}.tar.gz error !!!"
# exit $exitcode
# fi
# fi
cd ${curr_dir}

View File

@ -475,6 +475,7 @@ typedef struct SStreamFillSupporter {
STimeWindow winRange;
int32_t pkColBytes;
__compar_fn_t comparePkColFn;
int32_t* pOffsetInfo;
} SStreamFillSupporter;
typedef struct SStreamScanInfo {
@ -822,10 +823,11 @@ typedef struct SStreamFillOperatorInfo {
int32_t primaryTsCol;
int32_t primarySrcSlotId;
SStreamFillInfo* pFillInfo;
SStreamAggSupporter* pStreamAggSup;
SArray* pCloseTs;
SArray* pUpdated;
SGroupResInfo groupResInfo;
SStreamState* pState;
SStateStore stateStore;
} SStreamFillOperatorInfo;
typedef struct SStreamTimeSliceOperatorInfo {
@ -884,6 +886,7 @@ typedef struct SStreamIntervalSliceOperatorInfo {
struct SOperatorInfo* pOperator;
bool hasFill;
bool hasInterpoFunc;
int32_t* pOffsetInfo;
} SStreamIntervalSliceOperatorInfo;
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
@ -1059,6 +1062,7 @@ void destroyFlusedPos(void* pRes);
bool isIrowtsPseudoColumn(SExprInfo* pExprInfo);
bool isIsfilledPseudoColumn(SExprInfo* pExprInfo);
bool isInterpFunc(SExprInfo* pExprInfo);
bool isIrowtsOriginPseudoColumn(SExprInfo* pExprInfo);
int32_t encodeSSessionKey(void** buf, SSessionKey* key);
void* decodeSSessionKey(void* buf, SSessionKey* key);

View File

@ -150,7 +150,7 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo);
int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);
int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pInfo);

View File

@ -90,19 +90,21 @@ int32_t saveTimeSliceWinResult(SWinKey* pKey, SSHashObj* pUpdatedMap);
int winPosCmprImpl(const void* pKey1, const void* pKey2);
void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI);
SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index);
SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index, int32_t* pCellOffsetInfo);
int32_t getDownstreamRes(struct SOperatorInfo* downstream, SSDataBlock** ppRes, SColumnInfo** ppPkCol);
void destroyFlusedppPos(void* ppRes);
void doBuildStreamIntervalResult(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
SGroupResInfo* pGroupResInfo);
void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal,
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol);
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol, int32_t* pCellOffsetInfo);
int32_t getQualifiedRowNumDesc(SExprSupp* pExprSup, SSDataBlock* pBlock, TSKEY* tsCols, int32_t rowId, bool ignoreNull);
int32_t createStreamIntervalSliceOperatorInfo(struct SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
struct SOperatorInfo** ppOptInfo);
int32_t buildAllResultKey(SStreamAggSupporter* pAggSup, TSKEY ts, SArray* pUpdated);
int32_t buildAllResultKey(SStateStore* pStateStore, SStreamState* pState, TSKEY ts, SArray* pUpdated);
int32_t initOffsetInfo(int32_t** ppOffset, SSDataBlock* pRes);
TSKEY compareTs(void* pKey);
#ifdef __cplusplus
}

View File

@ -119,7 +119,8 @@ typedef struct SStreamFillInfo {
int32_t delIndex;
uint64_t curGroupId;
bool hasNext;
SResultRowData* pNonFillRow;
SResultRowData* pNonFillRow;
void* pTempBuff;
} SStreamFillInfo;
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);

View File

@ -2390,6 +2390,9 @@ int32_t convertFillType(int32_t mode) {
case FILL_MODE_LINEAR:
type = TSDB_FILL_LINEAR;
break;
case FILL_MODE_NEAR:
type = TSDB_FILL_NEAR;
break;
default:
type = TSDB_FILL_NONE;
}

View File

@ -614,7 +614,7 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
} else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) {
code = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL == type) {
code = createStreamFillOperatorInfo(ops[0], (SStreamFillPhysiNode*)pPhyNode, pTaskInfo, &pOptr);
code = createStreamFillOperatorInfo(ops[0], (SStreamFillPhysiNode*)pPhyNode, pTaskInfo, pHandle, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) {
code = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo, &pOptr);
} else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) {

View File

@ -100,6 +100,8 @@ void destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
taosMemoryFree(pFillSup->next.pRowVal);
taosMemoryFree(pFillSup->nextNext.pRowVal);
taosMemoryFree(pFillSup->pOffsetInfo);
taosMemoryFree(pFillSup);
}
@ -129,6 +131,7 @@ void destroyStreamFillInfo(SStreamFillInfo* pFillInfo) {
pFillInfo->pLinearInfo = NULL;
taosArrayDestroy(pFillInfo->delRanges);
taosMemoryFreeClear(pFillInfo->pTempBuff);
taosMemoryFree(pFillInfo);
}
@ -148,6 +151,14 @@ static void destroyStreamFillOperatorInfo(void* param) {
clearGroupResInfo(&pInfo->groupResInfo);
taosArrayDestroy(pInfo->pCloseTs);
if (pInfo->stateStore.streamFileStateDestroy != NULL) {
pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState);
}
if (pInfo->pState != NULL) {
taosMemoryFreeClear(pInfo->pState);
}
taosMemoryFree(pInfo);
}
@ -1157,14 +1168,19 @@ _end:
return code;
}
static void resetForceFillWindow(SResultRowData* pRowData) {
pRowData->key = INT64_MIN;
pRowData->pRowVal = NULL;
}
void doBuildForceFillResultImpl(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup,
SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
void* pState = pOperator->pTaskInfo->streamInfo.pState;
bool res = false;
int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
SStreamFillOperatorInfo* pInfo = pOperator->info;
bool res = false;
int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
for (; pGroupResInfo->index < numOfRows; pGroupResInfo->index++) {
SWinKey* pKey = (SWinKey*)taosArrayGet(pGroupResInfo->pRows, pGroupResInfo->index);
if (pBlock->info.id.groupId == 0) {
@ -1172,25 +1188,30 @@ void doBuildForceFillResultImpl(SOperatorInfo* pOperator, SStreamFillSupporter*
} else if (pBlock->info.id.groupId != pKey->groupId) {
break;
}
void* val = NULL;
int32_t len = 0;
int32_t winCode = pAPI->stateStore.streamStateFillGet(pOperator->pTaskInfo->streamInfo.pState, pKey, (void**)&val, &len, NULL);
SRowBuffPos* pValPos = NULL;
int32_t len = 0;
int32_t winCode = TSDB_CODE_SUCCESS;
code = pInfo->stateStore.streamStateFillGet(pInfo->pState, pKey, (void**)&pValPos, &len, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
qDebug("===stream=== build force fill res. key:%" PRId64 ",groupId:%" PRId64".res:%d", pKey->ts, pKey->groupId, winCode);
if (winCode == TSDB_CODE_SUCCESS) {
pFillSup->cur.key = pKey->ts;
pFillSup->cur.pRowVal = val;
pFillSup->cur.pRowVal = pValPos->pRowBuff;
code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res);
QUERY_CHECK_CODE(code, lino, _end);
resetFillWindow(&pFillSup->cur);
resetForceFillWindow(&pFillSup->cur);
releaseOutputBuf(pInfo->pState, pValPos, &pInfo->stateStore);
} else {
SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, pKey);
SWinKey preKey = {.ts = INT64_MIN, .groupId = pKey->groupId};
void* preVal = NULL;
int32_t preVLen = 0;
winCode = pAPI->stateStore.streamStateFillGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen);
SWinKey preKey = {.ts = INT64_MIN, .groupId = pKey->groupId};
SRowBuffPos* prePos = NULL;
int32_t preVLen = 0;
code = pInfo->stateStore.streamStateFillGetPrev(pInfo->pState, pKey, &preKey,
(void**)&prePos, &preVLen, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
if (winCode == TSDB_CODE_SUCCESS) {
pFillSup->cur.key = pKey->ts;
pFillSup->cur.pRowVal = preVal;
pFillSup->cur.pRowVal = prePos->pRowBuff;
if (pFillInfo->type == TSDB_FILL_PREV) {
code = buildFillResult(&pFillSup->cur, pFillSup, pKey->ts, pBlock, &res);
QUERY_CHECK_CODE(code, lino, _end);
@ -1200,9 +1221,9 @@ void doBuildForceFillResultImpl(SOperatorInfo* pOperator, SStreamFillSupporter*
code = buildFillResult(pFillInfo->pResRow, pFillSup, pKey->ts, pBlock, &res);
QUERY_CHECK_CODE(code, lino, _end);
}
resetFillWindow(&pFillSup->cur);
resetForceFillWindow(&pFillSup->cur);
}
pAPI->stateStore.streamStateFreeCur(pCur);
releaseOutputBuf(pInfo->pState, prePos, &pInfo->stateStore);
}
}
@ -1247,6 +1268,45 @@ _end:
return code;
}
static void keepResultInStateBuf(SStreamFillOperatorInfo* pInfo, uint64_t groupId, SResultRowData* pRow) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SWinKey key = {.groupId = groupId, .ts = pRow->key};
int32_t curVLen = 0;
SRowBuffPos* pStatePos = NULL;
int32_t winCode = TSDB_CODE_SUCCESS;
code = pInfo->stateStore.streamStateFillAddIfNotExist(pInfo->pState, &key, (void**)&pStatePos,
&curVLen, &winCode);
QUERY_CHECK_CODE(code, lino, _end);
memcpy(pStatePos->pRowBuff, pRow->pRowVal, pInfo->pFillSup->rowSize);
qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
}
}
int32_t keepBlockRowInStateBuf(SStreamFillOperatorInfo* pInfo, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol,
int32_t rowId, uint64_t groupId, int32_t rowSize) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
TSKEY ts = tsCol[rowId];
pFillInfo->nextRowKey = ts;
TAOS_MEMSET(pFillInfo->pTempBuff, 0, rowSize);
SResultRowData tmpNextRow = {.key = ts, .pRowVal = pFillInfo->pTempBuff};
transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow);
keepResultInStateBuf(pInfo, groupId, &tmpNextRow);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
// force window close impl
static int32_t doStreamForceFillImpl(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
@ -1257,11 +1317,10 @@ static int32_t doStreamForceFillImpl(SOperatorInfo* pOperator) {
SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
SSDataBlock* pBlock = pInfo->pSrcBlock;
uint64_t groupId = pBlock->info.id.groupId;
SStreamAggSupporter* pAggSup = pInfo->pStreamAggSup;
SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol);
TSKEY* tsCol = (TSKEY*)pTsCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++){
code = keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, i, groupId, pFillSup->rowSize);
code = keepBlockRowInStateBuf(pInfo, pFillInfo, pBlock, tsCol, i, groupId, pFillSup->rowSize);
QUERY_CHECK_CODE(code, lino, _end);
int32_t size = taosArrayGetSize(pInfo->pCloseTs);
@ -1281,7 +1340,7 @@ static int32_t doStreamForceFillImpl(SOperatorInfo* pOperator) {
}
}
}
code = pAggSup->stateStore.streamStateGroupPut(pAggSup->pState, groupId, NULL, 0);
code = pInfo->stateStore.streamStateGroupPut(pInfo->pState, groupId, NULL, 0);
QUERY_CHECK_CODE(code, lino, _end);
_end:
@ -1291,13 +1350,13 @@ _end:
return code;
}
int32_t buildAllResultKey(SStreamAggSupporter* pAggSup, TSKEY ts, SArray* pUpdated) {
int32_t buildAllResultKey(SStateStore* pStateStore, SStreamState* pState, TSKEY ts, SArray* pUpdated) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int64_t groupId = 0;
SStreamStateCur* pCur = pAggSup->stateStore.streamStateGroupGetCur(pAggSup->pState);
SStreamStateCur* pCur = pStateStore->streamStateGroupGetCur(pState);
while (1) {
int32_t winCode = pAggSup->stateStore.streamStateGroupGetKVByCur(pCur, &groupId, NULL, NULL);
int32_t winCode = pStateStore->streamStateGroupGetKVByCur(pCur, &groupId, NULL, NULL);
if (winCode != TSDB_CODE_SUCCESS) {
break;
}
@ -1305,14 +1364,14 @@ int32_t buildAllResultKey(SStreamAggSupporter* pAggSup, TSKEY ts, SArray* pUpdat
void* pPushRes = taosArrayPush(pUpdated, &key);
QUERY_CHECK_NULL(pPushRes, code, lino, _end, terrno);
pAggSup->stateStore.streamStateGroupCurNext(pCur);
pStateStore->streamStateGroupCurNext(pCur);
}
pAggSup->stateStore.streamStateFreeCur(pCur);
pStateStore->streamStateFreeCur(pCur);
pCur = NULL;
_end:
if (code != TSDB_CODE_SUCCESS) {
pAggSup->stateStore.streamStateFreeCur(pCur);
pStateStore->streamStateFreeCur(pCur);
pCur = NULL;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
@ -1345,7 +1404,8 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
(*ppRes) = resBlock;
goto _end;
}
pInfo->pStreamAggSup->stateStore.streamStateClearExpiredState(pInfo->pStreamAggSup->pState);
pInfo->stateStore.streamStateClearExpiredState(pInfo->pState);
setStreamOperatorCompleted(pOperator);
(*ppRes) = NULL;
goto _end;
@ -1372,7 +1432,11 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
pInfo->srcRowIndex = -1;
} break;
case STREAM_CHECKPOINT:
case STREAM_CHECKPOINT: {
pInfo->stateStore.streamStateCommit(pInfo->pState);
(*ppRes) = pBlock;
goto _end;
} break;
case STREAM_CREATE_CHILD_TABLE: {
(*ppRes) = pBlock;
goto _end;
@ -1393,7 +1457,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pCloseTs); i++) {
TSKEY ts = *(TSKEY*) taosArrayGet(pInfo->pCloseTs, i);
code = buildAllResultKey(pInfo->pStreamAggSup, ts, pInfo->pUpdated);
code = buildAllResultKey(&pInfo->stateStore, pInfo->pState, ts, pInfo->pUpdated);
QUERY_CHECK_CODE(code, lino, _end);
}
taosArrayClear(pInfo->pCloseTs);
@ -1412,7 +1476,7 @@ static int32_t doStreamForceFillNext(SOperatorInfo* pOperator, SSDataBlock** ppR
QUERY_CHECK_CODE(code, lino, _end);
if ((*ppRes) == NULL) {
pInfo->pStreamAggSup->stateStore.streamStateClearExpiredState(pInfo->pStreamAggSup->pState);
pInfo->stateStore.streamStateClearExpiredState(pInfo->pState);
setStreamOperatorCompleted(pOperator);
}
@ -1573,10 +1637,7 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->pResRow->key = INT64_MIN;
pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize);
if (!pFillInfo->pResRow->pRowVal) {
code = terrno;
QUERY_CHECK_CODE(code, lino, _end);
}
QUERY_CHECK_NULL(pFillInfo->pResRow->pRowVal, code, lino, _end, terrno);
for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i);
@ -1590,6 +1651,21 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pCell->type = pColData->info.type;
}
int32_t numOfResCol = taosArrayGetSize(pRes->pDataBlock);
if (numOfResCol < pFillSup->numOfAllCols) {
int32_t* pTmpBuf = (int32_t*)taosMemoryRealloc(pFillSup->pOffsetInfo, pFillSup->numOfAllCols * sizeof(int32_t));
QUERY_CHECK_NULL(pTmpBuf, code, lino, _end, terrno);
pFillSup->pOffsetInfo = pTmpBuf;
SResultCellData* pCell = getResultCell(pFillInfo->pResRow, numOfResCol - 1);
int32_t preLength = pFillSup->pOffsetInfo[numOfResCol - 1] + pCell->bytes + sizeof(SResultCellData);
for (int32_t i = numOfResCol; i < pFillSup->numOfAllCols; i++) {
pFillSup->pOffsetInfo[i] = preLength;
pCell = getResultCell(pFillInfo->pResRow, i);
preLength += pCell->bytes + sizeof(SResultCellData);
}
}
pFillInfo->pNonFillRow = taosMemoryCalloc(1, sizeof(SResultRowData));
QUERY_CHECK_NULL(pFillInfo->pNonFillRow, code, lino, _end, terrno);
pFillInfo->pNonFillRow->key = INT64_MIN;
@ -1607,6 +1683,7 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock*
pFillInfo->delIndex = 0;
pFillInfo->curGroupId = 0;
pFillInfo->hasNext = false;
pFillInfo->pTempBuff = taosMemoryCalloc(1, pFillSup->rowSize);
return pFillInfo;
_end:
@ -1650,21 +1727,18 @@ static void setValueForFillInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo*
}
}
int32_t getDownStreamInfo(SOperatorInfo* downstream, int8_t* triggerType, SInterval* pInterval, SStreamAggSupporter** ppAggSup) {
int32_t getDownStreamInfo(SOperatorInfo* downstream, int8_t* triggerType, SInterval* pInterval) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (IS_NORMAL_INTERVAL_OP(downstream)) {
SStreamIntervalOperatorInfo* pInfo = downstream->info;
*triggerType = pInfo->twAggSup.calTrigger;
*pInterval = pInfo->interval;
(*ppAggSup) = NULL;
} else if (IS_CONTINUE_INTERVAL_OP(downstream)) {
SStreamIntervalSliceOperatorInfo* pInfo = downstream->info;
*triggerType = pInfo->twAggSup.calTrigger;
*pInterval = pInfo->interval;
pInfo->hasFill = true;
(*ppAggSup) = &pInfo->streamAggSup;
pInfo->streamAggSup.stateStore.streamStateSetFillInfo(pInfo->streamAggSup.pState);
} else {
code = TSDB_CODE_STREAM_INTERNAL_ERROR;
}
@ -1677,8 +1751,31 @@ _end:
return code;
}
int32_t initFillOperatorStateBuff(SStreamFillOperatorInfo* pInfo, SStreamState* pState, SStateStore* pStore,
SReadHandle* pHandle, const char* taskIdStr, SStorageAPI* pApi) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
pInfo->stateStore = *pStore;
pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
QUERY_CHECK_NULL(pInfo->pState, code, lino, _end, terrno);
*(pInfo->pState) = *pState;
pInfo->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsCol);
code = pInfo->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->pFillSup->rowSize, 0, compareTs,
pInfo->pState, INT64_MAX, taskIdStr, pHandle->checkpointId,
STREAM_STATE_BUFF_HASH_SORT, &pInfo->pState->pFileState);
QUERY_CHECK_CODE(code, lino, _end);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode,
SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
int32_t code = TSDB_CODE_SUCCESS;
@ -1704,7 +1801,7 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi
int8_t triggerType = 0;
SInterval interval = {0};
code = getDownStreamInfo(downstream, &triggerType, &interval, &pInfo->pStreamAggSup);
code = getDownStreamInfo(downstream, &triggerType, &interval);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->pFillSup = initStreamFillSup(pPhyFillNode, &interval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI,
@ -1759,9 +1856,13 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi
pTaskInfo);
if (triggerType == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {
code = initFillOperatorStateBuff(pInfo, pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.stateStore, pHandle,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI);
QUERY_CHECK_CODE(code, lino, _error);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamForceFillNext, NULL, destroyStreamFillOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
} else {
pInfo->pState = NULL;
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFillNext, NULL, destroyStreamFillOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
}

View File

@ -74,6 +74,7 @@ void destroyStreamIntervalSliceOperatorInfo(void* param) {
blockDataDestroy(pInfo->pDelRes);
blockDataDestroy(pInfo->pCheckpointRes);
taosMemoryFreeClear(pInfo->pOffsetInfo);
taosMemoryFreeClear(param);
}
@ -163,7 +164,7 @@ _end:
}
void doStreamSliceInterpolation(SSliceRowData* pPrevWinVal, TSKEY winKey, TSKEY curTs, SSDataBlock* pDataBlock,
int32_t curRowIndex, SExprSupp* pSup, SIntervalSliceType type) {
int32_t curRowIndex, SExprSupp* pSup, SIntervalSliceType type, int32_t* pOffsetInfo) {
SqlFunctionCtx* pCtx = pSup->pCtx;
for (int32_t k = 0; k < pSup->numOfExprs; ++k) {
if (!fmIsIntervalInterpoFunc(pCtx[k].functionId)) {
@ -175,7 +176,7 @@ void doStreamSliceInterpolation(SSliceRowData* pPrevWinVal, TSKEY winKey, TSKEY
SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, pParam->pCol->slotId);
double prevVal = 0, curVal = 0, winVal = 0;
SResultCellData* pCell = getSliceResultCell((SResultCellData*)pPrevWinVal->pRowVal, pParam->pCol->slotId);
SResultCellData* pCell = getSliceResultCell((SResultCellData*)pPrevWinVal->pRowVal, pParam->pCol->slotId, pOffsetInfo);
GET_TYPED_DATA(prevVal, double, pCell->type, pCell->pData);
GET_TYPED_DATA(curVal, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex));
@ -278,7 +279,7 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
doSetElapsedEndKey(prevPoint.winKey.win.ekey, &pOperator->exprSupp);
doStreamSliceInterpolation(prevPoint.pLastRow, prevPoint.winKey.win.ekey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_END);
doStreamSliceInterpolation(prevPoint.pLastRow, prevPoint.winKey.win.ekey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_END, pInfo->pOffsetInfo);
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &prevPoint.winKey.win, 1);
code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,
0, pBlock->info.rows, numOfOutput);
@ -294,7 +295,7 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
resetIntervalSliceFunctionKey(pSup->pCtx, numOfOutput);
if (pInfo->hasInterpoFunc && IS_VALID_WIN_KEY(prevPoint.winKey.win.skey) && curPoint.winKey.win.skey != curTs) {
doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_START);
doStreamSliceInterpolation(prevPoint.pLastRow, curPoint.winKey.win.skey, curTs, pBlock, startPos, &pOperator->exprSupp, INTERVAL_SLICE_START, pInfo->pOffsetInfo);
}
forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, curWin.ekey, binarySearchForKey, NULL,
TSDB_ORDER_ASC);
@ -302,7 +303,7 @@ static int32_t doStreamIntervalSliceAggImpl(SOperatorInfo* pOperator, SSDataBloc
if (pInfo->hasInterpoFunc && winCode != TSDB_CODE_SUCCESS) {
int32_t endRowId = getQualifiedRowNumDesc(pSup, pBlock, tsCols, prevEndPos, false);
TSKEY endRowTs = tsCols[endRowId];
transBlockToSliceResultRow(pBlock, endRowId, endRowTs, curPoint.pLastRow, 0, NULL, NULL);
transBlockToSliceResultRow(pBlock, endRowId, endRowTs, curPoint.pLastRow, 0, NULL, NULL, pInfo->pOffsetInfo);
}
SWinKey curKey = {.ts = curPoint.winKey.win.skey, .groupId = curPoint.winKey.groupId};
if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS) {
@ -359,9 +360,14 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
return code;
}
if (pInfo->hasFill == false) {
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
}
if (pInfo->recvCkBlock) {
pInfo->recvCkBlock = false;
printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pCheckpointRes;
return code;
}
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
setStreamOperatorCompleted(pOperator);
(*ppRes) = NULL;
return code;
@ -392,8 +398,6 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
case STREAM_CHECKPOINT: {
pInfo->recvCkBlock = true;
pAggSup->stateStore.streamStateCommit(pAggSup->pState);
// doStreamIntervalSliceSaveCheckpoint(pOperator);
pInfo->recvCkBlock = true;
code = copyDataBlock(pInfo->pCheckpointRes, pBlock);
QUERY_CHECK_CODE(code, lino, _end);
continue;
@ -451,9 +455,13 @@ static int32_t doStreamIntervalSliceNext(SOperatorInfo* pOperator, SSDataBlock**
QUERY_CHECK_CODE(code, lino, _end);
if ((*ppRes) == NULL) {
if (pInfo->hasFill == false) {
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
}
if (pInfo->recvCkBlock) {
pInfo->recvCkBlock = false;
printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
(*ppRes) = pInfo->pCheckpointRes;
return code;
}
pAggSup->stateStore.streamStateClearExpiredState(pAggSup->pState);
setStreamOperatorCompleted(pOperator);
}
@ -596,6 +604,9 @@ int32_t createStreamIntervalSliceOperatorInfo(SOperatorInfo* downstream, SPhysiN
code = getDownstreamRes(downstream, &pDownRes, &pPkCol);
QUERY_CHECK_CODE(code, lino, _error);
code = initOffsetInfo(&pInfo->pOffsetInfo, pDownRes);
QUERY_CHECK_CODE(code, lino, _error);
int32_t keyBytes = sizeof(TSKEY);
keyBytes += blockDataGetRowSize(pDownRes) + sizeof(SResultCellData) * taosArrayGetSize(pDownRes->pDataBlock) + sizeof(bool);
if (pPkCol) {

View File

@ -281,8 +281,34 @@ static int32_t initTimeSliceResultBuf(SStreamFillSupporter* pFillSup, SExprSupp*
return TSDB_CODE_SUCCESS;
}
static int32_t initTimeSliceFillSup(SStreamInterpFuncPhysiNode* pPhyFillNode, SExprSupp* pExprSup, int32_t numOfExprs, SColumnInfo* pPkCol,
SStreamFillSupporter** ppResFillSup) {
int32_t initOffsetInfo(int32_t** ppOffset, SSDataBlock* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t numOfCol = taosArrayGetSize(pRes->pDataBlock);
int32_t preLength = 0;
int32_t* pOffsetInfo = taosMemoryCalloc(numOfCol, sizeof(int32_t));
QUERY_CHECK_NULL(pOffsetInfo, code, lino, _end, lino);
for (int32_t i = 0; i < numOfCol; i++) {
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, i);
pOffsetInfo[i] = preLength;
int32_t bytes = 1;
if (pColInfo != NULL) {
bytes = pColInfo->info.bytes;
}
preLength += bytes + sizeof(SResultCellData);
}
(*ppOffset) = pOffsetInfo;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static int32_t initTimeSliceFillSup(SStreamInterpFuncPhysiNode* pPhyFillNode, SExprSupp* pExprSup, int32_t numOfExprs,
SSDataBlock* pInputRes, SColumnInfo* pPkCol, SStreamFillSupporter** ppResFillSup) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter));
@ -320,6 +346,9 @@ static int32_t initTimeSliceFillSup(SStreamInterpFuncPhysiNode* pPhyFillNode, SE
pFillSup->comparePkColFn = NULL;
}
code = initOffsetInfo(&pFillSup->pOffsetInfo, pInputRes);
QUERY_CHECK_CODE(code, lino, _end);
(*ppResFillSup) = pFillSup;
_end:
@ -359,17 +388,11 @@ _end:
}
}
SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index) {
SResultCellData* getSliceResultCell(SResultCellData* pRowVal, int32_t index, int32_t* pCellOffsetInfo) {
if (!pRowVal) {
return NULL;
}
char* pData = (char*)pRowVal;
SResultCellData* pCell = pRowVal;
for (int32_t i = 0; i < index; i++) {
pData += (pCell->bytes + sizeof(SResultCellData));
pCell = (SResultCellData*)pData;
}
return pCell;
return POINTER_SHIFT(pRowVal, pCellOffsetInfo[index]);
}
static bool isGroupKeyFunc(SExprInfo* pExprInfo) {
@ -414,9 +437,9 @@ static int32_t fillPointResult(SStreamFillSupporter* pFillSup, SResultRowData* p
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = NULL;
if (IS_FILL_CONST_VALUE(pFillSup->type) && (isGroupKeyFunc(pFillCol->pExpr) || isSelectGroupConstValueFunc(pFillCol->pExpr)) ) {
pCell = getSliceResultCell(pNonFillRow->pRowVal, srcSlot);
pCell = getSliceResultCell(pNonFillRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
} else {
pCell = getSliceResultCell(pResRow->pRowVal, srcSlot);
pCell = getSliceResultCell(pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
}
code = setRowCell(pDstCol, pBlock->info.rows, pCell);
QUERY_CHECK_CODE(code, lino, _end);
@ -475,7 +498,7 @@ static void fillLinearRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFi
QUERY_CHECK_CODE(code, lino, _end);
} else if (isInterpFunc(pFillCol->pExpr)) {
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot);
SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) {
colDataSetNULL(pDstCol, index);
continue;
@ -498,7 +521,7 @@ static void fillLinearRange(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFi
destroySPoint(&cur);
} else {
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot);
SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
code = setRowCell(pDstCol, pBlock->info.rows, pCell);
QUERY_CHECK_CODE(code, lino, _end);
}
@ -956,8 +979,8 @@ static void copyNonFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo
if (!isInterpFunc(pFillCol->pExpr) && !isIrowtsPseudoColumn(pFillCol->pExpr) &&
!isIsfilledPseudoColumn(pFillCol->pExpr)) {
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pSrcCell = getResultCell(&pFillSup->cur, srcSlot);
SResultCellData* pDestCell = getResultCell(pFillInfo->pNonFillRow, srcSlot);
SResultCellData* pSrcCell = getSliceResultCell(pFillSup->cur.pRowVal, srcSlot, pFillSup->pOffsetInfo);
SResultCellData* pDestCell = getSliceResultCell(pFillInfo->pNonFillRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
pDestCell->isNull = pSrcCell->isNull;
if (!pDestCell->isNull) {
memcpy(pDestCell->pData, pSrcCell->pData, pSrcCell->bytes);
@ -966,11 +989,11 @@ static void copyNonFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo
}
}
static void copyCalcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, int32_t numOfCol) {
static void copyCalcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, int32_t numOfCol, int32_t* pOffsetInfo) {
for (int32_t i = 0; i < numOfCol; i++) {
if (isInterpFunc(pFillCol[i].pExpr)) {
int32_t slotId = pFillCol[i].pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pECell = getResultCell(pEndRow, slotId);
SResultCellData* pECell = getSliceResultCell(pEndRow->pRowVal, slotId, pOffsetInfo);
SPoint* pPoint = taosArrayGet(pEndPoins, slotId);
pPoint->key = pEndRow->key;
memcpy(pPoint->val, pECell->pData, pECell->bytes);
@ -1112,7 +1135,7 @@ static void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo
SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd);
pFillSup->next.key = pFillSup->nextOriginKey;
copyCalcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols);
pFillSup->numOfAllCols, pFillSup->pOffsetInfo);
pFillSup->prev.key = pFillSup->prevOriginKey;
pFillInfo->pResRow = &pFillSup->prev;
pFillInfo->pLinearInfo->hasNext = false;
@ -1121,7 +1144,7 @@ static void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo
pFillInfo->pos = FILL_POS_END;
SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd);
copyCalcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols);
pFillSup->numOfAllCols, pFillSup->pOffsetInfo);
pFillSup->prev.key = pFillSup->prevOriginKey;
pFillInfo->pResRow = &pFillSup->prev;
pFillInfo->pLinearInfo->hasNext = false;
@ -1132,7 +1155,7 @@ static void setTimeSliceFillRule(SStreamFillSupporter* pFillSup, SStreamFillInfo
SET_WIN_KEY_INVALID(pFillInfo->pLinearInfo->nextEnd);
pFillSup->next.key = pFillSup->nextOriginKey;
copyCalcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
pFillSup->numOfAllCols);
pFillSup->numOfAllCols, pFillSup->pOffsetInfo);
pFillInfo->pResRow = &pFillSup->cur;
pFillInfo->pLinearInfo->hasNext = false;
}
@ -1253,11 +1276,11 @@ static bool needAdjustValue(SSlicePoint* pPoint, TSKEY ts, void* pPkVal, SStream
}
void transBlockToSliceResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SSliceRowData* pRowVal,
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol) {
int32_t rowSize, void* pPkData, SColumnInfoData* pPkCol, int32_t* pCellOffsetInfo) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SResultCellData* pCell = getSliceResultCell((SResultCellData*)pRowVal->pRowVal, i);
SResultCellData* pCell = getSliceResultCell((SResultCellData*)pRowVal->pRowVal, i, pCellOffsetInfo);
if (!colDataIsNull_s(pColData, rowId)) {
pCell->isNull = false;
pCell->type = pColData->info.type;
@ -1378,7 +1401,7 @@ static void doStreamTimeSliceImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock)
}
right = needAdjustValue(&curPoint, tsCols[startPos], pPkVal, pFillSup, false, pFillSup->type);
if (right) {
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo);
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo, pFillSup->pOffsetInfo);
bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS;
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &curPoint.key, pInfo->pUpdatedMap, needDel,
pInfo->pDeletedMap);
@ -1397,7 +1420,7 @@ static void doStreamTimeSliceImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock)
}
left = needAdjustValue(&nextPoint, tsCols[leftRowId], pPkVal, pFillSup, true, pFillSup->type);
if (left) {
transBlockToSliceResultRow(pBlock, leftRowId, tsCols[leftRowId], nextPoint.pLeftRow, pFillSup->rowSize, pPkVal, pPkColDataInfo);
transBlockToSliceResultRow(pBlock, leftRowId, tsCols[leftRowId], nextPoint.pLeftRow, pFillSup->rowSize, pPkVal, pPkColDataInfo, pFillSup->pOffsetInfo);
bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS;
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &nextPoint.key, pInfo->pUpdatedMap,
needDel, pInfo->pDeletedMap);
@ -1422,7 +1445,7 @@ static void doStreamTimeSliceImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock)
}
right = needAdjustValue(&curPoint, tsCols[startPos], pPkVal, pFillSup, false, pFillSup->type);
if (right) {
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo);
transBlockToSliceResultRow(pBlock, startPos, tsCols[startPos], curPoint.pRightRow, pFillSup->rowSize, pPkVal, pPkColDataInfo, pFillSup->pOffsetInfo);
bool needDel = pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS;
code = saveTimeSliceWinResultInfo(pAggSup, &pInfo->twAggSup, &curPoint.key, pInfo->pUpdatedMap, needDel,
pInfo->pDeletedMap);
@ -1886,7 +1909,7 @@ static int32_t doStreamTimeSliceNext(SOperatorInfo* pOperator, SSDataBlock** ppR
qDebug("===stream===build stream result, ts count:%d", size);
for (int32_t i = 0; i < size; i++) {
TSKEY ts = *(TSKEY*) taosArrayGet(pInfo->pCloseTs, i);
code = buildAllResultKey(&pInfo->streamAggSup, ts, pInfo->pUpdated);
code = buildAllResultKey(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pState, ts, pInfo->pUpdated);
QUERY_CHECK_CODE(code, lino, _end);
}
qDebug("===stream===build stream result, res count:%ld", taosArrayGetSize(pInfo->pUpdated));
@ -1951,7 +1974,7 @@ static void copyFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo* p
continue;
}
int32_t srcSlot = pFillCol->pExpr->base.pParam[0].pCol->slotId;
SResultCellData* pCell = getResultCell(pFillInfo->pResRow, srcSlot);
SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, srcSlot, pFillSup->pOffsetInfo);
SFillColInfo* pValueCol = pFillSup->pAllColInfo + valueIndex;
SVariant* pVar = &(pValueCol->fillVal);
if (pCell->type == TSDB_DATA_TYPE_FLOAT) {
@ -1975,7 +1998,7 @@ static void copyFillValueInfo(SStreamFillSupporter* pFillSup, SStreamFillInfo* p
for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
SFillColInfo* pFillCol = pFillSup->pAllColInfo + i;
int32_t slotId = GET_DEST_SLOT_ID(pFillCol);
SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId);
SResultCellData* pCell = getSliceResultCell(pFillInfo->pResRow->pRowVal, slotId, pFillSup->pOffsetInfo);
pCell->isNull = true;
}
}
@ -2095,7 +2118,7 @@ int32_t createStreamTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
QUERY_CHECK_CODE(code, lino, _error);
pInfo->pFillSup = NULL;
code = initTimeSliceFillSup(pInterpPhyNode, pExpSup, numOfExprs, pPkCol, &pInfo->pFillSup);
code = initTimeSliceFillSup(pInterpPhyNode, pExpSup, numOfExprs, pDownRes, pPkCol, &pInfo->pFillSup);
QUERY_CHECK_CODE(code, lino, _error);
int32_t ratio = 1;

View File

@ -1834,7 +1834,7 @@ int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption) {
return deleteMark;
}
static TSKEY compareTs(void* pKey) {
TSKEY compareTs(void* pKey) {
SWinKey* pWinKey = (SWinKey*)pKey;
return pWinKey->ts;
}

View File

@ -48,6 +48,7 @@ typedef struct STimeSliceOperatorInfo {
int32_t remainIndex; // the remaining index in the block to be processed
bool hasPk;
SColumn pkCol;
int64_t rangeInterval;
} STimeSliceOperatorInfo;
static void destroyTimeSliceOperatorInfo(void* param);
@ -179,6 +180,11 @@ bool isIsfilledPseudoColumn(SExprInfo* pExprInfo) {
return (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_isfilled") == 0);
}
bool isIrowtsOriginPseudoColumn(SExprInfo* pExprInfo) {
const char* name = pExprInfo->pExpr->_function.functionName;
return (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_irowts_origin") == 0);
}
static void tRowGetKeyFromColData(int64_t ts, SColumnInfoData* pPkCol, int32_t rowIndex, SRowKey* pKey) {
pKey->ts = ts;
pKey->numOfPKs = 1;
@ -277,6 +283,79 @@ bool checkNullRow(SExprSupp* pExprSup, SSDataBlock* pSrcBlock, int32_t index, bo
return false;
}
static int32_t interpColSetKey(SColumnInfoData* pDst, int32_t rowNum, SGroupKeys* pKey) {
int32_t code = 0;
if (pKey->isNull == false) {
code = colDataSetVal(pDst, rowNum, pKey->pData, false);
} else {
colDataSetNULL(pDst, rowNum);
}
return code;
}
static bool interpSetFillRowWithRangeIntervalCheck(STimeSliceOperatorInfo* pSliceInfo, SArray** ppFillRow, SArray* pFillRefRow, int64_t fillRefRowTs) {
*ppFillRow = NULL;
if (pSliceInfo->rangeInterval <= 0 || llabs(fillRefRowTs - pSliceInfo->current) <= pSliceInfo->rangeInterval) {
*ppFillRow = pFillRefRow;
return true;
}
return false;
}
static bool interpDetermineNearFillRow(STimeSliceOperatorInfo* pSliceInfo, SArray** ppNearRow) {
if (!pSliceInfo->isPrevRowSet && !pSliceInfo->isNextRowSet) {
*ppNearRow = NULL;
return false;
}
SGroupKeys *pPrevTsKey = NULL, *pNextTsKey = NULL;
int64_t* pPrevTs = NULL, *pNextTs = NULL;
if (pSliceInfo->isPrevRowSet) {
pPrevTsKey = taosArrayGet(pSliceInfo->pPrevRow, pSliceInfo->tsCol.slotId);
pPrevTs = (int64_t*)pPrevTsKey->pData;
}
if (pSliceInfo->isNextRowSet) {
pNextTsKey = taosArrayGet(pSliceInfo->pNextRow, pSliceInfo->tsCol.slotId);
pNextTs = (int64_t*)pNextTsKey->pData;
}
if (!pPrevTsKey) {
*ppNearRow = pSliceInfo->pNextRow;
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppNearRow, pSliceInfo->pNextRow, *pNextTs);
} else if (!pNextTsKey) {
*ppNearRow = pSliceInfo->pPrevRow;
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppNearRow, pSliceInfo->pPrevRow, *pPrevTs);
} else {
if (llabs(pSliceInfo->current - *pPrevTs) <= llabs(*pNextTs - pSliceInfo->current)) {
// take prev if euqal
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppNearRow, pSliceInfo->pPrevRow, *pPrevTs);
} else {
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppNearRow, pSliceInfo->pNextRow, *pNextTs);
}
}
return true;
}
static bool interpDetermineFillRefRow(STimeSliceOperatorInfo* pSliceInfo, SArray** ppOutRow) {
bool needFill = false;
if (pSliceInfo->fillType == TSDB_FILL_PREV) {
if (pSliceInfo->isPrevRowSet) {
SGroupKeys* pTsCol = taosArrayGet(pSliceInfo->pPrevRow, pSliceInfo->tsCol.slotId);
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppOutRow, pSliceInfo->pPrevRow, *(int64_t*)pTsCol->pData);
needFill = true;
}
} else if (pSliceInfo->fillType == TSDB_FILL_NEXT) {
if (pSliceInfo->isNextRowSet) {
SGroupKeys* pTsCol = taosArrayGet(pSliceInfo->pNextRow, pSliceInfo->tsCol.slotId);
(void)interpSetFillRowWithRangeIntervalCheck(pSliceInfo, ppOutRow, pSliceInfo->pNextRow, *(int64_t*)pTsCol->pData);
needFill = true;
}
} else if (pSliceInfo->fillType == TSDB_FILL_NEAR) {
needFill = interpDetermineNearFillRow(pSliceInfo, ppOutRow);
} else {
needFill = true;
}
return needFill;
}
static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock,
SSDataBlock* pSrcBlock, int32_t index, bool beforeTs, SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
@ -290,6 +369,8 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
int32_t fillColIndex = 0;
int32_t groupKeyIndex = 0;
bool hasInterp = true;
SArray* pFillRefRow = NULL;
bool needFill = interpDetermineFillRefRow(pSliceInfo, &pFillRefRow);
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
@ -305,7 +386,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
code = colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false);
QUERY_CHECK_CODE(code, lino, _end);
continue;
} else if (!isInterpFunc(pExprInfo)) {
} else if (!isInterpFunc(pExprInfo) && !isIrowtsOriginPseudoColumn(pExprInfo)) {
if (isGroupKeyFunc(pExprInfo) || isSelectGroupConstValueFunc(pExprInfo)) {
if (pSrcBlock != NULL) {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
@ -344,7 +425,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
continue;
}
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
int32_t srcSlot = isIrowtsOriginPseudoColumn(pExprInfo) ? pSliceInfo->tsCol.slotId : pExprInfo->base.pParam[0].pCol->slotId;
switch (pSliceInfo->fillType) {
case TSDB_FILL_NULL:
case TSDB_FILL_NULL_F: {
@ -352,6 +433,25 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
break;
}
case TSDB_FILL_PREV:
case TSDB_FILL_NEAR:
case TSDB_FILL_NEXT: {
if (!needFill) {
hasInterp = false;
break;
}
if (pFillRefRow) {
code = interpColSetKey(pDst, rows, taosArrayGet(pFillRefRow, srcSlot));
QUERY_CHECK_CODE(code, lino, _end);
break;
}
// no fillRefRow, fall through to fill specified values
if (srcSlot == pSliceInfo->tsCol.slotId) {
// if is _irowts_origin, there is no value to fill, just set to null
colDataSetNULL(pDst, rows);
break;
}
}
case TSDB_FILL_SET_VALUE:
case TSDB_FILL_SET_VALUE_F: {
SVariant* pVar = &pSliceInfo->pFillColInfo[fillColIndex].fillVal;
@ -444,38 +544,6 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
taosMemoryFree(current.val);
break;
}
case TSDB_FILL_PREV: {
if (!pSliceInfo->isPrevRowSet) {
hasInterp = false;
break;
}
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
if (pkey->isNull == false) {
code = colDataSetVal(pDst, rows, pkey->pData, false);
QUERY_CHECK_CODE(code, lino, _end);
} else {
colDataSetNULL(pDst, rows);
}
break;
}
case TSDB_FILL_NEXT: {
if (!pSliceInfo->isNextRowSet) {
hasInterp = false;
break;
}
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
if (pkey->isNull == false) {
code = colDataSetVal(pDst, rows, pkey->pData, false);
QUERY_CHECK_CODE(code, lino, _end);
} else {
colDataSetNULL(pDst, rows);
}
break;
}
case TSDB_FILL_NONE:
default:
break;
@ -507,7 +575,7 @@ static int32_t addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSu
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (isIrowtsPseudoColumn(pExprInfo)) {
if (isIrowtsPseudoColumn(pExprInfo) || isIrowtsOriginPseudoColumn(pExprInfo)) {
code = colDataSetVal(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false);
QUERY_CHECK_CODE(code, lino, _end);
} else if (isIsfilledPseudoColumn(pExprInfo)) {
@ -1233,6 +1301,7 @@ int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyN
pInfo->pNextGroupRes = NULL;
pInfo->pRemainRes = NULL;
pInfo->remainIndex = 0;
pInfo->rangeInterval = pInterpPhyNode->rangeInterval;
if (pInfo->hasPk) {
pInfo->prevKey.numOfPKs = 1;

View File

@ -3094,7 +3094,7 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf
qptCtx.result.code = createFillOperatorInfo(NULL, (SFillPhysiNode*)pNode, pTaskInfo, ppOperaotr);
break;
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
qptCtx.result.code = createStreamFillOperatorInfo(NULL, (SStreamFillPhysiNode*)pNode, pTaskInfo, ppOperaotr);
qptCtx.result.code = createStreamFillOperatorInfo(NULL, (SStreamFillPhysiNode*)pNode, pTaskInfo, pReadHandle, ppOperaotr);
break;
case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
qptCtx.result.code = createSessionAggOperatorInfo(NULL, (SSessionWinodwPhysiNode*)pNode, pTaskInfo, ppOperaotr);

View File

@ -5596,6 +5596,20 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{
.name = "_irowts_origin",
.type = FUNCTION_TYPE_IROWTS_ORIGIN,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.parameters = {.minParamNum = 0,
.maxParamNum = 0,
.paramInfoPattern = 0,
.outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_TIMESTAMP_TYPE}},
.translateFunc = translateTimePseudoColumn,
.getEnvFunc = getTimePseudoFuncEnv,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
};
// clang-format on

View File

@ -703,3 +703,10 @@ bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId) {
bool fmIsCountLikeFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_COUNT_LIKE_FUNC);
}
bool fmIsRowTsOriginFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNCTION_TYPE_IROWTS_ORIGIN == funcMgtBuiltins[funcId].type;
}

View File

@ -684,6 +684,8 @@ static int32_t logicInterpFuncCopy(const SInterpFuncLogicNode* pSrc, SInterpFunc
CLONE_NODE_FIELD(pFillValues);
CLONE_NODE_FIELD(pTimeSeries);
COPY_OBJECT_FIELD(streamNodeOption, sizeof(SStreamNodeOption));
COPY_SCALAR_FIELD(rangeInterval);
COPY_SCALAR_FIELD(rangeIntervalUnit);
return TSDB_CODE_SUCCESS;
}

View File

@ -1298,6 +1298,8 @@ static const char* jkInterpFuncLogicPlanFillMode = "fillMode";
static const char* jkInterpFuncLogicPlanFillValues = "FillValues";
static const char* jkInterpFuncLogicPlanTimeSeries = "TimeSeries";
static const char* jkInterpFuncLogicPlanStreamNodeOption = "StreamNodeOption";
static const char* jkInterpFuncLogicPlanRangeInterval = "RangeInterval";
static const char* jkInterpFuncLogicPlanRangeIntervalUnit = "RangeIntervalUnit";
static int32_t logicInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
const SInterpFuncLogicNode* pNode = (const SInterpFuncLogicNode*)pObj;
@ -1333,6 +1335,12 @@ static int32_t logicInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncLogicPlanRangeInterval, pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncLogicPlanRangeIntervalUnit, pNode->rangeIntervalUnit);
}
return code;
}
@ -1371,6 +1379,12 @@ static int32_t jsonToLogicInterpFuncNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkInterpFuncLogicPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncLogicPlanRangeInterval, &pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkInterpFuncLogicPlanRangeIntervalUnit, &pNode->rangeIntervalUnit);
}
return code;
}
@ -3312,6 +3326,8 @@ static const char* jkInterpFuncPhysiPlanFillMode = "FillMode";
static const char* jkInterpFuncPhysiPlanFillValues = "FillValues";
static const char* jkInterpFuncPhysiPlanTimeSeries = "TimeSeries";
static const char* jkInterpFuncPhysiPlanStreamNodeOption = "StreamNodeOption";
static const char* jkInterpFuncPhysiPlanRangeInterval = "RangeInterval";
static const char* jkInterpFuncPhysiPlanRangeIntervalUnit = "RangeIntervalUnit";
static int32_t physiInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
const SInterpFuncPhysiNode* pNode = (const SInterpFuncPhysiNode*)pObj;
@ -3350,6 +3366,12 @@ static int32_t physiInterpFuncNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, streamNodeOptionToJson, &pNode->streamNodeOption);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanRangeInterval, pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkInterpFuncPhysiPlanRangeIntervalUnit, pNode->rangeIntervalUnit);
}
return code;
}
@ -3391,6 +3413,12 @@ static int32_t jsonToPhysiInterpFuncNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkInterpFuncPhysiPlanStreamNodeOption, jsonToStreamNodeOption, &pNode->streamNodeOption);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkInterpFuncPhysiPlanRangeInterval, &pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetTinyIntValue(pJson, jkInterpFuncPhysiPlanRangeIntervalUnit, &pNode->rangeIntervalUnit);
}
return code;
}

View File

@ -3746,7 +3746,9 @@ enum {
PHY_INERP_FUNC_CODE_INTERVAL_UNIT,
PHY_INERP_FUNC_CODE_FILL_MODE,
PHY_INERP_FUNC_CODE_FILL_VALUES,
PHY_INERP_FUNC_CODE_TIME_SERIES
PHY_INERP_FUNC_CODE_TIME_SERIES,
PHY_INTERP_FUNC_CODE_RANGE_INTERVAL,
PHY_INTERP_FUNC_CODE_RANGE_INTERVAL_UNIT,
};
static int32_t physiInterpFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -3777,6 +3779,12 @@ static int32_t physiInterpFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder)
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_TIME_SERIES, nodeToMsg, pNode->pTimeSeries);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_INTERP_FUNC_CODE_RANGE_INTERVAL, pNode->rangeInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI8(pEncoder, PHY_INTERP_FUNC_CODE_RANGE_INTERVAL_UNIT, pNode->rangeIntervalUnit);
}
return code;
}
@ -3815,6 +3823,12 @@ static int32_t msgToPhysiInterpFuncNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_INERP_FUNC_CODE_TIME_SERIES:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTimeSeries);
break;
case PHY_INTERP_FUNC_CODE_RANGE_INTERVAL:
code = tlvDecodeI64(pTlv, &pNode->rangeInterval);
break;
case PHY_INTERP_FUNC_CODE_RANGE_INTERVAL_UNIT:
code = tlvDecodeI8(pTlv, &pNode->rangeIntervalUnit);
break;
default:
break;
}

View File

@ -470,6 +470,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
case QUERY_NODE_SET_OPERATOR:
code = makeNode(type, sizeof(SSetOperator), &pNode);
break;
case QUERY_NODE_RANGE_AROUND:
code = makeNode(type, sizeof(SRangeAroundNode), &pNode); break;
case QUERY_NODE_SELECT_STMT:
code = makeNode(type, sizeof(SSelectStmt), &pNode);
break;
@ -1245,6 +1247,12 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pWin->pEndOffset);
break;
}
case QUERY_NODE_RANGE_AROUND: {
SRangeAroundNode* pAround = (SRangeAroundNode*)pNode;
nodesDestroyNode(pAround->pInterval);
nodesDestroyNode(pAround->pTimepoint);
break;
}
case QUERY_NODE_SET_OPERATOR: {
SSetOperator* pStmt = (SSetOperator*)pNode;
nodesDestroyList(pStmt->pProjectionList);
@ -1266,6 +1274,7 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyList(pStmt->pGroupByList);
nodesDestroyNode(pStmt->pHaving);
nodesDestroyNode(pStmt->pRange);
nodesDestroyNode(pStmt->pRangeAround);
nodesDestroyNode(pStmt->pEvery);
nodesDestroyNode(pStmt->pFill);
nodesDestroyList(pStmt->pOrderByList);

View File

@ -163,6 +163,7 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValue
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
SNode* createInterpTimeAround(SAstCreateContext* pCxt, SNode* pTimepoint, SNode* pInterval);
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
SNode* createAlterSingleTagColumnNode(SAstCreateContext* pCtx, SToken* token, SNode* pVal);

View File

@ -1209,6 +1209,7 @@ pseudo_column(A) ::= QTAGS(B).
pseudo_column(A) ::= FLOW(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= FHIGH(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= FROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= IROWTS_ORIGIN(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
@ -1454,7 +1455,7 @@ jlimit_clause_opt(A) ::= JLIMIT NK_INTEGER(B).
query_specification(A) ::=
SELECT hint_list(M) set_quantifier_opt(B) tag_mode_opt(N) select_list(C) from_clause_opt(D)
where_clause_opt(E) partition_by_clause_opt(F) range_opt(J) every_opt(K)
fill_opt(L) twindow_clause_opt(G) group_by_clause_opt(H) having_clause_opt(I). {
interp_fill_opt(L) twindow_clause_opt(G) group_by_clause_opt(H) having_clause_opt(I). {
A = createSelectStmt(pCxt, B, C, D, M);
A = setSelectStmtTagMode(pCxt, A, N);
A = addWhereClause(pCxt, A, E);
@ -1539,19 +1540,41 @@ interval_sliding_duration_literal(A) ::= NK_VARIABLE(B).
interval_sliding_duration_literal(A) ::= NK_STRING(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
interval_sliding_duration_literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
interp_fill_opt(A) ::= . { A = NULL; }
interp_fill_opt(A) ::= fill_value(B). { A = B; }
interp_fill_opt(A) ::=
FILL NK_LP fill_position_mode_extension(B) NK_COMMA expression_list(C) NK_RP. { A = createFillNode(pCxt, B, createNodeListNode(pCxt, C)); }
interp_fill_opt(A) ::= FILL NK_LP interp_fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); }
fill_opt(A) ::= . { A = NULL; }
fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); }
fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); }
fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); }
fill_opt(A) ::= fill_value(B). { A = B; }
fill_value(A) ::= FILL NK_LP VALUE NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); }
fill_value(A) ::= FILL NK_LP VALUE_F NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); }
%type fill_mode { EFillMode }
%destructor fill_mode { }
fill_mode(A) ::= NONE. { A = FILL_MODE_NONE; }
fill_mode(A) ::= PREV. { A = FILL_MODE_PREV; }
fill_mode(A) ::= NULL. { A = FILL_MODE_NULL; }
fill_mode(A) ::= NULL_F. { A = FILL_MODE_NULL_F; }
fill_mode(A) ::= LINEAR. { A = FILL_MODE_LINEAR; }
fill_mode(A) ::= NEXT. { A = FILL_MODE_NEXT; }
fill_mode(A) ::= fill_position_mode(B). { A = B; }
%type fill_position_mode { EFillMode }
%destructor fill_position_mode { }
fill_position_mode(A) ::= PREV. { A = FILL_MODE_PREV; }
fill_position_mode(A) ::= NEXT. { A = FILL_MODE_NEXT; }
%type fill_position_mode_extension { EFillMode }
%destructor fill_position_mode_extension { }
fill_position_mode_extension(A) ::= fill_position_mode(B). { A = B; }
fill_position_mode_extension(A) ::= NEAR. { A = FILL_MODE_NEAR; }
%type interp_fill_mode { EFillMode }
%destructor interp_fill_mode { }
interp_fill_mode(A) ::= fill_mode(B). { A = B; }
interp_fill_mode(A) ::= NEAR. { A = FILL_MODE_NEAR; }
%type group_by_clause_opt { SNodeList* }
%destructor group_by_clause_opt { nodesDestroyList($$); }

View File

@ -1460,6 +1460,9 @@ _err:
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd) {
CHECK_PARSER_STATUS(pCxt);
if (pEnd && nodeType(pEnd) == QUERY_NODE_VALUE && ((SValueNode*)pEnd)->flag & VALUE_FLAG_IS_DURATION) {
return createInterpTimeAround(pCxt, pStart, pEnd);
}
return createBetweenAnd(pCxt, createPrimaryKeyCol(pCxt, NULL), pStart, pEnd);
_err:
nodesDestroyNode(pStart);
@ -1475,6 +1478,19 @@ _err:
return NULL;
}
SNode* createInterpTimeAround(SAstCreateContext* pCxt, SNode* pTimepoint, SNode* pInterval) {
CHECK_PARSER_STATUS(pCxt);
SRangeAroundNode* pAround = NULL;
pCxt->errCode = nodesMakeNode(QUERY_NODE_RANGE_AROUND, (SNode**)&pAround);
CHECK_PARSER_STATUS(pCxt);
pAround->pTimepoint = createInterpTimePoint(pCxt, pTimepoint);
pAround->pInterval = pInterval;
CHECK_PARSER_STATUS(pCxt);
return (SNode*)pAround;
_err:
return NULL;
}
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen) {
CHECK_PARSER_STATUS(pCxt);
SWhenThenNode* pWhenThen = NULL;
@ -1632,8 +1648,15 @@ _err:
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange) {
CHECK_PARSER_STATUS(pCxt);
SSelectStmt* pSelect = (SSelectStmt*)pStmt;
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
((SSelectStmt*)pStmt)->pRange = pRange;
if (pRange && nodeType(pRange) == QUERY_NODE_RANGE_AROUND) {
pSelect->pRangeAround = pRange;
SRangeAroundNode* pAround = (SRangeAroundNode*)pRange;
TSWAP(pSelect->pRange, pAround->pTimepoint);
} else {
pSelect->pRange = pRange;
}
}
return pStmt;
_err:

View File

@ -176,6 +176,7 @@ static SKeyword keywordTable[] = {
{"NORMAL", TK_NORMAL},
{"NCHAR", TK_NCHAR},
{"NEXT", TK_NEXT},
{"NEAR", TK_NEAR},
{"NMATCH", TK_NMATCH},
{"NONE", TK_NONE},
{"NOT", TK_NOT},
@ -326,6 +327,7 @@ static SKeyword keywordTable[] = {
{"WRITE", TK_WRITE},
{"_C0", TK_ROWTS},
{"_IROWTS", TK_IROWTS},
{"_IROWTS_ORIGIN", TK_IROWTS_ORIGIN},
{"_ISFILLED", TK_ISFILLED},
{"_QDURATION", TK_QDURATION},
{"_QEND", TK_QEND},

View File

@ -1260,7 +1260,8 @@ bool isPrimaryKeyImpl(SNode* pExpr) {
FUNCTION_TYPE_LAST_ROW == pFunc->funcType || FUNCTION_TYPE_TIMETRUNCATE == pFunc->funcType) {
return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0));
} else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType ||
FUNCTION_TYPE_IROWTS == pFunc->funcType || FUNCTION_TYPE_FORECAST_ROWTS == pFunc->funcType) {
FUNCTION_TYPE_IROWTS == pFunc->funcType || FUNCTION_TYPE_IROWTS_ORIGIN == pFunc->funcType ||
FUNCTION_TYPE_FORECAST_ROWTS == pFunc->funcType) {
return true;
}
} else if (QUERY_NODE_OPERATOR == nodeType(pExpr)) {
@ -5388,14 +5389,11 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList
return code;
}
static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) {
if (FILL_MODE_VALUE != pFill->mode && FILL_MODE_VALUE_F != pFill->mode) {
return TSDB_CODE_SUCCESS;
}
static int32_t doCheckFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) {
int32_t fillNo = 0;
SNodeListNode* pFillValues = (SNodeListNode*)pFill->pValues;
SNode* pProject = NULL;
if (!pFillValues) return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch");
FOREACH(pProject, pProjectionList) {
if (needFill(pProject)) {
if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) {
@ -5415,6 +5413,14 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL
return TSDB_CODE_SUCCESS;
}
static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) {
if (FILL_MODE_VALUE != pFill->mode && FILL_MODE_VALUE_F != pFill->mode) {
return TSDB_CODE_SUCCESS;
}
return doCheckFillValues(pCxt, pFill, pProjectionList);
return TSDB_CODE_SUCCESS;
}
static int32_t translateFillValues(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) ||
NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) {
@ -6185,6 +6191,31 @@ static int32_t translateInterpEvery(STranslateContext* pCxt, SNode** pEvery) {
return code;
}
static EDealRes hasRowTsOriginFuncWalkNode(SNode* pNode, void* ctx) {
bool *hasRowTsOriginFunc = ctx;
if (nodeType(pNode) == QUERY_NODE_FUNCTION) {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
if (fmIsRowTsOriginFunc(pFunc->funcId)) {
*hasRowTsOriginFunc = true;
return DEAL_RES_END;
}
}
return DEAL_RES_CONTINUE;
}
static int32_t checkInterpForStream(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (pCxt->createStream) {
SFillNode* pFill = (SFillNode*)pSelect->pFill;
if (pFill->mode == FILL_MODE_NEAR) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "FILL NEAR is not supported by stream");
}
if (pSelect->pRangeAround) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "RANGE with around is not supported by stream");
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect) {
int32_t code = TSDB_CODE_SUCCESS;
@ -6200,13 +6231,60 @@ static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect
if (TSDB_CODE_SUCCESS == code) {
code = checkFill(pCxt, (SFillNode*)pSelect->pFill, (SValueNode*)pSelect->pEvery, true);
}
bool hasRowTsOriginFunc = false;
nodesWalkExprs(pSelect->pProjectionList, hasRowTsOriginFuncWalkNode, &hasRowTsOriginFunc);
if (hasRowTsOriginFunc && pCxt->createStream) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"_irowts_origin is not supported by stream");
}
if (TSDB_CODE_SUCCESS == code) {
code = checkFillValues(pCxt, (SFillNode*)pSelect->pFill, pSelect->pProjectionList);
SFillNode* pFill = (SFillNode*)pSelect->pFill;
if (pSelect->pRangeAround) {
if (pFill->mode != FILL_MODE_PREV && pFill->mode != FILL_MODE_NEXT && pFill->mode != FILL_MODE_NEAR) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE,
"Range with interval can only used with fill PREV/NEXT/NEAR");
}
if (TSDB_CODE_SUCCESS == code)
code = doCheckFillValues(pCxt, pFill, pSelect->pProjectionList);
} else {
if (FILL_MODE_PREV == pFill->mode || FILL_MODE_NEXT == pFill->mode || FILL_MODE_NEAR == pFill->mode) {
if (pFill->pValues) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Can't specify fill values");
}
} else {
if (hasRowTsOriginFunc) return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, "_irowts_origin can only be used with FILL PREV/NEXT/NEAR");
}
code = checkFillValues(pCxt, pFill, pSelect->pProjectionList);
}
}
return code;
}
static int32_t translateInterpAround(STranslateContext* pCxt, SSelectStmt* pSelect) {
int32_t code = 0;
if (pSelect->pRangeAround) {
SRangeAroundNode* pAround = (SRangeAroundNode*)pSelect->pRangeAround;
code = translateExpr(pCxt, &pAround->pInterval);
if (TSDB_CODE_SUCCESS == code) {
if (nodeType(pAround->pInterval) == QUERY_NODE_VALUE) {
SValueNode* pVal = (SValueNode*)pAround->pInterval;
if (pVal->datum.i == 0) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, "Range interval cannot be 0");
}
int8_t unit = pVal->unit;
if (unit == TIME_UNIT_YEAR || unit == TIME_UNIT_MONTH) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE,
"Unsupported time unit in RANGE clause");
}
} else {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, "Invalid range interval");
}
}
}
return code;
}
static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (!pSelect->hasInterpFunc) {
if (NULL != pSelect->pRange || NULL != pSelect->pEvery || NULL != pSelect->pFill) {
@ -6227,6 +6305,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
}
}
int32_t code = 0;
if (pCxt->createStream) {
if (NULL != pSelect->pRange) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
@ -6238,23 +6317,40 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
"Missing EVERY clause or FILL clause");
}
} else {
if (NULL == pSelect->pRange || NULL == pSelect->pEvery || NULL == pSelect->pFill) {
if (pSelect->pRange != NULL && QUERY_NODE_OPERATOR == nodeType(pSelect->pRange) && pSelect->pEvery == NULL) {
// single point interp every can be omitted
} else {
if (!pSelect->pRangeAround) {
if (NULL == pSelect->pRange || NULL == pSelect->pEvery || NULL == pSelect->pFill) {
if (pSelect->pRange != NULL && QUERY_NODE_OPERATOR == nodeType(pSelect->pRange) && pSelect->pEvery == NULL) {
// single point interp every can be omitted
} else {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE,
"Missing RANGE clause, EVERY clause or FILL clause");
}
}
} else {
if (pSelect->pEvery) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE,
"Missing RANGE clause, EVERY clause or FILL clause");
"Range clause with around interval can't be used with EVERY clause");
}
if (!pSelect->pFill) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE,
"Missing FILL clause");
}
}
}
int32_t code = translateExpr(pCxt, &pSelect->pRange);
code = translateExpr(pCxt, &pSelect->pRange);
if (TSDB_CODE_SUCCESS == code) {
code = translateInterpEvery(pCxt, &pSelect->pEvery);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateInterpFill(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = translateInterpAround(pCxt, pSelect);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkInterpForStream(pCxt, pSelect);
}
return code;
}

View File

@ -973,6 +973,16 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
pInterpFunc->precision = pSelect->precision;
}
if (TSDB_CODE_SUCCESS == code && pSelect->pRangeAround) {
SNode* pRangeInterval = ((SRangeAroundNode*)pSelect->pRangeAround)->pInterval;
if (!pRangeInterval || nodeType(pRangeInterval) != QUERY_NODE_VALUE) {
code = TSDB_CODE_PAR_INTERNAL_ERROR;
} else {
pInterpFunc->rangeInterval = ((SValueNode*)pRangeInterval)->datum.i;
pInterpFunc->rangeIntervalUnit = ((SValueNode*)pRangeInterval)->unit;
}
}
// set the output
if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExprs(pInterpFunc->pFuncs, &pInterpFunc->node.pTargets);

View File

@ -1958,6 +1958,8 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh
pInterpFunc->intervalUnit = pFuncLogicNode->intervalUnit;
pInterpFunc->precision = pFuncLogicNode->node.precision;
pInterpFunc->pFillValues = NULL;
pInterpFunc->rangeInterval = pFuncLogicNode->rangeInterval;
pInterpFunc->rangeIntervalUnit = pFuncLogicNode->rangeIntervalUnit;
code = nodesCloneNode(pFuncLogicNode->pFillValues, &pInterpFunc->pFillValues);
if (TSDB_CODE_SUCCESS != code) {
code = code;

View File

@ -4130,7 +4130,7 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const
SStreamStateCur* streamStateFillSeekToLast_rocksdb(SStreamState* pState) {
SWinKey key = {.groupId = UINT64_MAX, .ts = INT64_MAX};
return streamStateFillSeekKeyNext_rocksdb(pState, &key);
return streamStateFillSeekKeyPrev_rocksdb(pState, &key);
}
#ifdef BUILD_NO_CALL

View File

@ -14,6 +14,7 @@
*/
#include "streamInt.h"
#include "ttime.h"
static int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) {
*pSubmit = NULL;
@ -307,13 +308,17 @@ void streamFreeQitem(SStreamQueueItem* data) {
}
}
int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t trigger, SInterval* pInterval, STimeWindow* pLatestWindow, const char* id) {
int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t interval, SInterval* pInterval,
STimeWindow* pLatestWindow, const char* id) {
QRY_PARAM_CHECK(pTrigger);
int64_t ts = INT64_MIN;
SStreamTrigger* p = NULL;
int64_t ts = taosGetTimestamp(pInterval->precision);
int64_t skey = pLatestWindow->skey + interval;
int32_t code = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0, (void**)&p);
if (code) {
stError("s-task:%s failed to create force_window trigger, code:%s", id, tstrerror(code));
return code;
}
@ -324,26 +329,10 @@ int32_t streamCreateForcewindowTrigger(SStreamTrigger** pTrigger, int32_t trigge
return terrno;
}
// let's calculate the previous time window
SInterval interval = {.interval = trigger,
.sliding = trigger,
.intervalUnit = pInterval->intervalUnit,
.slidingUnit = pInterval->slidingUnit};
ts = taosGetTimestampMs();
if (pLatestWindow->skey == INT64_MIN) {
STimeWindow window = getAlignQueryTimeWindow(&interval, ts - trigger);
p->pBlock->info.window.skey = window.skey;
p->pBlock->info.window.ekey = TMAX(ts, window.ekey);
} else {
int64_t skey = pLatestWindow->skey + trigger;
p->pBlock->info.window.skey = skey;
p->pBlock->info.window.ekey = TMAX(ts, skey + trigger);
}
p->pBlock->info.window.skey = skey;
p->pBlock->info.window.ekey = TMAX(ts, skey + interval);
p->pBlock->info.type = STREAM_GET_RESULT;
stDebug("s-task:%s force_window_close trigger block generated, window range:%" PRId64 "-%" PRId64, id,
p->pBlock->info.window.skey, p->pBlock->info.window.ekey);

View File

@ -17,6 +17,9 @@
#include "streamInt.h"
#include "ttimer.h"
#define TRIGGER_RECHECK_INTERVAL (5 * 1000)
#define INITIAL_TRIGGER_INTERVAL (120 * 1000)
static void streamTaskResumeHelper(void* param, void* tmrId);
static void streamTaskSchedHelper(void* param, void* tmrId);
@ -24,7 +27,7 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
int64_t delay = 0;
int32_t code = 0;
const char* id = pTask->id.idStr;
int64_t* pTaskRefId = NULL;
int64_t* pTaskRefId = NULL;
if (pTask->info.fillHistory == 1) {
return;
@ -32,8 +35,8 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
// dynamic set the trigger & triggerParam for STREAM_TRIGGER_FORCE_WINDOW_CLOSE
if ((pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) && (pTask->info.taskLevel == TASK_LEVEL__SOURCE)) {
int64_t waterMark = 0;
SInterval interval = {0};
int64_t waterMark = 0;
SInterval interval = {0};
STimeWindow lastTimeWindow = {0};
code = qGetStreamIntervalExecInfo(pTask->exec.pExecutor, &waterMark, &interval, &lastTimeWindow);
if (code) {
@ -41,19 +44,37 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
return;
}
pTask->status.latestForceWindow = lastTimeWindow;
pTask->info.delaySchedParam = interval.sliding;
pTask->info.watermark = waterMark;
pTask->info.interval = interval;
// calculate the first start timestamp
int64_t now = taosGetTimestamp(interval.precision);
int64_t now = taosGetTimestamp(interval.precision);
STimeWindow curWin = getAlignQueryTimeWindow(&pTask->info.interval, now);
delay = (curWin.ekey + 1) - now + waterMark;
if (lastTimeWindow.skey == INT64_MIN) { // start from now, not the exec task timestamp after delay
pTask->status.latestForceWindow.skey = curWin.skey - pTask->info.interval.interval;
pTask->status.latestForceWindow.ekey = now;
delay = (curWin.ekey + 1) - now + waterMark;
delay = convertTimePrecision(delay, interval.precision, TSDB_TIME_PRECISION_MILLI);
} else {
pTask->status.latestForceWindow = lastTimeWindow;
// It's the current calculated time window
int64_t calEkey = lastTimeWindow.skey + pTask->info.interval.interval * 2;
if (calEkey + waterMark < now) { // unfinished time window existed
delay = INITIAL_TRIGGER_INTERVAL; // wait for 2min to start to calculate
} else {
delay = (curWin.ekey + 1) - now + waterMark;
delay = convertTimePrecision(delay, interval.precision, TSDB_TIME_PRECISION_MILLI);
}
}
stInfo("s-task:%s extract interval info from executor, wm:%" PRId64 " interval:%" PRId64 " unit:%c sliding:%" PRId64
" unit:%c, initial start after:%" PRId64,
id, waterMark, interval.interval, interval.intervalUnit, interval.sliding, interval.slidingUnit, delay);
" unit:%c, initial start after:%" PRId64 "ms last_win:%" PRId64 "-%" PRId64,
id, waterMark, interval.interval, interval.intervalUnit, interval.sliding, interval.slidingUnit, delay,
pTask->status.latestForceWindow.skey, pTask->status.latestForceWindow.ekey);
} else {
delay = pTask->info.delaySchedParam;
if (delay == 0) {
@ -66,8 +87,8 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) {
stDebug("s-task:%s refId:%" PRId64 " enable the scheduler trigger, delay:%" PRId64, pTask->id.idStr,
pTask->id.refId, delay);
streamTmrStart(streamTaskSchedHelper, (int32_t)delay, pTaskRefId, streamTimer,
&pTask->schedInfo.pDelayTimer, pTask->pMeta->vgId, "sched-tmr");
streamTmrStart(streamTaskSchedHelper, (int32_t)delay, pTaskRefId, streamTimer, &pTask->schedInfo.pDelayTimer,
pTask->pMeta->vgId, "sched-tmr");
pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE;
}
}
@ -184,7 +205,62 @@ void streamTaskResumeHelper(void* param, void* tmrId) {
streamTaskFreeRefId(param);
}
static int32_t doCreateForceWindowTrigger(SStreamTask* pTask, int32_t* pNextTrigger) {
int32_t num = 0;
int32_t code = 0;
const char* id = pTask->id.idStr;
int8_t precision = pTask->info.interval.precision;
SStreamTrigger* pTrigger = NULL;
while (1) {
code = streamCreateForcewindowTrigger(&pTrigger, pTask->info.delaySchedParam, &pTask->info.interval,
&pTask->status.latestForceWindow, id);
if (code != 0) {
*pNextTrigger = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI);
stError("s-task:%s failed to prepare force window close trigger, code:%s, try again in %dms", id,
tstrerror(code), *pNextTrigger);
return code;
}
// in the force window close model, status trigger does not matter. So we do not set the trigger model
code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to put retrieve aggRes block into q, code:%s", pTask->id.idStr, tstrerror(code));
return code;
}
num += 1;
// check whether the time window gaps exist or not
int64_t now = taosGetTimestamp(precision);
int64_t ekey = pTrigger->pBlock->info.window.skey + pTask->info.interval.interval;
// there are gaps, needs to be filled
STimeWindow w = pTrigger->pBlock->info.window;
w.ekey = w.skey + pTask->info.interval.interval;
if (w.skey <= pTask->status.latestForceWindow.skey) {
stFatal("s-task:%s invalid new time window in force_window_close trigger model, skey:%" PRId64
" should be greater than latestForceWindow skey:%" PRId64,
pTask->id.idStr, w.skey, pTask->status.latestForceWindow.skey);
}
pTask->status.latestForceWindow = w;
if (ekey + pTask->info.watermark + pTask->info.interval.interval > now) {
int64_t prev = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI);
*pNextTrigger = ekey + pTask->info.watermark + pTask->info.interval.interval - now;
*pNextTrigger = convertTimePrecision(*pNextTrigger, precision, TSDB_TIME_PRECISION_MILLI);
stDebug("s-task:%s generate %d time window(s), trigger delay adjust from %" PRId64 " to %d", id, num, prev,
*pNextTrigger);
return code;
} else {
stDebug("s-task:%s gap exist for force_window_close, current force_window_skey:%" PRId64, id, w.skey);
}
}
}
void streamTaskSchedHelper(void* param, void* tmrId) {
int32_t code = 0;
int64_t taskRefId = *(int64_t*)param;
SStreamTask* pTask = taosAcquireRef(streamTaskRefPool, taskRefId);
if (pTask == NULL) {
@ -193,15 +269,20 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
return;
}
stDebug("s-task:%s acquire task, refId:%"PRId64, pTask->id.idStr, pTask->id.refId);
stDebug("s-task:%s acquire task, refId:%" PRId64, pTask->id.idStr, pTask->id.refId);
const char* id = pTask->id.idStr;
int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam;
int32_t vgId = pTask->pMeta->vgId;
int32_t code = 0;
const char* id = pTask->id.idStr;
int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam;
int32_t vgId = pTask->pMeta->vgId;
int8_t status = atomic_load_8(&pTask->schedInfo.status);
stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
int32_t next = convertTimePrecision(nextTrigger, pTask->info.interval.precision, TSDB_TIME_PRECISION_MILLI);
stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, next);
} else {
stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger);
}
if (streamTaskShouldStop(pTask)) {
stDebug("s-task:%s should stop, jump out of schedTimer", id);
@ -211,62 +292,30 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
}
if (streamTaskShouldPause(pTask)) {
stDebug("s-task:%s is paused, recheck in %.2fs", id, nextTrigger/1000.0);
streamTmrStart(streamTaskSchedHelper, nextTrigger, param, streamTimer, &pTask->schedInfo.pDelayTimer, vgId,
"sched-run-tmr");
stDebug("s-task:%s is paused, recheck in %.2fs", id, TRIGGER_RECHECK_INTERVAL / 1000.0);
streamTmrStart(streamTaskSchedHelper, TRIGGER_RECHECK_INTERVAL, param, streamTimer, &pTask->schedInfo.pDelayTimer,
vgId, "sched-run-tmr");
streamMetaReleaseTask(pTask->pMeta, pTask);
return;
}
if (streamTaskShouldPause(pTask)) {
stDebug("s-task:%s is paused, check in nextTrigger:%ds", id, nextTrigger/1000);
streamTmrStart(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId,
"sched-run-tmr");
if (pTask->status.downstreamReady == 0) {
stDebug("s-task:%s downstream not ready, recheck in %.2fs", id, TRIGGER_RECHECK_INTERVAL / 1000.0);
streamTmrStart(streamTaskSchedHelper, TRIGGER_RECHECK_INTERVAL, param, streamTimer, &pTask->schedInfo.pDelayTimer,
vgId, "sched-run-tmr");
streamMetaReleaseTask(pTask->pMeta, pTask);
return;
}
if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) {
stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, nextTrigger);
nextTrigger = TRIGGER_RECHECK_INTERVAL; // retry in 10 seec
stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, TRIGGER_RECHECK_INTERVAL);
} else {
if (pTask->info.trigger == STREAM_TRIGGER_FORCE_WINDOW_CLOSE && pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
SStreamTrigger* pTrigger = NULL;
while (1) {
code = streamCreateForcewindowTrigger(&pTrigger, pTask->info.delaySchedParam, &pTask->info.interval,
&pTask->status.latestForceWindow, id);
if (code != 0) {
stError("s-task:%s failed to prepare force window close trigger, code:%s, try again in %dms", id,
tstrerror(code), nextTrigger);
goto _end;
}
// in the force window close model, status trigger does not matter. So we do not set the trigger model
code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to put retrieve aggRes block into q, code:%s", pTask->id.idStr, tstrerror(code));
goto _end;
}
// check whether the time window gaps exist or not
int64_t now = taosGetTimestamp(pTask->info.interval.precision);
int64_t intervalEndTs = pTrigger->pBlock->info.window.skey + pTask->info.interval.interval;
// there are gaps, needs to be filled
STimeWindow w = pTrigger->pBlock->info.window;
w.ekey = w.skey + pTask->info.interval.interval;
if (w.skey <= pTask->status.latestForceWindow.skey) {
stFatal("s-task:%s invalid new time window in force_window_close model, skey:%" PRId64
" should be greater than latestForceWindow skey:%" PRId64,
pTask->id.idStr, w.skey, pTask->status.latestForceWindow.skey);
}
pTask->status.latestForceWindow = w;
if (intervalEndTs + pTask->info.watermark + pTask->info.interval.interval > now) {
break;
} else {
stDebug("s-task:%s gap exist for force_window_close, current force_window_skey:%" PRId64, id, w.skey);
}
code = doCreateForceWindowTrigger(pTask, &nextTrigger);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
} else if (status == TASK_TRIGGER_STATUS__MAY_ACTIVE) {
SStreamTrigger* pTrigger = NULL;
code = streamCreateSinkResTrigger(&pTrigger);

View File

@ -23,6 +23,33 @@
#define NUM_OF_CACHE_WIN 64
#define MAX_NUM_OF_CACHE_WIN 128
int32_t recoverSearchBuff(SStreamFileState* pFileState, SArray* pWinStates, uint64_t groupId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SWinKey start = {.groupId = groupId, .ts = INT64_MAX};
void* pState = getStateFileStore(pFileState);
SStreamStateCur* pCur = streamStateFillSeekKeyPrev_rocksdb(pState, &start);
for (int32_t i = 0; i < NUM_OF_CACHE_WIN; i++) {
SWinKey tmpKey = {.groupId = groupId};
int32_t tmpRes = streamStateFillGetGroupKVByCur_rocksdb(pCur, &tmpKey, NULL, 0);
if (tmpRes != TSDB_CODE_SUCCESS) {
break;
}
void* tmp = taosArrayPush(pWinStates, &tmpKey);
QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
streamStateCurPrev_rocksdb(pCur);
}
taosArraySort(pWinStates, winKeyCmprImpl);
streamStateFreeCur(pCur);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t getHashSortRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
@ -38,22 +65,8 @@ int32_t getHashSortRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, vo
// recover
if (taosArrayGetSize(pWinStates) == 0 && needClearDiskBuff(pFileState)) {
TSKEY ts = getFlushMark(pFileState);
SWinKey start = {.groupId = pKey->groupId, .ts = INT64_MAX};
void* pState = getStateFileStore(pFileState);
SStreamStateCur* pCur = streamStateFillSeekKeyPrev_rocksdb(pState, &start);
for (int32_t i = 0; i < NUM_OF_CACHE_WIN; i++) {
SWinKey tmpKey = {.groupId = pKey->groupId};
int32_t tmpRes = streamStateFillGetGroupKVByCur_rocksdb(pCur, &tmpKey, NULL, 0);
if (tmpRes != TSDB_CODE_SUCCESS) {
break;
}
void* tmp = taosArrayPush(pWinStates, &tmpKey);
QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
streamStateCurPrev_rocksdb(pCur);
}
taosArraySort(pWinStates, winKeyCmprImpl);
streamStateFreeCur(pCur);
code = recoverSearchBuff(pFileState, pWinStates, pKey->groupId);
QUERY_CHECK_CODE(code, lino, _end);
}
code = addSearchItem(pFileState, pWinStates, pKey);
@ -203,29 +216,17 @@ int32_t getHashSortPrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SW
SArray* pWinStates = NULL;
SSHashObj* pSearchBuff = getSearchBuff(pFileState);
void* pState = getStateFileStore(pFileState);
void** ppBuff = (void**) tSimpleHashGet(pSearchBuff, &pKey->groupId, sizeof(uint64_t));
if (ppBuff) {
pWinStates = (SArray*)(*ppBuff);
} else {
qDebug("===stream=== search buff is empty.group id:%" PRId64, pKey->groupId);
SStreamStateCur* pCur = streamStateFillSeekKeyPrev_rocksdb(pState, pKey);
void* tmpVal = NULL;
int32_t len = 0;
(*pWinCode) = streamStateFillGetGroupKVByCur_rocksdb(pCur, pResKey, (const void**)&tmpVal, &len);
if ((*pWinCode) == TSDB_CODE_SUCCESS) {
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
if (!pNewPos || !pNewPos->pRowBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
QUERY_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pRowBuff, tmpVal, len);
taosMemoryFreeClear(tmpVal);
*pVLen = getRowStateRowSize(pFileState);
(*ppVal) = pNewPos;
}
streamStateFreeCur(pCur);
return code;
// void** ppBuff = (void**) tSimpleHashGet(pSearchBuff, &pKey->groupId, sizeof(uint64_t));
code = addArrayBuffIfNotExist(pSearchBuff, pKey->groupId, &pWinStates);
QUERY_CHECK_CODE(code, lino, _end);
// recover
if (taosArrayGetSize(pWinStates) == 0 && needClearDiskBuff(pFileState)) {
code = recoverSearchBuff(pFileState, pWinStates, pKey->groupId);
QUERY_CHECK_CODE(code, lino, _end);
}
int32_t size = taosArrayGetSize(pWinStates);
int32_t index = binarySearch(pWinStates, size, pKey, fillStateKeyCompare);
if (index >= 0) {

View File

@ -259,7 +259,7 @@ int32_t streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize,
if (type == STREAM_STATE_BUFF_HASH || type == STREAM_STATE_BUFF_HASH_SEARCH) {
code = recoverSnapshot(pFileState, checkpointId);
} else if (type == STREAM_STATE_BUFF_SORT) {
code = recoverSesssion(pFileState, checkpointId);
code = recoverSession(pFileState, checkpointId);
} else if (type == STREAM_STATE_BUFF_HASH_SORT) {
code = recoverFillSnapshot(pFileState, checkpointId);
}
@ -914,7 +914,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
return code;
}
int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) {
int32_t recoverSession(SStreamFileState* pFileState, int64_t ckId) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t winRes = TSDB_CODE_SUCCESS;
@ -991,6 +991,7 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) {
}
winCode = streamStateGetKVByCur_rocksdb(getStateFileStore(pFileState), pCur, pNewPos->pKey, (const void**)&pVal, &vlen);
qDebug("===stream=== get state by cur winres:%d. %s", winCode, __func__);
if (winCode != TSDB_CODE_SUCCESS || pFileState->getTs(pNewPos->pKey) < pFileState->flushMark) {
destroyRowBuffPos(pNewPos);
SListNode* pNode = tdListPopTail(pFileState->usedBuffs);
@ -1007,6 +1008,7 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) {
memcpy(pNewPos->pRowBuff, pVal, vlen);
taosMemoryFreeClear(pVal);
pNewPos->beFlushed = true;
qDebug("===stream=== read checkpoint state from disc. %s", __func__);
code = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES);
if (code != TSDB_CODE_SUCCESS) {
destroyRowBuffPos(pNewPos);
@ -1077,6 +1079,7 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) {
int32_t vlen = 0;
SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState);
winRes = streamStateFillGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen);
qDebug("===stream=== get state by cur winres:%d. %s", winRes, __func__);
if (winRes != TSDB_CODE_SUCCESS || isFlushedState(pFileState, pFileState->getTs(pNewPos->pKey), 0)) {
destroyRowBuffPos(pNewPos);
SListNode* pNode = tdListPopTail(pFileState->usedBuffs);
@ -1085,9 +1088,17 @@ int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId) {
break;
}
if (vlen != pFileState->rowSize) {
qError("row size mismatch, expect:%d, actual:%d", pFileState->rowSize, vlen);
code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
taosMemoryFreeClear(pVal);
QUERY_CHECK_CODE(code, lino, _end);
}
memcpy(pNewPos->pRowBuff, pVal, vlen);
taosMemoryFreeClear(pVal);
pNewPos->beFlushed = true;
qDebug("===stream=== read checkpoint state from disc. %s", __func__);
winRes = tSimpleHashPut(pFileState->rowStateBuff, pNewPos->pKey, pFileState->keyLen, &pNewPos, POINTER_BYTES);
if (winRes != TSDB_CODE_SUCCESS) {
destroyRowBuffPos(pNewPos);
@ -1232,11 +1243,6 @@ void clearExpiredState(SStreamFileState* pFileState) {
int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey);
qTrace("clear expired file, ts:%" PRId64 ". %s at line %d res:%d", pKey->ts, __func__, __LINE__, code_file);
}
if (pFileState->hasFillCatch == false) {
int32_t code_file = streamStateFillDel_rocksdb(pFileState->pFileStore, pKey);
qTrace("force clear expired file, ts:%" PRId64 ". %s at line %d res %d", pKey->ts, __func__, __LINE__, code_file);
}
}
taosArrayRemoveBatch(pWinStates, 0, size - 1, NULL);
}

View File

@ -159,6 +159,11 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsma2.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery2.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp_extension.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py

View File

@ -0,0 +1,19 @@
#Column Define
#caseID,rerunTimes,Run with Sanitizer,casePath,caseCommand
#NA,NA,y or n,script,./test.sh -f tsim/user/basic.sim
# system test
#
#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_multi_agg.py
#,,n,system-test,python3 ./test.py -f 8-stream/stream_basic.py
# army-test
#,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2
#tsim test
#,,y,script,./test.sh -f tsim/query/timeline.sim
#docs-examples test
#,,n,docs-examples-test,bash c.sh

View File

@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print =============== create database
print ========================================== create database
sql create database test vgroups 2;
sql select * from information_schema.ins_databases
if $rows != 3 then
@ -18,7 +18,47 @@ sql create stable st(ts timestamp, a int) tags(t int);
sql create table tu1 using st tags(1);
sql create stream stream1 trigger force_window_close into str_dst as select _wstart, count(*) from st partition by tbname interval(5s);
run tsim/stream/checkTaskStatus.sim
sql insert into tu1 values(now, 1);
sleep 5500
sql pause stream stream1
$loop_count = 0
loop1:
sleep 500
$loop_count = $loop_count + 1
if $loop_count == 20 then
goto end_loop1
endi
sql insert into tu1 values(now, 1);
goto loop1
end_loop1:
sql resume stream stream1
sleep 5000
sql select sum(`count(*)`) from (select * from str_dst)
if $data00 != 20 then
print expect 20, actual: $data00
return -1
endi
sql drop database test
print ===================================== micro precision db test
print ============ create db
sql create database test vgroups 2 precision 'us';
sql use test
sql create stable st(ts timestamp, a int) tags(t int);
sql create table tu1 using st tags(1);
sql create stream stream1 trigger force_window_close into str_dst as select _wstart, count(*) from st partition by tbname interval(5s);
run tsim/stream/checkTaskStatus.sim
sql insert into tu1 values(now, 1);
@ -41,10 +81,58 @@ goto loop0
end_loop:
sql resume stream stream1
sql select * from str_dst
sleep 5000
if $rows != 3 then
print expect 3, actual: $rows
sql select sum(`count(*)`) from (select * from str_dst)
if $data00 != 20 then
print expect 20, actual: $data00
return -1
endi
sql drop stream stream1
sql drop table str_dst
print ============================= too long watermark test
sql drop table tu1;
sql create table tu1 using st tags(1);
sql create stream stream2 trigger force_window_close watermark 30s into str_dst as select _wstart, count(*), now() from st partition by tbname interval(5s);
run tsim/stream/checkTaskStatus.sim
$loop_count = 0
loop2:
sleep 500
$loop_count = $loop_count + 1
if $loop_count == 20 then
goto end_loop3
endi
sql insert into tu1 values(now, 1);
goto loop2
end_loop3:
sql select count(*) from str_dst
print =================rows: $data00
if $data00 != 0 then
print expect 0, actual $data00
return -1
endi
sleep 35000
sql select sum(`count(*)`) from (select * from str_dst)
if $data00 != 19 then
print expect 19, actual: $data00
return -1
endi
sql select round(timediff(`now()`, `_wstart`)/1000000) from str_dst;
if $data00 != 35.000000000 then
print expect 35.000000000 , actual $data00
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -1,67 +0,0 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c checkpointInterval -v 60
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 4;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int)tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams1 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt1 as select _wstart, count(a) from st partition by tbname interval(2s);
sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(a) from st interval(2s);
run tsim/stream/checkTaskStatus.sim
sleep 70000
print restart taosd 01 ......
system sh/stop_dnodes.sh
system sh/exec.sh -n dnode1 -s start
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(now + 3000a,1,1,1);
$loop_count = 0
loop0:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print select * from streamt1;
sql select * from streamt1;
print $data00 $data01 $data02
if $rows == 0 then
goto loop0
endi
print select * from streamt2;
sql select * from streamt2;
print $data00 $data01 $data02
if $rows == 0 then
goto loop0
endi
print end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -304,6 +304,253 @@ if $rows != 1 then
return -1
endi
print step3
print =============== create database
sql create database test4 vgroups 4;
sql use test4;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1234567890t1 using st tags(1,1,1);
sql create table t1234567890t2 using st tags(2,2,2);
sql create stable streamt9(ts timestamp,a varchar(10),b tinyint,c tinyint) tags(ta varchar(3),cc int,tc int);
sql create stable streamt10(ts timestamp,a varchar(10),b tinyint,c tinyint) tags(ta varchar(3),cc int,tc int);
sql create stable streamt11(ts timestamp,a varchar(10),b tinyint,c tinyint) tags(ta varchar(3),cc int,tc int);
sql create stream streams9 trigger FORCE_WINDOW_CLOSE IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt9 TAGS(cc,ta) SUBTABLE(concat(concat("tbn-", tbname), "_1")) as select _irowts, interp(a), _isfilled as a1, interp(b) from st partition by tbname as ta, b as cc every(2s) fill(value, 100000,200000);
sql create stream streams10 trigger FORCE_WINDOW_CLOSE IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt10 TAGS(cc,ta) SUBTABLE(concat(concat("tbn-", tbname), "_2")) as select _wstart, twa(a), sum(b),max(c) from st partition by tbname as ta, b as cc interval(2s) fill(NULL);
sql create stream streams11 trigger FORCE_WINDOW_CLOSE IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt11 TAGS(cc,ta) SUBTABLE(concat(concat("tbn-", tbname), "_3")) as select _wstart, count(a),avg(c),min(b) from st partition by tbname as ta, b as cc interval(2s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1234567890t1 values(now + 3s,100000,1,1);
$loop_count = 0
loop9:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print 2 sql select cc,ta, * from streamt9;
sql select cc,ta, * from streamt9;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
# row 0
if $rows < 2 then
print ======rows=$rows
goto loop9
endi
if $data00 != 1 then
return -1
endi
if $data01 != @t12@ then
return -1
endi
if $data03 != @100000@ then
return -1
endi
if $data04 != 1 then
return -1
endi
if $data05 != 64 then
return -1
endi
print 3 sql select * from information_schema.ins_tables where stable_name = "streamt9";
sql select * from information_schema.ins_tables where stable_name = "streamt9";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
print 4 sql select * from information_schema.ins_tables where stable_name = "streamt9" and table_name like "tbn-t1234567890t1_1%";
sql select * from information_schema.ins_tables where stable_name = "streamt9" and table_name like "tbn-t1234567890t1_1%";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
$loop_count = 0
loop10:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print 2 sql select cc,ta, * from streamt10;
sql select cc,ta, * from streamt10;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
# row 0
if $rows < 2 then
print ======rows=$rows
goto loop10
endi
if $data00 != 1 then
return -1
endi
if $data01 != @t12@ then
return -1
endi
if $data03 != @100000.000@ then
return -1
endi
if $data04 != 1 then
return -1
endi
if $data05 != 1 then
return -1
endi
print 3 sql select * from information_schema.ins_tables where stable_name = "streamt10";
sql select * from information_schema.ins_tables where stable_name = "streamt10";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
print 4 sql select * from information_schema.ins_tables where stable_name = "streamt10" and table_name like "tbn-t1234567890t1_2%";
sql select * from information_schema.ins_tables where stable_name = "streamt10" and table_name like "tbn-t1234567890t1_2%";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
$loop_count = 0
loop11:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print 2 sql select cc,ta,* from streamt11;
sql select cc,ta,* from streamt11;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
# row 0
if $rows < 1 then
print ======rows=$rows
goto loop11
endi
if $data00 != 1 then
return -1
endi
if $data01 != @t12@ then
return -1
endi
if $data03 != @1@ then
return -1
endi
if $data04 != 1 then
return -1
endi
if $data05 != 1 then
return -1
endi
print 3 sql select * from information_schema.ins_tables where stable_name = "streamt11";
sql select * from information_schema.ins_tables where stable_name = "streamt11";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
print 4 sql select * from information_schema.ins_tables where stable_name = "streamt11" and table_name like "tbn-t1234567890t1_3%";
sql select * from information_schema.ins_tables where stable_name = "streamt11" and table_name like "tbn-t1234567890t1_3%";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows != 1 then
return -1
endi
print end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -0,0 +1,180 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c checkpointInterval -v 60
system sh/cfg.sh -n dnode1 -c ratioOfVnodeStreamThreads -v 4
system sh/exec.sh -n dnode1 -s start
sleep 50
sql connect
print step1
print =============== create database
sql create database test vgroups 1;
sql use test;
sql create stable st(ts timestamp, a int, b int , c int)tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams1 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt1 as select _wstart, count(a),max(b) from st partition by tbname interval(5s);
sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(a), max(b) from st interval(5s);
sql create stream streams3 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt3 as select _wstart, count(a), twa(b) from st partition by tbname interval(5s) fill(prev);
sql create stream streams4 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 1 into streamt4 as select _irowts, interp(a), interp(b) from st partition by tbname every(5s) fill(prev);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(now + 3000a,1,1,1);
$loop_count = 0
loop0:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print select * from streamt3;
sql select * from streamt3;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows == 0 then
goto loop0
endi
print select * from streamt4;
sql select * from streamt4;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows == 0 then
goto loop0
endi
sleep 70000
$loop_count = 0
loop0_1:
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print sql select * from information_schema.ins_stream_tasks where checkpoint_time is null;
sql select * from information_schema.ins_stream_tasks where checkpoint_time is null;
sleep 10000
if $rows > 0 then
print wait checkpoint.rows = $rows
goto loop0_1
endi
print restart taosd 01 ......
system sh/stop_dnodes.sh
system sh/exec.sh -n dnode1 -s start
run tsim/stream/checkTaskStatus.sim
print select * from streamt3;
sql select * from streamt3;
$streamt3_rows = $rows
print =====streamt3_rows=$streamt3_rows
print select * from streamt4;
sql select * from streamt4;
$streamt4_rows = $rows
print =====streamt4_rows=$streamt4_rows
$loop_count = 0
loop1:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print select * from streamt3;
sql select * from streamt3;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows <= $streamt3_rows then
print =====rows=$rows
print =====streamt3_rows=$streamt3_rows
goto loop1
endi
print select * from streamt4;
sql select * from streamt4;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $rows <= $streamt4_rows then
print =====rows=$rows
print =====streamt4_rows=$streamt4_rows
goto loop1
endi
sql insert into t1 values(now + 3000a,10,10,10);
$loop_count = 0
loop2:
sleep 2000
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
print select * from streamt1 order by 1;
sql select * from streamt1 order by 1;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
if $data12 != 10 then
goto loop2
endi
print end
system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -0,0 +1,522 @@
import queue
from random import randrange
import time
import threading
import secrets
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
from datetime import timezone
from tzlocal import get_localzone
# from tmqCommon import *
ROUND: int = 500
class TDTestCase:
updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'debugFlag': 143}
check_failed: bool = False
def __init__(self):
self.vgroups = 4
self.ctbNum = 10
self.rowsPerTbl = 10000
self.duraion = '1h'
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), False)
def create_database(self, tsql, dbName, dropFlag=1, vgroups=2, replica=1, duration: str = '1d'):
if dropFlag == 1:
tsql.execute("drop database if exists %s" % (dbName))
tsql.execute("create database if not exists %s vgroups %d replica %d duration %s" % (
dbName, vgroups, replica, duration))
tdLog.debug("complete to create database %s" % (dbName))
return
def create_stable(self, tsql, paraDict):
colString = tdCom.gen_column_type_str(
colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"])
tagString = tdCom.gen_tag_type_str(
tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"])
sqlString = f"create table if not exists %s.%s (%s) tags (%s)" % (
paraDict["dbName"], paraDict["stbName"], colString, tagString)
tdLog.debug("%s" % (sqlString))
tsql.execute(sqlString)
return
def create_ctable(self, tsql=None, dbName='dbx', stbName='stb', ctbPrefix='ctb', ctbNum=1, ctbStartIdx=0):
for i in range(ctbNum):
sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % (dbName, ctbPrefix, i+ctbStartIdx, dbName, stbName, (i+ctbStartIdx) % 5, i+ctbStartIdx + random.randint(
1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100), i+ctbStartIdx + random.randint(1, 100))
tsql.execute(sqlString)
tdLog.debug("complete to create %d child tables by %s.%s" %
(ctbNum, dbName, stbName))
return
def init_normal_tb(self, tsql, db_name: str, tb_name: str, rows: int, start_ts: int, ts_step: int):
sql = 'CREATE TABLE %s.%s (ts timestamp, c1 INT, c2 INT, c3 INT, c4 double, c5 VARCHAR(255))' % (
db_name, tb_name)
tsql.execute(sql)
sql = 'INSERT INTO %s.%s values' % (db_name, tb_name)
for j in range(rows):
sql += f'(%d, %d,%d,%d,{random.random()},"varchar_%d"),' % (start_ts + j * ts_step + randrange(500), j %
10 + randrange(200), j % 10, j % 10, j % 10 + randrange(100))
tsql.execute(sql)
def insert_data(self, tsql, dbName, ctbPrefix, ctbNum, rowsPerTbl, batchNum, startTs, tsStep):
tdLog.debug("start to insert data ............")
tsql.execute("use %s" % dbName)
pre_insert = "insert into "
sql = pre_insert
for i in range(ctbNum):
rowsBatched = 0
sql += " %s.%s%d values " % (dbName, ctbPrefix, i)
for j in range(rowsPerTbl):
if (i < ctbNum/2):
sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') " % (startTs + j*tsStep + randrange(
500), j % 10 + randrange(100), j % 10 + randrange(200), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
else:
sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') " % (
startTs + j*tsStep + randrange(500), j % 10, j % 10, j % 10, j % 10, j % 10, j % 10)
rowsBatched += 1
if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
tsql.execute(sql)
rowsBatched = 0
if j < rowsPerTbl - 1:
sql = "insert into %s.%s%d values " % (dbName, ctbPrefix, i)
else:
sql = "insert into "
if sql != pre_insert:
tsql.execute(sql)
tdLog.debug("insert data ............ [OK]")
return
def init_data(self, db: str = 'test', ctb_num: int = 10, rows_per_ctb: int = 10000, start_ts: int = 1537146000000, ts_step: int = 500):
tdLog.printNoPrefix(
"======== prepare test env include database, stable, ctables, and insert data: ")
paraDict = {'dbName': db,
'dropFlag': 1,
'vgroups': 4,
'stbName': 'meters',
'colPrefix': 'c',
'tagPrefix': 't',
'colSchema': [{'type': 'INT', 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'FLOAT', 'count': 1}, {'type': 'DOUBLE', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'tinyint', 'count': 1}, {'type': 'bool', 'count': 1}, {'type': 'binary', 'len': 10, 'count': 1}, {'type': 'nchar', 'len': 10, 'count': 1}],
'tagSchema': [{'type': 'INT', 'count': 1}, {'type': 'nchar', 'len': 20, 'count': 1}, {'type': 'binary', 'len': 20, 'count': 1}, {'type': 'BIGINT', 'count': 1}, {'type': 'smallint', 'count': 1}, {'type': 'DOUBLE', 'count': 1}],
'ctbPrefix': 't',
'ctbStartIdx': 0,
'ctbNum': ctb_num,
'rowsPerTbl': rows_per_ctb,
'batchNum': 3000,
'startTs': start_ts,
'tsStep': ts_step}
paraDict['vgroups'] = self.vgroups
paraDict['ctbNum'] = ctb_num
paraDict['rowsPerTbl'] = rows_per_ctb
tdLog.info("create database")
self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"],
vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion)
tdLog.info("create stb")
self.create_stable(tsql=tdSql, paraDict=paraDict)
tdLog.info("create child tables")
self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"],
stbName=paraDict["stbName"], ctbPrefix=paraDict["ctbPrefix"],
ctbNum=paraDict["ctbNum"], ctbStartIdx=paraDict["ctbStartIdx"])
self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],
ctbPrefix=paraDict["ctbPrefix"], ctbNum=paraDict["ctbNum"],
rowsPerTbl=paraDict["rowsPerTbl"], batchNum=paraDict["batchNum"],
startTs=paraDict["startTs"], tsStep=paraDict["tsStep"])
self.init_normal_tb(tdSql, paraDict['dbName'], 'norm_tb',
paraDict['rowsPerTbl'], paraDict['startTs'], paraDict['tsStep'])
def run(self):
self.init_data()
self.test_interp_extension()
def datetime_add_tz(self, dt):
if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:
return dt.replace(tzinfo=get_localzone())
return dt
def binary_search_ts(self, select_results, ts):
mid = 0
try:
found: bool = False
start = 0
end = len(select_results) - 1
while start <= end:
mid = (start + end) // 2
if self.datetime_add_tz(select_results[mid][0]) == ts:
found = True
return mid
elif self.datetime_add_tz(select_results[mid][0]) < ts:
start = mid + 1
else:
end = mid - 1
if not found:
tdLog.exit(f"cannot find ts in select results {ts} {select_results}")
return start
except Exception as e:
tdLog.debug(f"{select_results[mid][0]}, {ts}, {len(select_results)}, {select_results[mid]}")
self.check_failed = True
tdLog.exit(f"binary_search_ts error: {e}")
def distance(self, ts1, ts2):
return abs(self.datetime_add_tz(ts1) - self.datetime_add_tz(ts2))
## TODO pass last position to avoid search from the beginning
def is_nearest(self, select_results, irowts_origin, irowts):
if len(select_results) <= 1:
return True
try:
#tdLog.debug(f"check is_nearest for: {irowts_origin} {irowts}")
idx = self.binary_search_ts(select_results, irowts_origin)
if idx == 0:
#tdLog.debug(f"prev row: null,cur row: {select_results[idx]}, next row: {select_results[idx + 1]}")
res = self.distance(irowts, select_results[idx][0]) <= self.distance(irowts, select_results[idx + 1][0])
if not res:
tdLog.debug(f"prev row: null,cur row: {select_results[idx]}, next row: {select_results[idx + 1]}, irowts_origin: {irowts_origin}, irowts: {irowts}")
return res
if idx == len(select_results) - 1:
#tdLog.debug(f"prev row: {select_results[idx - 1]},cur row: {select_results[idx]}, next row: null")
res = self.distance(irowts, select_results[idx][0]) <= self.distance(irowts, select_results[idx - 1][0])
if not res:
tdLog.debug(f"prev row: {select_results[idx - 1]},cur row: {select_results[idx]}, next row: null, irowts_origin: {irowts_origin}, irowts: {irowts}")
return res
#tdLog.debug(f"prev row: {select_results[idx - 1]},cur row: {select_results[idx]}, next row: {select_results[idx + 1]}")
res = self.distance(irowts, select_results[idx][0]) <= self.distance(irowts, select_results[idx - 1][0]) and self.distance(irowts, select_results[idx][0]) <= self.distance(irowts, select_results[idx + 1][0])
if not res:
tdLog.debug(f"prev row: {select_results[idx - 1]},cur row: {select_results[idx]}, next row: {select_results[idx + 1]}, irowts_origin: {irowts_origin}, irowts: {irowts}")
return res
except Exception as e:
self.check_failed = True
tdLog.exit(f"is_nearest error: {e}")
## interp_results: _irowts_origin, _irowts, ..., _isfilled
## select_all_results must be sorted by ts in ascending order
def check_result_for_near(self, interp_results, select_all_results, sql, sql_select_all):
#tdLog.debug(f"check_result_for_near for sql: {sql}, sql_select_all{sql_select_all}")
for row in interp_results:
if row[0].tzinfo is None or row[0].tzinfo.utcoffset(row[0]) is None:
irowts_origin = row[0].replace(tzinfo=get_localzone())
irowts = row[1].replace(tzinfo=get_localzone())
else:
irowts_origin = row[0]
irowts = row[1]
if not self.is_nearest(select_all_results, irowts_origin, irowts):
self.check_failed = True
tdLog.exit(f"interp result is not the nearest for row: {row}, {sql}")
def query_routine(self, sql_queue: queue.Queue, output_queue: queue.Queue):
try:
tdcom = TDCom()
cli = tdcom.newTdSql()
while True:
item = sql_queue.get()
if item is None or self.check_failed:
output_queue.put(None)
break
(sql, sql_select_all, _) = item
cli.query(sql, queryTimes=1)
interp_results = cli.queryResult
if sql_select_all is not None:
cli.query(sql_select_all, queryTimes=1)
output_queue.put((sql, interp_results, cli.queryResult, sql_select_all))
cli.close()
except Exception as e:
self.check_failed = True
tdLog.exit(f"query_routine error: {e}")
def interp_check_near_routine(self, select_all_results, output_queue: queue.Queue):
try:
while True:
item = output_queue.get()
if item is None:
break
(sql, interp_results, all_results, sql_select_all) = item
if all_results is not None:
self.check_result_for_near(interp_results, all_results, sql, sql_select_all)
else:
self.check_result_for_near(interp_results, select_all_results, sql, None)
except Exception as e:
self.check_failed = True
tdLog.exit(f"interp_check_near_routine error: {e}")
def create_qt_threads(self, sql_queue: queue.Queue, output_queue: queue.Queue, num: int):
qts = []
for _ in range(0, num):
qt = threading.Thread(target=self.query_routine, args=(sql_queue, output_queue))
qt.start()
qts.append(qt)
return qts
def wait_qt_threads(self, qts: list):
for qt in qts:
qt.join()
### first(ts) | last(ts)
### 2018-09-17 09:00:00.047 | 2018-09-17 10:23:19.863
def test_interp_fill_extension_near(self):
sql = f"select last(ts), c1, c2 from test.t0"
tdSql.query(sql, queryTimes=1)
lastRow = tdSql.queryResult[0]
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(near)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(61)
for i in range(0, 61):
tdSql.checkData(i, 0, lastRow[0])
tdSql.checkData(i, 2, lastRow[1])
tdSql.checkData(i, 3, lastRow[2])
tdSql.checkData(i, 4, True)
sql = f"select ts, c1, c2 from test.t0 where ts between '2018-09-17 08:59:59' and '2018-09-17 09:00:06' order by ts asc"
tdSql.query(sql, queryTimes=1)
select_all_results = tdSql.queryResult
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 range('2018-09-17 09:00:00', '2018-09-17 09:00:05') every(1s) fill(near)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(6)
self.check_result_for_near(tdSql.queryResult, select_all_results, sql, None)
start = 1537146000000
end = 1537151000000
tdSql.query("select ts, c1, c2 from test.t0 order by ts asc", queryTimes=1)
select_all_results = tdSql.queryResult
qt_threads_num = 4
sql_queue = queue.Queue()
output_queue = queue.Queue()
qts = self.create_qt_threads(sql_queue, output_queue, qt_threads_num)
ct = threading.Thread(target=self.interp_check_near_routine, args=(select_all_results, output_queue))
ct.start()
for i in range(0, ROUND):
range_start = random.randint(start, end)
range_end = random.randint(range_start, end)
every = random.randint(1, 15)
#tdLog.debug(f"range_start: {range_start}, range_end: {range_end}")
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 range({range_start}, {range_end}) every({every}s) fill(near)"
sql_queue.put((sql, None, None))
### no prev only, no next only, no prev and no next, have prev and have next
for i in range(0, ROUND):
range_point = random.randint(start, end)
## all data points are can be filled by near
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 range({range_point}, 1h) fill(near, 1, 2)"
sql_queue.put((sql, None, None))
for i in range(0, ROUND):
range_start = random.randint(start, end)
range_end = random.randint(range_start, end)
range_where_start = random.randint(start, end)
range_where_end = random.randint(range_where_start, end)
every = random.randint(1, 15)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 where ts between {range_where_start} and {range_where_end} range({range_start}, {range_end}) every({every}s) fill(near)"
tdSql.query(f'select to_char(cast({range_where_start} as timestamp), \'YYYY-MM-DD HH24:MI:SS.MS\'), to_char(cast({range_where_end} as timestamp), \'YYYY-MM-DD HH24:MI:SS.MS\')', queryTimes=1)
where_start_str = tdSql.queryResult[0][0]
where_end_str = tdSql.queryResult[0][1]
sql_select_all = f"select ts, c1, c2 from test.t0 where ts between '{where_start_str}' and '{where_end_str}' order by ts asc"
sql_queue.put((sql, sql_select_all, None))
for i in range(0, ROUND):
range_start = random.randint(start, end)
range_end = random.randint(range_start, end)
range_where_start = random.randint(start, end)
range_where_end = random.randint(range_where_start, end)
range_point = random.randint(start, end)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.t0 where ts between {range_where_start} and {range_where_end} range({range_point}, 1h) fill(near, 1, 2)"
tdSql.query(f'select to_char(cast({range_where_start} as timestamp), \'YYYY-MM-DD HH24:MI:SS.MS\'), to_char(cast({range_where_end} as timestamp), \'YYYY-MM-DD HH24:MI:SS.MS\')', queryTimes=1)
where_start_str = tdSql.queryResult[0][0]
where_end_str = tdSql.queryResult[0][1]
sql_select_all = f"select ts, c1, c2 from test.t0 where ts between '{where_start_str}' and '{where_end_str}' order by ts asc"
sql_queue.put((sql, sql_select_all, None))
for i in range(0, qt_threads_num):
sql_queue.put(None)
self.wait_qt_threads(qts)
ct.join()
if self.check_failed:
tdLog.exit("interp check near failed")
def test_interp_extension_irowts_origin(self):
sql = f"select _irowts, _irowts_origin, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(near)"
tdSql.query(sql, queryTimes=1)
sql = f"select _irowts, _irowts_origin, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(NULL)"
tdSql.error(sql, -2147473833)
sql = f"select _irowts, _irowts_origin, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(linear)"
tdSql.error(sql, -2147473833)
sql = f"select _irowts, _irowts_origin, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(NULL_F)"
tdSql.error(sql, -2147473833)
def test_interp_fill_extension(self):
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(near, 0, 0)"
tdSql.query(sql, queryTimes=1)
### must specify value
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(near)"
tdSql.error(sql, -2147473915)
### num of fill value mismatch
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(near, 1)"
tdSql.error(sql, -2147473915)
### range with around interval cannot specify two timepoints, currently not supported
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:02:00', 1h) fill(near, 1, 1)"
tdSql.error(sql, -2147473920) ## syntax error
### NULL/linear cannot specify other values
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:02:00') fill(NULL, 1, 1)"
tdSql.error(sql, -2147473920)
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:02:00') fill(linear, 1, 1)"
tdSql.error(sql, -2147473920) ## syntax error
### cannot have every clause with range around
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) every(1s) fill(prev, 1, 1)"
tdSql.error(sql, -2147473827) ## TSDB_CODE_PAR_INVALID_INTERP_CLAUSE
### cannot specify near/prev/next values when using range
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', '2020-02-01 00:01:00') every(1s) fill(near, 1, 1)"
tdSql.error(sql, -2147473915) ## cannot specify values
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00') every(1s) fill(near, 1, 1)"
tdSql.error(sql, -2147473915) ## cannot specify values
### when range around interval is set, only prev/next/near is supported
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(NULL, 1, 1)"
tdSql.error(sql, -2147473920)
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(NULL)"
tdSql.error(sql, -2147473861) ## TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(linear, 1, 1)"
tdSql.error(sql, -2147473920)
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1h) fill(linear)"
tdSql.error(sql, -2147473861) ## TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE
### range interval cannot be 0
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 0h) fill(near, 1, 1)"
tdSql.error(sql, -2147473861) ## TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1y) fill(near, 1, 1)"
tdSql.error(sql, -2147473915) ## TSDB_CODE_PAR_WRONG_VALUE_TYPE
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1n) fill(near, 1, 1)"
tdSql.error(sql, -2147473915) ## TSDB_CODE_PAR_WRONG_VALUE_TYPE
sql = f"select _irowts, interp(c1), interp(c2), _isfilled from test.meters where ts between '2020-02-01 00:00:00' and '2020-02-01 00:00:00' range('2020-02-01 00:00:00', 1h) fill(near, 1, 1)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(0)
### first(ts) | last(ts)
### 2018-09-17 09:00:00.047 | 2018-09-17 10:23:19.863
sql = "select to_char(first(ts), 'YYYY-MM-DD HH24:MI:SS.MS') from test.meters"
tdSql.query(sql, queryTimes=1)
first_ts = tdSql.queryResult[0][0]
sql = "select to_char(last(ts), 'YYYY-MM-DD HH24:MI:SS.MS') from test.meters"
tdSql.query(sql, queryTimes=1)
last_ts = tdSql.queryResult[0][0]
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2020-02-01 00:00:00', 1d) fill(near, 1, 2)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, '2020-02-01 00:00:00.000')
tdSql.checkData(0, 2, 1)
tdSql.checkData(0, 3, 2)
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2018-09-18 10:25:00', 1d) fill(prev, 3, 4)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, '2018-09-18 10:25:00.000')
tdSql.checkData(0, 2, 3)
tdSql.checkData(0, 3, 4)
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2018-09-16 08:25:00', 1d) fill(next, 5, 6)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, None)
tdSql.checkData(0, 1, '2018-09-16 08:25:00.000')
tdSql.checkData(0, 2, 5)
tdSql.checkData(0, 3, 6)
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2018-09-16 09:00:01', 1d) fill(next, 1, 2)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, first_ts)
tdSql.checkData(0, 1, '2018-09-16 09:00:01')
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('2018-09-18 10:23:19', 1d) fill(prev, 1, 2)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, last_ts)
tdSql.checkData(0, 1, '2018-09-18 10:23:19')
tdSql.checkData(0, 4, True)
sql = f"select _irowts_origin, _irowts, interp(c1), interp(c2), _isfilled from test.meters range('{last_ts}', 1a) fill(next, 1, 2)"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(1)
tdSql.checkData(0, 0, last_ts)
tdSql.checkData(0, 1, last_ts)
tdSql.checkData(0, 4, False)
def test_interval_fill_extension(self):
## not allowed
sql = f"select count(*) from test.meters interval(1s) fill(near)"
tdSql.error(sql, -2147473920) ## syntax error
sql = f"select count(*) from test.meters interval(1s) fill(prev, 1)"
tdSql.error(sql, -2147473920) ## syntax error
sql = f"select count(*) from test.meters interval(1s) fill(next, 1)"
tdSql.error(sql, -2147473920) ## syntax error
sql = f"select _irowts_origin, count(*) from test.meters where ts between '2018-09-17 08:59:59' and '2018-09-17 09:00:06' interval(1s) fill(next)"
tdSql.error(sql, -2147473918) ## invalid column name _irowts_origin
def test_interp_fill_extension_stream(self):
## near is not supported
sql = f"create stream s1 trigger force_window_close into test.s_res_tb as select _irowts, interp(c1), interp(c2)from test.meters partition by tbname every(1s) fill(near);"
tdSql.error(sql, -2147473851) ## TSDB_CODE_PAR_INVALID_STREAM_QUERY
## _irowts_origin is not support
sql = f"create stream s1 trigger force_window_close into test.s_res_tb as select _irowts_origin, interp(c1), interp(c2)from test.meters partition by tbname every(1s) fill(prev);"
tdSql.error(sql, -2147473851) ## TSDB_CODE_PAR_INVALID_STREAM_QUERY
sql = f"create stream s1 trigger force_window_close into test.s_res_tb as select _irowts, interp(c1), interp(c2)from test.meters partition by tbname every(1s) fill(next, 1, 1);"
tdSql.error(sql, -2147473915) ## cannot specify values
def test_interp_extension(self):
self.test_interp_fill_extension_near()
self.test_interp_extension_irowts_origin()
self.test_interp_fill_extension()
self.test_interval_fill_extension()
self.test_interp_fill_extension_stream()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
event = threading.Event()
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

View File

@ -93,6 +93,18 @@ class TDTestCase:
tdSql.error(
f"create stream itp_force_error_1 trigger force_window_close IGNORE EXPIRED 1 IGNORE UPDATE 0 into itp_force_error_1 as select _irowts,tbname,_isfilled,interp(c11,1) from {self.stb_name} partition by tbname every(5s) fill(prev) ;"
)
tdSql.error(
f"create stream itp_1d_next_error_1 trigger force_window_close FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 1 into itp_1d_next_error_t1 as select _irowts,tbname,_isfilled,interp(current) from {self.stb_name} where groupid=100 partition by every(5s) fill(next) ;"
)
tdSql.error(
f"create stream itp_1d_next_error_1 trigger at_once FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 1 into itp_1d_next_error_t1 as select _irowts,tbname,_isfilled,interp(current) from {self.stb_name} where groupid=100 partition by every(5s) fill(next) ;"
)
tdSql.error(
f"create stream itp_1d_next_error_1 trigger window_close FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 1 into itp_1d_next_error_t1 as select _irowts,tbname,_isfilled,interp(current) from {self.stb_name} where groupid=100 partition by every(5s) fill(next) ;"
)
tdSql.error(
f"create stream itp_1d_next_error_1 trigger max_delay 5s FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 1 into itp_1d_next_error_t1 as select _irowts,tbname,_isfilled,interp(current) from {self.stb_name} where groupid=100 partition by every(5s) fill(next) ;"
)
# function name : interp
trigger_mode = "force_window_close"

View File

@ -24,6 +24,7 @@ import time
import traceback
import os
from os import path
import psutil
class TDTestCase:
@ -117,6 +118,69 @@ class TDTestCase:
if not tdSql.getData(2, 0).startswith('new-t3_stb_'):
tdLog.exit("error6")
def caseDropStream(self):
tdLog.info(f"start caseDropStream")
sql = "drop database if exists d1;"
tdSql.query(sql)
sql = "drop database if exists db;"
tdSql.query(sql)
sql ="show streams;"
tdSql.query(sql)
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
sql ="select * from information_schema.ins_stream_tasks;"
tdSql.query(sql)
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
self.taosBenchmark(" -d db -t 2 -v 2 -n 1000000 -y")
# create stream
tdSql.execute("use db;")
tdSql.execute("create stream stream4 fill_history 1 into sta4 as select _wstart, sum(current),avg(current),last(current),min(voltage),first(voltage),last(phase),max(phase),count(phase), _wend, _wduration from meters partition by tbname, ts interval(10a);", show=True)
time.sleep(10)
sql ="select * from information_schema.ins_stream_tasks where status == 'ready';"
tdSql.query(sql, show=True)
tdSql.check_rows_loop(4, sql, loopCount=100, waitTime=0.5)
pl = psutil.pids()
for pid in pl:
try:
if psutil.Process(pid).name() == 'taosd':
taosdPid = pid
break
except psutil.NoSuchProcess:
pass
tdLog.info("taosd pid:{}".format(taosdPid))
p = psutil.Process(taosdPid)
cpuInfo = p.cpu_percent(interval=5)
tdLog.info("taosd cpu:{}".format(cpuInfo))
tdSql.execute("drop stream stream4;", show=True)
sql ="show streams;"
tdSql.query(sql, show=True)
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
sql ="select * from information_schema.ins_stream_tasks;"
tdSql.query(sql, show=True)
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
for i in range(10):
cpuInfo = p.cpu_percent(interval=5)
tdLog.info("taosd cpu:{}".format(cpuInfo))
if cpuInfo < 10:
return
else:
time.sleep(1)
continue
cpuInfo = p.cpu_percent(interval=5)
tdLog.info("taosd cpu:{}".format(cpuInfo))
if cpuInfo > 10:
tdLog.exit("drop stream failed, stream tasks are still running")
# run
def run(self):
self.case1()
@ -145,6 +209,9 @@ class TDTestCase:
tdSql.query(sql)
tdSql.checkRows(0)
self.caseDropStream()
# stop
def stop(self):
tdSql.close()