diff --git a/.gitignore b/.gitignore
index 76b581b182..5f1e24109d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
build/
compile_commands.json
+CMakeSettings.json
.cache
.ycm_extra_conf.py
.tasks
diff --git a/CMakeSettings.json b/CMakeSettings.json
deleted file mode 100644
index d3f2c27bf6..0000000000
--- a/CMakeSettings.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "configurations": [
- {
- "name": "WSL-GCC-Debug",
- "generator": "Unix Makefiles",
- "configurationType": "Debug",
- "buildRoot": "${projectDir}\\build\\",
- "installRoot": "${projectDir}\\build\\",
- "cmakeExecutable": "/usr/bin/cmake",
- "cmakeCommandArgs": "",
- "buildCommandArgs": "",
- "ctestCommandArgs": "",
- "inheritEnvironments": [ "linux_x64" ],
- "wslPath": "${defaultWSLPath}",
- "addressSanitizerRuntimeFlags": "detect_leaks=0",
- "variables": [
- {
- "name": "CMAKE_INSTALL_PREFIX",
- "value": "/mnt/d/TDengine/TDengine/build",
- "type": "PATH"
- }
- ]
- }
- ]
-}
\ No newline at end of file
diff --git a/TDenginelogo.png b/TDenginelogo.png
deleted file mode 100644
index 19a92592d7..0000000000
Binary files a/TDenginelogo.png and /dev/null differ
diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 68caf9a9ac..9edbb7f1cd 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
- GIT_TAG 9cb965f
+ GIT_TAG d2b5dec
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/docs/en/07-develop/01-connect/index.md b/docs/en/07-develop/01-connect/index.md
index 901fe69d24..61eb8f04eb 100644
--- a/docs/en/07-develop/01-connect/index.md
+++ b/docs/en/07-develop/01-connect/index.md
@@ -1,6 +1,7 @@
---
-title: Connect
-description: "This document explains how to establish connections to TDengine and how to install and use TDengine connectors."
+sidebar_label: Connect
+title: Connect to TDengine
+description: "How to establish connections to TDengine and how to install and use TDengine connectors."
---
import Tabs from "@theme/Tabs";
diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md
index d9dadae976..5a84bbf370 100644
--- a/docs/en/12-taos-sql/02-database.md
+++ b/docs/en/12-taos-sql/02-database.md
@@ -71,9 +71,9 @@ database_option: {
- SINGLE_STABLE: specifies whether the database can contain more than one supertable.
- 0: The database can contain multiple supertables.
- 1: The database can contain only one supertable.
-- WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. Enter a time in seconds. The default value is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted.
-- WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted.
-- WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk.
+- WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. Enter a time in seconds. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is 4 days.
+- WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. Enter a size in KB. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is -1.
+- WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value of single copy is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk. The default values of multiple copy is 1 day.
- WAL_SEGMENT_SIZE: specifies the maximum size of a WAL file. After the current WAL file reaches this size, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk.
### Example Statement
diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md
index d6905c84a1..f39096cf5e 100644
--- a/docs/en/12-taos-sql/10-function.md
+++ b/docs/en/12-taos-sql/10-function.md
@@ -613,6 +613,7 @@ SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHER
**Explanations**：
- _P_ is in range [0,100], when _P_ is 0, the result is same as using function MIN; when _P_ is 100, the result is same as function MAX.
- `algo_type` can only be input as `default` or `t-digest` Enter `default` to use a histogram-based algorithm. Enter `t-digest` to use the t-digest algorithm to calculate the approximation of the quantile. `default` is used by default.
+- The approximation result of `t-digest` algorithm is sensitive to input data order. For example, when querying STable with different input data order there might be minor differences in calculated results.
### AVG
diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md
index fcd7876510..deb2522309 100644
--- a/docs/en/12-taos-sql/14-stream.md
+++ b/docs/en/12-taos-sql/14-stream.md
@@ -50,7 +50,7 @@ SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVA
## Delete a Stream
```sql
-DROP STREAM [IF NOT EXISTS] stream_name
+DROP STREAM [IF EXISTS] stream_name
```
This statement deletes the stream processing service only. The data generated by the stream is retained.
diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md
index c9adb0cf78..5f3bef3546 100644
--- a/docs/en/12-taos-sql/24-show.md
+++ b/docs/en/12-taos-sql/24-show.md
@@ -5,16 +5,6 @@ title: SHOW Statement for Metadata
`SHOW` command can be used to get brief system information. To get details about metatadata, information, and status in the system, please use `select` to query the tables in database `INFORMATION_SCHEMA`.
-## SHOW ACCOUNTS
-
-```sql
-SHOW ACCOUNTS;
-```
-
-Shows information about tenants on the system.
-
-Note: TDengine Enterprise Edition only.
-
## SHOW APPS
```sql
diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx
index 00e3bc1bc3..518d3625d5 100644
--- a/docs/en/14-reference/03-connector/05-go.mdx
+++ b/docs/en/14-reference/03-connector/05-go.mdx
@@ -7,7 +7,7 @@ title: TDengine Go Connector
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-import Preparition from "./_preparition.mdx"
+import Preparition from "./_preparation.mdx"
import GoInsert from "../../07-develop/03-insert-data/_go_sql.mdx"
import GoInfluxLine from "../../07-develop/03-insert-data/_go_line.mdx"
import GoOpenTSDBTelnet from "../../07-develop/03-insert-data/_go_opts_telnet.mdx"
diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx
index 1184c98a28..0d391c6ac3 100644
--- a/docs/en/14-reference/03-connector/06-rust.mdx
+++ b/docs/en/14-reference/03-connector/06-rust.mdx
@@ -7,7 +7,7 @@ title: TDengine Rust Connector
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-import Preparition from "./_preparition.mdx"
+import Preparition from "./_preparation.mdx"
import RustInsert from "../../07-develop/03-insert-data/_rust_sql.mdx"
import RustBind from "../../07-develop/03-insert-data/_rust_stmt.mdx"
import RustQuery from "../../07-develop/04-query-data/_rust.mdx"
diff --git a/docs/en/14-reference/03-connector/08-node.mdx b/docs/en/14-reference/03-connector/08-node.mdx
index f93632b417..bf7c6b95ea 100644
--- a/docs/en/14-reference/03-connector/08-node.mdx
+++ b/docs/en/14-reference/03-connector/08-node.mdx
@@ -7,7 +7,7 @@ title: TDengine Node.js Connector
import Tabs from "@theme/Tabs";
import TabItem from "@theme/TabItem";
-import Preparition from "./_preparition.mdx";
+import Preparition from "./_preparation.mdx";
import NodeInsert from "../../07-develop/03-insert-data/_js_sql.mdx";
import NodeInfluxLine from "../../07-develop/03-insert-data/_js_line.mdx";
import NodeOpenTSDBTelnet from "../../07-develop/03-insert-data/_js_opts_telnet.mdx";
diff --git a/docs/en/14-reference/03-connector/09-csharp.mdx b/docs/en/14-reference/03-connector/09-csharp.mdx
index 823e907599..bc16cd086b 100644
--- a/docs/en/14-reference/03-connector/09-csharp.mdx
+++ b/docs/en/14-reference/03-connector/09-csharp.mdx
@@ -7,7 +7,7 @@ title: C# Connector
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-import Preparition from "./_preparition.mdx"
+import Preparition from "./_preparation.mdx"
import CSInsert from "../../07-develop/03-insert-data/_cs_sql.mdx"
import CSInfluxLine from "../../07-develop/03-insert-data/_cs_line.mdx"
import CSOpenTSDBTelnet from "../../07-develop/03-insert-data/_cs_opts_telnet.mdx"
diff --git a/docs/en/14-reference/03-connector/_preparition.mdx b/docs/en/14-reference/03-connector/_preparition.mdx
deleted file mode 100644
index 87538ebfd8..0000000000
--- a/docs/en/14-reference/03-connector/_preparition.mdx
+++ /dev/null
@@ -1,10 +0,0 @@
-- 已安装客户端驱动（使用原生连接必须安装，使用 REST 连接无需安装）
-
-:::info
-
-由于 TDengine 的客户端驱动使用 C 语言编写，使用原生连接时需要加载系统对应安装在本地的客户端驱动共享库文件，通常包含在 TDengine 安装包。TDengine Linux 服务端安装包附带了 TDengine 客户端，也可以单独安装 [Linux 客户端](/get-started/) 。在 Windows 环境开发时需要安装 TDengine 对应的 [Windows 客户端](https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client) 。
-
-- libtaos.so: 在 Linux 系统中成功安装 TDengine 后，依赖的 Linux 版客户端驱动 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。
-- taos.dll: 在 Windows 系统中安装完客户端之后，依赖的 Windows 版客户端驱动 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。
-
-:::
diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md
index 012c49d2c3..9a0a6fb547 100644
--- a/docs/zh/02-intro.md
+++ b/docs/zh/02-intro.md
@@ -26,7 +26,7 @@ TDengine 的主要功能如下：
- [Icinga2](../third-party/icinga2)
- [TCollector](../third-party/tcollector)
- [EMQX](../third-party/emq-broker)
- - [HiveMQ](../third-party/hive-mq-broker) ；
+ - [HiveMQ](../third-party/hive-mq-broker)
2. 查询数据，支持
- [标准 SQL](../taos-sql)，含嵌套查询
- [时序数据特色函数](../taos-sql/function/#time-series-extensions)
@@ -85,14 +85,14 @@ TDengine 的主要功能如下：
-图 1. TDengine技术生态图
+图 1. TDengine 技术生态图
上图中，左侧是各种数据采集或消息队列，包括 OPC-UA、MQTT、Telegraf、也包括 Kafka，他们的数据将被源源不断的写入到 TDengine。右侧则是可视化、BI 工具、组态软件、应用程序。下侧则是 TDengine 自身提供的命令行程序（CLI）以及可视化管理工具。
## 典型适用场景
-作为一个高性能、分布式、支持 SQL 的时序数据库（Database)，TDengine 的典型适用场景包括但不限于 IoT、工业互联网、车联网、IT 运维、能源、金融证券等领域。需要指出的是，TDengine 是针对时序数据场景设计的专用数据库和专用大数据处理工具，因其充分利用了时序大数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。下面本文将对适用场景做更多详细的分析。
+作为一个高性能、分布式、支持 SQL 的时序数据库（Database），TDengine 的典型适用场景包括但不限于 IoT、工业互联网、车联网、IT 运维、能源、金融证券等领域。需要指出的是，TDengine 是针对时序数据场景设计的专用数据库和专用大数据处理工具，因其充分利用了时序大数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。下面本文将对适用场景做更多详细的分析。
### 数据源特点和需求
diff --git a/docs/zh/08-connector/09-csharp.mdx b/docs/zh/08-connector/09-csharp.mdx
index be27bfb685..4e49d84835 100644
--- a/docs/zh/08-connector/09-csharp.mdx
+++ b/docs/zh/08-connector/09-csharp.mdx
@@ -169,9 +169,9 @@ namespace TDengineExample
### 第三方驱动
-`Maikebing.Data.Taos` 是一个 TDengine 的 ADO.NET 连接器，支持 Linux，Windows 平台。该连接器由社区贡献者`麦壳饼@@maikebing` 提供，具体请参考:
+[`IoTSharp.Data.Taos`](https://github.com/IoTSharp/EntityFrameworkCore.Taos) 是一个 TDengine 的 ADO.NET 连接器,其中包含了用于EntityFrameworkCore 的提供程序 IoTSharp.EntityFrameworkCore.Taos 和健康检查组件 IoTSharp.HealthChecks.Taos ，支持 Linux，Windows 平台。该连接器由社区贡献者`麦壳饼@@maikebing` 提供，具体请参考:
-* 接口下载:
+* 接口下载:
* 用法说明:
## 常见问题
diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md
index 1675356c49..2e5a42d7bd 100644
--- a/docs/zh/12-taos-sql/02-database.md
+++ b/docs/zh/12-taos-sql/02-database.md
@@ -71,9 +71,9 @@ database_option: {
- SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。
- 0：表示可以创建多张超级表。
- 1：表示只可以创建一张超级表。
-- WAL_RETENTION_PERIOD：wal 文件的额外保留策略，用于数据订阅。wal 的保存时长，单位为 s。默认为 0，即落盘后立即删除。-1 表示不删除。
-- WAL_RETENTION_SIZE：wal 文件的额外保留策略，用于数据订阅。wal 的保存的最大上限，单位为 KB。默认为 0，即落盘后立即删除。-1 表示不删除。
-- WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当 wal 文件创建并写入后，经过该时间，会自动创建一个新的 wal 文件。默认为 0，即仅在落盘时创建新文件。
+- WAL_RETENTION_PERIOD：wal 文件的额外保留策略，用于数据订阅。wal 的保存时长，单位为 s。单副本默认为 0，即落盘后立即删除。-1 表示不删除。多副本默认为 4 天。
+- WAL_RETENTION_SIZE：wal 文件的额外保留策略，用于数据订阅。wal 的保存的最大上限，单位为 KB。单副本默认为 0，即落盘后立即删除。多副本默认为-1，表示不删除。
+- WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当 wal 文件创建并写入后，经过该时间，会自动创建一个新的 wal 文件。单副本默认为 0，即仅在落盘时创建新文件。多副本默认为 1 天。
- WAL_SEGMENT_SIZE：wal 单个文件大小，单位为 KB。当前写入文件大小超过上限后会自动创建一个新的 wal 文件。默认为 0，即仅在落盘时创建新文件。
### 创建数据库示例
diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md
index 9f999181c4..9c5b7f771e 100644
--- a/docs/zh/12-taos-sql/10-function.md
+++ b/docs/zh/12-taos-sql/10-function.md
@@ -614,6 +614,7 @@ SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHER
**说明**：
- P值范围是[0,100]，当为0时等同于MIN，为100时等同于MAX。
- algo_type 取值为 "default" 或 "t-digest"。 输入为 "default" 时函数使用基于直方图算法进行计算。输入为 "t-digest" 时使用t-digest算法计算分位数的近似结果。如果不指定 algo_type 则使用 "default" 算法。
+- "t-digest"算法的近似结果对于输入数据顺序敏感，对超级表查询时不同的输入排序结果可能会有微小的误差。
### AVG
@@ -917,7 +918,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause];
**返回数据类型**：同应用的字段。
-**适用数据类型**：数值类型。
+**适用数据类型**：数值类型，时间戳类型。
**适用于**：表和超级表。
@@ -932,7 +933,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause];
**返回数据类型**：同应用的字段。
-**适用数据类型**：数值类型。
+**适用数据类型**：数值类型，时间戳类型。
**适用于**：表和超级表。
diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md
index 70b062a6ca..86437c762e 100644
--- a/docs/zh/12-taos-sql/14-stream.md
+++ b/docs/zh/12-taos-sql/14-stream.md
@@ -58,7 +58,7 @@ SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVA
## 删除流式计算
```sql
-DROP STREAM [IF NOT EXISTS] stream_name;
+DROP STREAM [IF EXISTS] stream_name;
```
仅删除流式计算任务，由流式计算写入的数据不会被删除。
diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md
index b4aafdaa0a..6594bde880 100644
--- a/docs/zh/12-taos-sql/24-show.md
+++ b/docs/zh/12-taos-sql/24-show.md
@@ -6,16 +6,6 @@ description: SHOW 命令的完整列表
SHOW 命令可以用来获取简要的系统信息。若想获取系统中详细的各种元数据、系统信息和状态，请使用 select 语句查询 INFORMATION_SCHEMA 数据库中的表。
-## SHOW ACCOUNTS
-
-```sql
-SHOW ACCOUNTS;
-```
-
-显示当前系统中所有租户的信息。
-
-注：企业版独有
-
## SHOW APPS
```sql
diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md
index f84ec65b4c..0d6aad6240 100644
--- a/docs/zh/14-reference/05-taosbenchmark.md
+++ b/docs/zh/14-reference/05-taosbenchmark.md
@@ -405,37 +405,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
订阅子表或者普通表的配置参数在 `specified_table_query` 中设置。
-- **threads** : 执行 SQL 的线程数，默认为 1。
-
-- **interval** : 执行订阅的时间间隔，单位为秒，默认为 0。
-
-- **restart** : "yes" 表示开始新的订阅，"no" 表示继续之前的订阅，默认值为 "no"。
-
-- **keepProgress** : "yes" 表示保留订阅进度，"no" 表示不保留，默认值为 "no"。
-
-- **resubAfterConsume** : "yes" 表示取消之前的订阅然后再次订阅， "no" 表示继续之前的订阅，默认值为 "no"。
+- **threads/concurrent** : 执行 SQL 的线程数，默认为 1。
- **sqls** ：
- **sql** : 执行的 SQL 命令，必填。
- - **result** : 保存查询结果的文件，未指定则不保存。
-
-#### 订阅超级表的配置参数
-
-订阅超级表的配置参数在 `super_table_query` 中设置。
-
-- **stblname** : 要订阅的超级表名称，必填。
-
-- **threads** : 执行 SQL 的线程数，默认为 1。
-
-- **interval** : 执行订阅的时间间隔，单位为秒，默认为 0。
-
-- **restart** : "yes" 表示开始新的订阅，"no" 表示继续之前的订阅，默认值为 "no"。
-
-- **keepProgress** : "yes" 表示保留订阅进度，"no" 表示不保留，默认值为 "no"。
-
-- **resubAfterConsume** : "yes" 表示取消之前的订阅然后再次订阅， "no" 表示继续之前的订阅，默认值为 "no"。
-
-- **sqls** ：
- - **sql** : 执行的 SQL 命令，必填；对于超级表的查询 SQL，在 SQL 命令中保留 "xxxx"，程序会自动将其替换为超级表的所有子表名。
- 替换为超级表中所有的子表名。
- - **result** : 保存查询结果的文件，未指定则不保存。
diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index f72cb3d6d9..2de4ffdc17 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -89,11 +89,12 @@ extern uint16_t tsTelemPort;
// query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
-extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node
+extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node
// query client
extern int32_t tsQueryPolicy;
extern int32_t tsQuerySmaOptimize;
+extern bool tsQueryPlannerTrace;
// client
extern int32_t tsMinSlidingTime;
@@ -147,7 +148,7 @@ struct SConfig *taosGetCfg();
void taosSetAllDebugFlag(int32_t flag, bool rewrite);
void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite);
int32_t taosSetCfg(SConfig *pCfg, char *name);
-void taosLocalCfgForbiddenToChange(char* name, bool* forbidden);
+void taosLocalCfgForbiddenToChange(char *name, bool *forbidden);
#ifdef __cplusplus
}
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index c8b90a9991..8693061d12 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2994,6 +2994,7 @@ typedef struct {
int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const STaosxRsp* pRsp);
int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, STaosxRsp* pRsp);
+void tDeleteSTaosxRsp(STaosxRsp* pRsp);
typedef struct {
SMqRspHead head;
diff --git a/include/common/ttypes.h b/include/common/ttypes.h
index ceb3eae033..a88f65f6ac 100644
--- a/include/common/ttypes.h
+++ b/include/common/ttypes.h
@@ -49,9 +49,6 @@ typedef struct {
#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
-#define IS_VAR_DATA_TYPE(t) \
- (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
-#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
@@ -268,11 +265,16 @@ typedef struct {
#define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT)
#define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE)
#define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)))
+#define IS_TIMESTAMP_TYPE(_t) ((_t) == TSDB_DATA_TYPE_TIMESTAMP)
#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)))
#define IS_MATHABLE_TYPE(_t) \
(IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
+#define IS_VAR_DATA_TYPE(t) \
+ (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
+#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
+
#define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
#define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
#define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h
index 3223d4cdb8..9652579705 100644
--- a/include/libs/nodes/cmdnodes.h
+++ b/include/libs/nodes/cmdnodes.h
@@ -78,6 +78,9 @@ typedef struct SDatabaseOptions {
int32_t walRetentionSize;
int32_t walRollPeriod;
int32_t walSegmentSize;
+ bool walRetentionPeriodIsSet;
+ bool walRetentionSizeIsSet;
+ bool walRollPeriodIsSet;
} SDatabaseOptions;
typedef struct SCreateDatabaseStmt {
diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h
index 5743d33608..687949a9c0 100644
--- a/include/libs/nodes/nodes.h
+++ b/include/libs/nodes/nodes.h
@@ -244,6 +244,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE,
QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE,
QUERY_NODE_PHYSICAL_PLAN_PARTITION,
+ QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION,
QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC,
QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC,
QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h
index 6fd6a316eb..8aeb86102a 100644
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@@ -488,6 +488,8 @@ typedef struct SPartitionPhysiNode {
SNodeList* pTargets;
} SPartitionPhysiNode;
+typedef SPartitionPhysiNode SStreamPartitionPhysiNode;
+
typedef struct SDataSinkNode {
ENodeType type;
SDataBlockDescNode* pInputDataBlockDesc;
diff --git a/include/os/osDir.h b/include/os/osDir.h
index 9019d4f802..95b1a6ee1d 100644
--- a/include/os/osDir.h
+++ b/include/os/osDir.h
@@ -56,6 +56,7 @@ void taosRemoveDir(const char *dirname);
bool taosDirExist(const char *dirname);
int32_t taosMkDir(const char *dirname);
int32_t taosMulMkDir(const char *dirname);
+int32_t taosMulModeMkDir(const char *dirname, int mode);
void taosRemoveOldFiles(const char *dirname, int32_t keepDays);
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen);
diff --git a/include/util/tdef.h b/include/util/tdef.h
index 2bc821b873..4efc79a3ba 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -360,14 +360,17 @@ typedef enum ELogicConditionType {
#define TSDB_DB_SCHEMALESS_OFF 0
#define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF
-#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1
-#define TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD (24 * 60 * 60 * 4)
-#define TSDB_DB_MIN_WAL_RETENTION_SIZE -1
-#define TSDB_DEFAULT_DB_WAL_RETENTION_SIZE -1
-#define TSDB_DB_MIN_WAL_ROLL_PERIOD 0
-#define TSDB_DEFAULT_DB_WAL_ROLL_PERIOD (24 * 60 * 60 * 1)
-#define TSDB_DB_MIN_WAL_SEGMENT_SIZE 0
-#define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE 0
+#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1
+#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 0
+#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD (24 * 60 * 60 * 4)
+#define TSDB_DB_MIN_WAL_RETENTION_SIZE -1
+#define TSDB_REP_DEF_DB_WAL_RET_SIZE 0
+#define TSDB_REPS_DEF_DB_WAL_RET_SIZE -1
+#define TSDB_DB_MIN_WAL_ROLL_PERIOD 0
+#define TSDB_REP_DEF_DB_WAL_ROLL_PERIOD 0
+#define TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD (24 * 60 * 60 * 1)
+#define TSDB_DB_MIN_WAL_SEGMENT_SIZE 0
+#define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE 0
#define TSDB_MIN_ROLLUP_MAX_DELAY 1 // unit millisecond
#define TSDB_MAX_ROLLUP_MAX_DELAY (15 * 60 * 1000)
@@ -386,7 +389,7 @@ typedef enum ELogicConditionType {
#define TSDB_DEFAULT_EXPLAIN_VERBOSE false
-#define TSDB_EXPLAIN_RESULT_ROW_SIZE (16*1024)
+#define TSDB_EXPLAIN_RESULT_ROW_SIZE (16 * 1024)
#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
#define TSDB_MAX_FIELD_LEN 16384
diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh
index 3db9005f95..94a24a4148 100755
--- a/packaging/deb/makedeb.sh
+++ b/packaging/deb/makedeb.sh
@@ -45,6 +45,7 @@ mkdir -p ${pkg_dir}${install_home_path}/include
mkdir -p ${pkg_dir}${install_home_path}/script
cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg
+cp ${compile_dir}/../packaging/cfg/taosd.service ${pkg_dir}${install_home_path}/cfg
if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then
cp ${compile_dir}/test/cfg/taosadapter.toml ${pkg_dir}${install_home_path}/cfg || :
fi
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index f08f54ef4b..9f9a14952e 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -1773,7 +1773,7 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) {
}
return TMQ_RES_TABLE_META;
} else if (TD_RES_TMQ_TAOSX(res)) {
- return TMQ_RES_TAOSX;
+ return TMQ_RES_DATA;
} else {
return TMQ_RES_INVALID;
}
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 0bab6a8611..908f7c014e 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -91,6 +91,7 @@ bool tsSmlDataFormat =
// query
int32_t tsQueryPolicy = 1;
int32_t tsQuerySmaOptimize = 0;
+bool tsQueryPlannerTrace = false;
/*
* denote if the server needs to compress response message at the application layer to client, including query rsp,
@@ -286,6 +287,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1;
+ if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1;
if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
@@ -429,9 +431,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
static int32_t taosUpdateServerCfg(SConfig *pCfg) {
SConfigItem *pItem;
- ECfgSrcType stype;
- int32_t numOfCores;
- int64_t totalMemoryKB;
+ ECfgSrcType stype;
+ int32_t numOfCores;
+ int64_t totalMemoryKB;
pItem = cfgGetItem(tsCfg, "numOfCores");
if (pItem == NULL) {
@@ -572,7 +574,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
return 0;
}
-
static void taosSetClientLogCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "logDir");
tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX);
@@ -643,6 +644,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32;
+ tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
return 0;
}
@@ -972,6 +974,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32;
} else if (strcasecmp("qDebugFlag", name) == 0) {
qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
+ } else if (strcasecmp("queryPlannerTrace", name) == 0) {
+ tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
}
break;
}
@@ -1129,7 +1133,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false);
- if (taosMulMkDir(tsLogDir) != 0) {
+ if (taosMulModeMkDir(tsLogDir, 0777) != 0) {
uError("failed to create dir:%s since %s", tsLogDir, terrstr());
cfgCleanup(pCfg);
return -1;
@@ -1241,16 +1245,14 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
}
const char *options[] = {
- "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", "tsdbDebugFlag",
- "tqDebugFlag", "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag",
- "tmrDebugFlag", "uDebugFlag", "smaDebugFlag", "rpcDebugFlag", "qDebugFlag", "metaDebugFlag",
- "jniDebugFlag",
+ "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", "tsdbDebugFlag", "tqDebugFlag",
+ "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag", "tmrDebugFlag", "uDebugFlag",
+ "smaDebugFlag", "rpcDebugFlag", "qDebugFlag", "metaDebugFlag", "jniDebugFlag",
};
int32_t *optionVars[] = {
- &dDebugFlag, &vDebugFlag, &mDebugFlag, &wDebugFlag, &sDebugFlag, &tsdbDebugFlag,
- &tqDebugFlag, &fsDebugFlag, &udfDebugFlag, &smaDebugFlag, &idxDebugFlag, &tdbDebugFlag,
- &tmrDebugFlag, &uDebugFlag, &smaDebugFlag, &rpcDebugFlag, &qDebugFlag, &metaDebugFlag,
- &jniDebugFlag,
+ &dDebugFlag, &vDebugFlag, &mDebugFlag, &wDebugFlag, &sDebugFlag, &tsdbDebugFlag, &tqDebugFlag,
+ &fsDebugFlag, &udfDebugFlag, &smaDebugFlag, &idxDebugFlag, &tdbDebugFlag, &tmrDebugFlag, &uDebugFlag,
+ &smaDebugFlag, &rpcDebugFlag, &qDebugFlag, &metaDebugFlag, &jniDebugFlag,
};
int32_t optionSize = tListLen(options);
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 1921415239..b056677a03 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -5984,6 +5984,17 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) {
}
return 0;
}
+
+void tDeleteSTaosxRsp(STaosxRsp *pRsp) {
+ taosArrayDestroy(pRsp->blockDataLen);
+ taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
+ taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);
+ taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
+
+ taosArrayDestroy(pRsp->createTableLen);
+ taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree);
+}
+
int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) {
if (tEncodeI64(pEncoder, pReq->uid) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->ts) < 0) return -1;
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 8c1c3ba873..fee322ecad 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -358,10 +358,10 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0;
if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF;
if (pCfg->walRetentionPeriod < 0 && pCfg->walRetentionPeriod != -1)
- pCfg->walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD;
+ pCfg->walRetentionPeriod = TSDB_REPS_DEF_DB_WAL_RET_PERIOD;
if (pCfg->walRetentionSize < 0 && pCfg->walRetentionSize != -1)
- pCfg->walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE;
- if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD;
+ pCfg->walRetentionSize = TSDB_REPS_DEF_DB_WAL_RET_SIZE;
+ if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD;
if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
}
diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h
index 19dd321814..c3441a43f0 100644
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@@ -140,11 +140,12 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle);
int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
// tqRead
-int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
+int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
+int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset);
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);
// tqExec
-int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp* pRsp);
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp);
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp);
// tqMeta
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index eed997b486..3f42d8360e 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -196,6 +196,66 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
return 0;
}
+int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) {
+ ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
+ ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
+
+ if (pRsp->withSchema) {
+ ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum);
+ } else {
+ ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0);
+ }
+
+ if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
+ if (pRsp->blockNum > 0) {
+ ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);
+ } else {
+ ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version);
+ }
+ }
+
+ int32_t len = 0;
+ int32_t code = 0;
+ tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code);
+ if (code < 0) {
+ return -1;
+ }
+ int32_t tlen = sizeof(SMqRspHead) + len;
+ void* buf = rpcMallocCont(tlen);
+ if (buf == NULL) {
+ return -1;
+ }
+
+ ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__TAOSX_RSP;
+ ((SMqRspHead*)buf)->epoch = pReq->epoch;
+ ((SMqRspHead*)buf)->consumerId = pReq->consumerId;
+
+ void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+
+ SEncoder encoder = {0};
+ tEncoderInit(&encoder, abuf, len);
+ tEncodeSTaosxRsp(&encoder, pRsp);
+ tEncoderClear(&encoder);
+
+ SRpcMsg rsp = {
+ .info = pMsg->info,
+ .pCont = buf,
+ .contLen = tlen,
+ .code = 0,
+ };
+ tmsgSendRsp(&rsp);
+
+ char buf1[80] = {0};
+ char buf2[80] = {0};
+ tFormatOffset(buf1, 80, &pRsp->reqOffset);
+ tFormatOffset(buf2, 80, &pRsp->rspOffset);
+ tqDebug("taosx rsp, vgId:%d, from consumer:%" PRId64
+ ", (epoch %d) send rsp, block num: %d, reqOffset:%s, rspOffset:%s",
+ TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
+
+ return 0;
+}
+
static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) {
return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG &&
pLeft->val.version <= pRight->val.version;
@@ -303,6 +363,22 @@ static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t su
return 0;
}
+static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) {
+ pRsp->reqOffset = pReq->reqOffset;
+
+ pRsp->withTbName = 1;
+ pRsp->withSchema = 1;
+ pRsp->blockData = taosArrayInit(0, sizeof(void*));
+ pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
+ pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
+ pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
+
+ if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
+ return -1;
+ }
+ return 0;
+}
+
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
SMqPollReq* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId;
@@ -341,9 +417,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
tqDebug("tmq poll: consumer %" PRId64 " (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId,
pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf);
- SMqDataRsp dataRsp = {0};
- tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
-
// 2.reset offset if needed
if (reqOffset.type > 0) {
fetchOffsetNew = reqOffset;
@@ -367,6 +440,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal));
}
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
+ SMqDataRsp dataRsp = {0};
+ tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+
tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, offset reset to %" PRId64, consumerId,
pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version);
@@ -380,16 +456,38 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
" in vg %d, subkey %s, reset none failed",
pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey);
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
- code = -1;
- tDeleteSMqDataRsp(&dataRsp);
- return code;
+ return -1;
}
}
}
- if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN || fetchOffsetNew.type != TMQ_OFFSET__LOG) {
- SMqMetaRsp metaRsp = {0};
- tqScan(pTq, pHandle, &dataRsp, &metaRsp, &fetchOffsetNew);
+ if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+ SMqDataRsp dataRsp = {0};
+ tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+ tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew);
+
+ if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+ code = -1;
+ }
+
+ tqDebug("tmq poll: consumer %ld, subkey %s, vg %d, send data blockNum:%d, offset type:%d, uid:%ld, version:%ld",
+ consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type,
+ dataRsp.rspOffset.uid, dataRsp.rspOffset.version);
+
+ tDeleteSMqDataRsp(&dataRsp);
+ return code;
+ }
+
+ // for taosx
+ ASSERT(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN);
+
+ SMqMetaRsp metaRsp = {0};
+
+ STaosxRsp taosxRsp = {0};
+ tqInitTaosxRsp(&taosxRsp, pReq);
+
+ if (fetchOffsetNew.type != TMQ_OFFSET__LOG) {
+ tqScan(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew);
if (metaRsp.metaRspLen > 0) {
if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
@@ -399,29 +497,30 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
pHandle->subKey, TD_VID(pTq->pVnode), metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
metaRsp.rspOffset.version);
taosMemoryFree(metaRsp.metaRsp);
- goto OVER;
+ tDeleteSTaosxRsp(&taosxRsp);
+ return code;
}
- if (dataRsp.blockNum > 0) {
- if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+ if (taosxRsp.blockNum > 0) {
+ if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
code = -1;
}
- goto OVER;
+ tDeleteSTaosxRsp(&taosxRsp);
+ return code;
} else {
- fetchOffsetNew = dataRsp.rspOffset;
+ fetchOffsetNew = taosxRsp.rspOffset;
}
- tqDebug("tmq poll: consumer %ld, subkey %s, vg %d, send data blockNum:%d, offset type:%d,uid:%ld,version:%ld",
- consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type,
- dataRsp.rspOffset.uid, dataRsp.rspOffset.version);
+ tqDebug("taosx poll: consumer %ld, subkey %s, vg %d, send data blockNum:%d, offset type:%d,uid:%ld,version:%ld",
+ consumerId, pHandle->subKey, TD_VID(pTq->pVnode), taosxRsp.blockNum, taosxRsp.rspOffset.type,
+ taosxRsp.rspOffset.uid, taosxRsp.rspOffset.version);
}
- if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) {
+ if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
int64_t fetchVer = fetchOffsetNew.version + 1;
pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
if (pCkHead == NULL) {
- code = -1;
- goto OVER;
+ return -1;
}
walSetReaderCapacity(pHandle->pWalReader, 2048);
@@ -436,13 +535,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
}
if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) {
- // TODO add push mgr
-
- tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
- if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+ tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
+ if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
code = -1;
}
- goto OVER;
+ tDeleteSTaosxRsp(&taosxRsp);
+ if (pCkHead) taosMemoryFree(pCkHead);
+ return code;
}
SWalCont* pHead = &pCkHead->head;
@@ -453,17 +552,19 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
if (pHead->msgType == TDMT_VND_SUBMIT) {
SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
- if (tqLogScanExec(pTq, pHandle, pCont, &dataRsp) < 0) {
+ if (tqTaosxScanLog(pTq, pHandle, pCont, &taosxRsp) < 0) {
/*ASSERT(0);*/
}
// TODO batch optimization:
// TODO continue scan until meeting batch requirement
- if (dataRsp.blockNum > 0 /* threshold */) {
- tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
- if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+ if (taosxRsp.blockNum > 0 /* threshold */) {
+ tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
+ if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
code = -1;
}
- goto OVER;
+ tDeleteSTaosxRsp(&taosxRsp);
+ if (pCkHead) taosMemoryFree(pCkHead);
+ return code;
} else {
fetchVer++;
}
@@ -472,30 +573,22 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
ASSERT(pHandle->fetchMeta);
ASSERT(IS_META_MSG(pHead->msgType));
tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType);
- SMqMetaRsp metaRsp = {0};
tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer);
metaRsp.resMsgType = pHead->msgType;
metaRsp.metaRspLen = pHead->bodyLen;
metaRsp.metaRsp = pHead->body;
if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
code = -1;
- goto OVER;
+ taosMemoryFree(pCkHead);
+ return code;
}
code = 0;
- goto OVER;
+ if (pCkHead) taosMemoryFree(pCkHead);
+ return code;
}
}
}
-
- // send empty to client
- if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
- code = -1;
- }
-
-OVER:
- if (pCkHead) taosMemoryFree(pCkHead);
- tDeleteSMqDataRsp(&dataRsp);
- return code;
+ return 0;
}
int32_t tqProcessVgDeleteReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
@@ -602,10 +695,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe
pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode);
pHandle->execHandle.execDb.pFilterOutTbUid =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
- buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext **)(&handle.sContext));
+ buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
+ (SSnapContext**)(&handle.sContext));
- pHandle->execHandle.task =
- qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL);
+ pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL);
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c
index da596d07f9..8c3fa25446 100644
--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqExec.c
@@ -60,7 +60,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) {
return 0;
}
-int64_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
+int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
const STqExecHandle* pExec = &pHandle->execHandle;
ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
@@ -89,18 +89,41 @@ int64_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
if (qExecTask(task, &pDataBlock, &ts) < 0) {
ASSERT(0);
}
- tqDebug("tmq task execute end, get %p", pDataBlock);
+ tqDebug("tmq task executed, get %p", pDataBlock);
- if (pDataBlock) {
- tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
- pRsp->blockNum++;
+ if (pDataBlock == NULL) {
+ break;
+ }
+
+ tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
+ pRsp->blockNum++;
+
+ if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+ rowCnt += pDataBlock->info.rows;
+ if (rowCnt >= 4096) break;
}
}
+ if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) {
+ ASSERT(0);
+ return -1;
+ }
+ ASSERT(pRsp->rspOffset.type != 0);
+
+ if (pRsp->withTbName) {
+ if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) {
+ int64_t uid = pExec->pExecReader->msgIter.uid;
+ tqAddTbNameToRsp(pTq, uid, pRsp);
+ } else {
+ pRsp->withTbName = false;
+ }
+ }
+ ASSERT(pRsp->withSchema == false);
+
return 0;
}
-int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
+int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
const STqExecHandle* pExec = &pHandle->execHandle;
qTaskInfo_t task = pExec->task;
@@ -134,7 +157,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
int64_t uid = 0;
if (pOffset->type == TMQ_OFFSET__LOG) {
uid = pExec->pExecReader->msgIter.uid;
- if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) {
+ if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp) < 0) {
continue;
}
} else {
@@ -144,18 +167,14 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
}
if (pRsp->withSchema) {
if (pOffset->type == TMQ_OFFSET__LOG) {
- tqAddBlockSchemaToRsp(pExec, pRsp);
+ tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
} else {
SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
taosArrayPush(pRsp->blockSchema, &pSW);
}
}
- if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
- tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
- } else {
- tqAddBlockDataToRsp(pDataBlock, pRsp, taosArrayGetSize(pDataBlock->pDataBlock));
- }
+ tqAddBlockDataToRsp(pDataBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pDataBlock->pDataBlock));
pRsp->blockNum++;
if (pOffset->type == TMQ_OFFSET__LOG) {
continue;
@@ -165,34 +184,32 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
}
}
- if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN) {
- if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
- if (qStreamExtractPrepareUid(task) != 0) {
- continue;
- }
- tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode),
- pHandle->snapshotVer + 1);
- break;
- }
-
- if (pRsp->blockNum > 0) {
- tqDebug("tmqsnap task exec exited, get data");
- break;
- }
-
- SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
- if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
- tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts);
- qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
- tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META;
- tqDebug("tmqsnap task exec change to get data");
+ if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+ if (qStreamExtractPrepareUid(task) != 0) {
continue;
}
-
- *pMetaRsp = *tmp;
- tqDebug("tmqsnap task exec exited, get meta");
+ tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode),
+ pHandle->snapshotVer + 1);
+ break;
}
+ if (pRsp->blockNum > 0) {
+ tqDebug("tmqsnap task exec exited, get data");
+ break;
+ }
+
+ SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
+ if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
+ tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts);
+ qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
+ tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META;
+ tqDebug("tmqsnap task exec change to get data");
+ continue;
+ }
+
+ *pMetaRsp = *tmp;
+ tqDebug("tmqsnap task exec exited, get meta");
+
tqDebug("task exec exited");
break;
}
@@ -205,12 +222,11 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
return 0;
}
-int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp* pRsp) {
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp) {
STqExecHandle* pExec = &pHandle->execHandle;
ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN);
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
- pRsp->withSchema = 1;
STqReader* pReader = pExec->pExecReader;
tqReaderSetDataMsg(pReader, pReq, 0);
while (tqNextDataBlock(pReader)) {
@@ -220,18 +236,17 @@ int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp
}
if (pRsp->withTbName) {
int64_t uid = pExec->pExecReader->msgIter.uid;
- if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) {
+ if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp) < 0) {
blockDataFreeRes(&block);
continue;
}
}
- tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock));
+ tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock));
blockDataFreeRes(&block);
- tqAddBlockSchemaToRsp(pExec, pRsp);
+ tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
pRsp->blockNum++;
}
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
- pRsp->withSchema = 1;
STqReader* pReader = pExec->pExecReader;
tqReaderSetDataMsg(pReader, pReq, 0);
while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
@@ -241,24 +256,25 @@ int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp
}
if (pRsp->withTbName) {
int64_t uid = pExec->pExecReader->msgIter.uid;
- if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) {
+ if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp) < 0) {
blockDataFreeRes(&block);
continue;
}
}
- tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock));
+ tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock));
blockDataFreeRes(&block);
- tqAddBlockSchemaToRsp(pExec, pRsp);
+ tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
pRsp->blockNum++;
}
-#if 0
+#if 1
if (pHandle->fetchMeta && pRsp->blockNum) {
SSubmitMsgIter iter = {0};
tInitSubmitMsgIter(pReq, &iter);
STaosxRsp* pXrsp = (STaosxRsp*)pRsp;
while (1) {
SSubmitBlk* pBlk = NULL;
- if (tGetSubmitMsgNext(&iter, &pBlk) < 0) return -1;
+ if (tGetSubmitMsgNext(&iter, &pBlk) < 0) break;
+ if (pBlk == NULL) break;
if (pBlk->schemaLen > 0) {
if (pXrsp->createTableNum == 0) {
pXrsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 6db9d5e6f4..cfb04881e6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -2039,7 +2039,7 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
size += pBlockCol->szOffset;
// value
- if (pColData->flag != (HAS_NULL | HAS_NONE)) {
+ if ((pColData->flag != (HAS_NULL | HAS_NONE)) && pColData->nData) {
code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, ppOut, nOut + size,
&pBlockCol->szValue, ppBuf);
if (code) goto _exit;
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index b4e2840330..3bed81f657 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -143,6 +143,7 @@ typedef struct {
STqOffsetVal prepareStatus; // for tmq
STqOffsetVal lastStatus; // for tmq
SMqMetaRsp metaRsp; // for tmq fetching meta
+ int8_t returned;
int64_t snapshotVer;
SSchemaWrapper *schema;
@@ -409,6 +410,7 @@ typedef enum EStreamScanMode {
STREAM_SCAN_FROM_READERHANDLE = 1,
STREAM_SCAN_FROM_RES,
STREAM_SCAN_FROM_UPDATERES,
+ STREAM_SCAN_FROM_DELETERES,
STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
STREAM_SCAN_FROM_DATAREADER_RANGE,
} EStreamScanMode;
@@ -437,12 +439,24 @@ typedef struct SStreamAggSupporter {
SSDataBlock* pScanBlock;
} SStreamAggSupporter;
-typedef struct SessionWindowSupporter {
+typedef struct SWindowSupporter {
SStreamAggSupporter* pStreamAggSup;
int64_t gap;
uint16_t parentType;
SAggSupporter* pIntervalAggSup;
-} SessionWindowSupporter;
+} SWindowSupporter;
+
+typedef struct SPartitionBySupporter {
+ SArray* pGroupCols; // group by columns, SArray
+ SArray* pGroupColVals; // current group column values, SArray
+ char* keyBuf; // group by keys for hash
+ bool needCalc; // partition by column
+} SPartitionBySupporter;
+
+typedef struct SPartitionDataInfo {
+ uint64_t groupId;
+ SArray* rowIds;
+} SPartitionDataInfo;
typedef struct STimeWindowSupp {
int8_t calTrigger;
@@ -477,7 +491,9 @@ typedef struct SStreamScanInfo {
SOperatorInfo* pStreamScanOp;
SOperatorInfo* pTableScanOp;
SArray* childIds;
- SessionWindowSupporter sessionSup;
+ SWindowSupporter windowSup;
+ SPartitionBySupporter partitionSup;
+ SExprSupp* pPartScalarSup;
bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
int32_t scanWinIndex; // for state operator
int32_t pullDataResIndex;
@@ -690,7 +706,6 @@ typedef struct SPartitionOperatorInfo {
SArray* sortedGroupArray; // SDataGroupInfo sorted by group id
int32_t groupIndex; // group index
int32_t pageIndex; // page index of current group
- SSDataBlock* pUpdateRes;
SExprSupp scalarSup;
} SPartitionOperatorInfo;
@@ -742,8 +757,8 @@ typedef struct SStreamSessionAggOperatorInfo {
SSDataBlock* pWinBlock; // window result
SqlFunctionCtx* pDummyCtx; // for combine
SSDataBlock* pDelRes; // delete result
- bool returnDelete;
SSDataBlock* pUpdateRes; // update window
+ bool returnUpdate;
SHashObj* pStDeleted;
void* pDelIterator;
SArray* pChildren; // cache for children's result; final stream operator
@@ -752,6 +767,16 @@ typedef struct SStreamSessionAggOperatorInfo {
bool ignoreExpiredData;
} SStreamSessionAggOperatorInfo;
+typedef struct SStreamPartitionOperatorInfo {
+ SOptrBasicInfo binfo;
+ SPartitionBySupporter partitionSup;
+ SExprSupp scalarSup;
+ SHashObj* pPartitions;
+ void* parIte;
+ SSDataBlock* pInputDataBlock;
+ int32_t tsColIndex;
+} SStreamPartitionOperatorInfo;
+
typedef struct STimeSliceOperatorInfo {
SSDataBlock* pRes;
STimeWindow win;
@@ -953,6 +978,9 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
+
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode,
SExecTaskInfo* pTaskInfo);
@@ -1021,8 +1049,9 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx);
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
-void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, int32_t uidCol, uint64_t* pID);
+void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp);
void printDataBlock(SSDataBlock* pBlock, const char* flag);
+uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition,
SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset,
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index f1ac9ef8b1..278f02b228 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -745,6 +745,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
SOperatorInfo* pOperator = pTaskInfo->pRoot;
ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
pTaskInfo->streamInfo.prepareStatus = *pOffset;
+ pTaskInfo->streamInfo.returned = 0;
if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) {
return 0;
}
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index b53d35a1a1..d8e5c0abe5 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -4150,6 +4150,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children);
} else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) {
pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo);
+ } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION == type) {
+ pOptr = createStreamPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE == type) {
SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode;
pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo);
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index 5d123f723e..599edb0722 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -830,3 +830,205 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo,
setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset);
return TSDB_CODE_SUCCESS;
}
+
+uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId) {
+ if (pExprSup->pExprInfo != NULL) {
+ int32_t code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("calaculate group id error, code:%d", code);
+ }
+ }
+ recordNewGroupKeys(pParSup->pGroupCols, pParSup->pGroupColVals, pBlock, rowId);
+ int32_t len = buildGroupKeys(pParSup->keyBuf, pParSup->pGroupColVals);
+ uint64_t groupId = calcGroupId(pParSup->keyBuf, len);
+ return groupId;
+}
+
+static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) {
+ return pInfo->parIte != NULL;
+}
+
+static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
+ SStreamPartitionOperatorInfo* pInfo = pOperator->info;
+ SSDataBlock* pDest = pInfo->binfo.pRes;
+ ASSERT(hasRemainPartion(pInfo));
+ SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte;
+ blockDataCleanup(pDest);
+ int32_t rows = taosArrayGetSize(pParInfo->rowIds);
+ SSDataBlock* pSrc = pInfo->pInputDataBlock;
+ for (int32_t i = 0; i < rows; i++) {
+ int32_t rowIndex = *(int32_t*)taosArrayGet(pParInfo->rowIds, i);
+ for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; j++) {
+ int32_t slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId;
+ SColumnInfoData* pSrcCol = taosArrayGet(pSrc->pDataBlock, slotId);
+ SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, j);
+ bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL);
+ char* pSrcData = colDataGetData(pSrcCol, rowIndex);
+ colDataAppend(pDestCol, pDest->info.rows, pSrcData, isNull);
+ }
+ pDest->info.rows++;
+ }
+ blockDataUpdateTsWindow(pDest, pInfo->tsColIndex);
+ pDest->info.groupId = pParInfo->groupId;
+ pOperator->resultInfo.totalRows += pDest->info.rows;
+ pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte);
+ ASSERT(pDest->info.rows > 0);
+ printDataBlock(pDest, "stream partitionby");
+ return pDest;
+}
+
+static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) {
+ pInfo->pInputDataBlock = pBlock;
+ for (int32_t i = 0; i < pBlock->info.rows; ++i) {
+ recordNewGroupKeys(pInfo->partitionSup.pGroupCols, pInfo->partitionSup.pGroupColVals, pBlock, i);
+ int32_t keyLen = buildGroupKeys(pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupColVals);
+ SPartitionDataInfo* pParData =
+ (SPartitionDataInfo*) taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen);
+ if (pParData) {
+ taosArrayPush(pParData->rowIds, &i);
+ } else {
+ SPartitionDataInfo newParData = {0};
+ newParData.groupId = calcGroupId(pInfo->partitionSup.keyBuf, keyLen);
+ newParData.rowIds = taosArrayInit(64, sizeof(int32_t));
+ taosArrayPush(newParData.rowIds, &i);
+ taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData,
+ sizeof(SPartitionDataInfo));
+ }
+ }
+}
+
+static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
+ if (pOperator->status == OP_EXEC_DONE) {
+ return NULL;
+ }
+
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStreamPartitionOperatorInfo* pInfo = pOperator->info;
+ if (hasRemainPartion(pInfo)) {
+ return buildStreamPartitionResult(pOperator);
+ }
+
+ int64_t st = taosGetTimestampUs();
+ SOperatorInfo* downstream = pOperator->pDownstream[0];
+ {
+ pInfo->pInputDataBlock = NULL;
+ SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
+ if (pBlock == NULL) {
+ doSetOperatorCompleted(pOperator);
+ return NULL;
+ }
+ printDataBlock(pBlock, "stream partitionby recv");
+ switch (pBlock->info.type) {
+ case STREAM_NORMAL:
+ case STREAM_PULL_DATA:
+ case STREAM_INVALID:
+ pInfo->binfo.pRes->info.type = pBlock->info.type;
+ break;
+ default:
+ return pBlock;
+ }
+
+ // there is an scalar expression that needs to be calculated right before apply the group aggregation.
+ if (pInfo->scalarSup.pExprInfo != NULL) {
+ pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock,
+ pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL);
+ if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
+ longjmp(pTaskInfo->env, pTaskInfo->code);
+ }
+ }
+ taosHashClear(pInfo->pPartitions);
+ doStreamHashPartitionImpl(pInfo, pBlock);
+ }
+ pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
+
+ pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL);
+ return buildStreamPartitionResult(pOperator);
+}
+
+static void destroyStreamPartitionOperatorInfo(void* param) {
+ SStreamPartitionOperatorInfo* pInfo = (SStreamPartitionOperatorInfo*)param;
+ cleanupBasicInfo(&pInfo->binfo);
+ taosArrayDestroy(pInfo->partitionSup.pGroupCols);
+
+ for(int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++){
+ SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->partitionSup.pGroupColVals, i);
+ taosMemoryFree(key.pData);
+ }
+ taosArrayDestroy(pInfo->partitionSup.pGroupColVals);
+
+ taosMemoryFree(pInfo->partitionSup.keyBuf);
+ cleanupExprSupp(&pInfo->scalarSup);
+ taosMemoryFreeClear(param);
+}
+
+void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) {
+ if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+ return;
+ }
+ SStreamScanInfo* pScanInfo = downstream->info;
+ pScanInfo->partitionSup = *pParSup;
+ pScanInfo->pPartScalarSup = pExpr;
+}
+
+SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) {
+ SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo));
+ SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+ if (pInfo == NULL || pOperator == NULL) {
+ goto _error;
+ }
+ int32_t code = TSDB_CODE_SUCCESS;
+ pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys);
+
+ if (pPartNode->pExprs != NULL) {
+ int32_t num = 0;
+ SExprInfo* pCalExprInfo = createExprInfo(pPartNode->pExprs, NULL, &num);
+ code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+ }
+
+ int32_t keyLen = 0;
+ code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+ pInfo->partitionSup.needCalc = true;
+
+ SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc);
+ if (!pResBlock) {
+ goto _error;
+ }
+ blockDataEnsureCapacity(pResBlock, 4096);
+ pInfo->binfo.pRes = pResBlock;
+ pInfo->parIte = NULL;
+ pInfo->pInputDataBlock = NULL;
+ _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+ pInfo->pPartitions = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
+ pInfo->tsColIndex = 0;
+
+ int32_t numOfCols = 0;
+ SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols);
+
+ pOperator->name = "StreamPartitionOperator";
+ pOperator->blocking = false;
+ pOperator->status = OP_NOT_OPENED;
+ pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION;
+ pOperator->exprSupp.numOfExprs = numOfCols;
+ pOperator->exprSupp.pExprInfo = pExprInfo;
+ pOperator->info = pInfo;
+ pOperator->pTaskInfo = pTaskInfo;
+ pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, destroyStreamPartitionOperatorInfo,
+ NULL, NULL, NULL);
+
+ initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup);
+ code = appendDownstream(pOperator, &downstream, 1);
+ return pOperator;
+
+ _error:
+ pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+ taosMemoryFreeClear(pInfo);
+ taosMemoryFreeClear(pOperator);
+ return NULL;
+}
+
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index de6768b83a..b740ec21d3 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -920,49 +920,28 @@ static void doClearBufferedBlocks(SStreamScanInfo* pInfo) {
}
static bool isSessionWindow(SStreamScanInfo* pInfo) {
- return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
- pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
+ return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
+ pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
}
static bool isStateWindow(SStreamScanInfo* pInfo) {
- return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
+ return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
}
static bool isIntervalWindow(SStreamScanInfo* pInfo) {
- return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL ||
- pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
- pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
+ return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL ||
+ pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
+ pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
}
static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) {
- return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
+ return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
}
static bool isSlidingWindow(SStreamScanInfo* pInfo) {
return isIntervalWindow(pInfo) && pInfo->interval.interval != pInfo->interval.sliding;
}
-static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) {
- uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t));
- if (groupId) {
- return *groupId;
- }
- return 0;
- /* Todo(liuyao) for partition by column
- recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, rowId);
- int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals);
- uint64_t resId = 0;
- uint64_t* groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len);
- if (groupId) {
- return *groupId;
- } else if (len != 0) {
- resId = calcGroupId(pTableScanInfo->keyBuf, len);
- taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &resId, sizeof(uint64_t));
- }
- return resId;
- */
-}
-
static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
uint64_t* groupCol = (uint64_t*)pColInfo->pData;
@@ -976,6 +955,62 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
pTableScanInfo->currentGroupId = -1;
}
+static void freeArray(void* array) {
+ taosArrayDestroy(array);
+}
+
+static void resetTableScanOperator(SOperatorInfo* pTableScanOp) {
+ STableScanInfo* pTableScanInfo = pTableScanOp->info;
+ pTableScanInfo->cond.startVersion = -1;
+ pTableScanInfo->cond.endVersion = -1;
+ SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList;
+ SArray* allTbls = pTableScanOp->pTaskInfo->tableqinfoList.pTableList;
+ taosArrayClearP(gpTbls, freeArray);
+ taosArrayPush(gpTbls, &allTbls);
+ STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX};
+ resetTableScanInfo(pTableScanOp->info, &win);
+}
+
+static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, int64_t maxVersion) {
+ SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList;
+ taosArrayClear(gpTbls);
+ STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0};
+ SArray* tbls = taosArrayInit(1, sizeof(STableKeyInfo));
+ taosArrayPush(tbls, &tblInfo);
+ taosArrayPush(gpTbls, &tbls);
+
+ STimeWindow win = {.skey = startTs, .ekey = endTs};
+ STableScanInfo* pTableScanInfo = pTableScanOp->info;
+ pTableScanInfo->cond.startVersion = -1;
+ pTableScanInfo->cond.endVersion = maxVersion;
+ resetTableScanInfo(pTableScanOp->info, &win);
+ SSDataBlock* pRes = doTableScan(pTableScanOp);
+ resetTableScanOperator(pTableScanOp);
+ return pRes;
+}
+
+static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
+ SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uid, ts, ts, maxVersion);
+ if (!pPreRes || pPreRes->info.rows == 0) {
+ return 0;
+ }
+ ASSERT(pPreRes->info.rows == 1);
+ return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0);
+}
+
+static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
+ if (pInfo->partitionSup.needCalc) {
+ return getGroupIdByCol(pInfo, uid, ts, maxVersion);
+ }
+
+ SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map;
+ uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t));
+ if (groupId) {
+ return *groupId;
+ }
+ return 0;
+}
+
static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
if ((*pRowIndex) == pBlock->info.rows) {
return false;
@@ -987,6 +1022,9 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endData = (TSKEY*)pEndTsCol->pData;
STimeWindow win = {.skey = startData[*pRowIndex], .ekey = endData[*pRowIndex]};
+ SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+ uint64_t* gpData = (uint64_t*)pGpCol->pData;
+ uint64_t groupId = gpData[*pRowIndex];
SColumnInfoData* pCalStartTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* calStartData = (TSKEY*)pCalStartTsCol->pData;
@@ -1001,11 +1039,11 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
(*pRowIndex)++;
for (; *pRowIndex < pBlock->info.rows; (*pRowIndex)++) {
- if (win.skey == startData[*pRowIndex]) {
+ if (win.skey == startData[*pRowIndex] && groupId == gpData[*pRowIndex]) {
win.ekey = TMAX(win.ekey, endData[*pRowIndex]);
continue;
}
- if (win.skey == endData[*pRowIndex]) {
+ if (win.skey == endData[*pRowIndex] && groupId == gpData[*pRowIndex]) {
win.skey = TMIN(win.skey, startData[*pRowIndex]);
continue;
}
@@ -1020,15 +1058,19 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
}
static STimeWindow getSlidingWindow(TSKEY* tsCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo,
- int32_t* pRowIndex) {
+ int32_t* pRowIndex, bool hasGroup) {
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC);
STimeWindow endWin = win;
STimeWindow preWin = win;
while (1) {
- (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey, binarySearchForKey, NULL,
- TSDB_ORDER_ASC);
+ if (hasGroup) {
+ (*pRowIndex) += 1;
+ } else {
+ (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey,
+ binarySearchForKey, NULL, TSDB_ORDER_ASC);
+ }
do {
preWin = endWin;
getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC);
@@ -1060,7 +1102,26 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
return NULL;
}
- if (pResult->info.groupId == pInfo->groupId) {
+ if (pInfo->partitionSup.needCalc) {
+ SSDataBlock* tmpBlock = createOneDataBlock(pResult, true);
+ blockDataCleanup(pResult);
+ for (int32_t i = 0; i < tmpBlock->info.rows; i++) {
+ if (calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, tmpBlock, i) == pInfo->groupId) {
+ for (int32_t j = 0; j < pInfo->pTableScanOp->exprSupp.numOfExprs; j++) {
+ SColumnInfoData* pSrcCol = taosArrayGet(tmpBlock->pDataBlock, j);
+ SColumnInfoData* pDestCol = taosArrayGet(pResult->pDataBlock, j);
+ bool isNull = colDataIsNull(pSrcCol, tmpBlock->info.rows, i, NULL);
+ char* pSrcData = colDataGetData(pSrcCol, i);
+ colDataAppend(pDestCol, pResult->info.rows, pSrcData, isNull);
+ }
+ pResult->info.rows++;
+ }
+ }
+ if (pResult->info.rows > 0) {
+ pResult->info.calWin = pInfo->updateWin;
+ return pResult;
+ }
+ } else if (pResult->info.groupId == pInfo->groupId) {
pResult->info.calWin = pInfo->updateWin;
return pResult;
}
@@ -1091,17 +1152,18 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr
SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
int32_t dummy = 0;
+ int64_t version = pSrcBlock->info.version - 1;
for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
- uint64_t groupId = getGroupId(pInfo->pTableScanOp, uidCol[i]);
+ uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version);
// gap must be 0.
SResultWindowInfo* pStartWin =
- getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy);
+ getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy);
if (!pStartWin) {
// window has been closed.
continue;
}
SResultWindowInfo* pEndWin =
- getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy);
+ getCurSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy);
ASSERT(pEndWin);
TSKEY ts = INT64_MIN;
colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false);
@@ -1121,34 +1183,49 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
if (rows == 0) {
return TSDB_CODE_SUCCESS;
}
- int32_t code = blockDataEnsureCapacity(pDestBlock, rows);
+ int32_t code = blockDataEnsureCapacity(pDestBlock, rows * 2);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
- SColumnInfoData* pTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
- SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
- uint64_t* uidCol = (uint64_t*)pUidCol->pData;
- ASSERT(pTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
- TSKEY* tsCol = (TSKEY*)pTsCol->pData;
+ SColumnInfoData* pSrcTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
+ SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
+ uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData;
+ SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+ uint64_t* srcGp = (uint64_t*)pSrcGpCol->pData;
+ ASSERT(pSrcTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
+ TSKEY* tsCol = (TSKEY*)pSrcTsCol->pData;
SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
+ SColumnInfoData* pDeUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX);
SColumnInfoData* pGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
SColumnInfoData* pCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
SColumnInfoData* pCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
- uint64_t groupId = getGroupId(pInfo->pTableScanOp, uidCol[0]);
+ int64_t version = pSrcBlock->info.version - 1;
for (int32_t i = 0; i < rows;) {
- colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(tsCol + i), false);
- STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i);
- colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(tsCol + i - 1), false);
-
+ uint64_t srcUid = srcUidData[i];
+ uint64_t groupId = getGroupIdByData(pInfo, srcUid, tsCol[i], version);
+ uint64_t srcGpId = srcGp[i];
+ TSKEY calStartTs = tsCol[i];
+ colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
+ STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i, pInfo->partitionSup.needCalc);
+ TSKEY calEndTs = tsCol[i - 1];
+ colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
+ colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false);
colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false);
pDestBlock->info.rows++;
+ if (pInfo->partitionSup.needCalc && srcGpId != 0 && groupId != srcGpId) {
+ colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
+ colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
+ colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false);
+ colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
+ colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
+ colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&srcGpId), false);
+ pDestBlock->info.rows++;
+ }
}
- // all rows have same group id
- pDestBlock->info.groupId = groupId;
return TSDB_CODE_SUCCESS;
}
@@ -1160,17 +1237,20 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock);
}
pDestBlock->info.type = STREAM_CLEAR;
+ pDestBlock->info.version = pSrcBlock->info.version;
blockDataUpdateTsWindow(pDestBlock, 0);
return code;
}
-void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, int32_t uidCol, uint64_t* pID) {
+void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp) {
SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
- SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, uidCol);
+ SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
+ SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)pStartTs, false);
colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false);
- colDataAppend(pUidCol, pBlock->info.rows, (const char*)pID, false);
+ colDataAppend(pUidCol, pBlock->info.rows, (const char*)pUid, false);
+ colDataAppend(pGpCol, pBlock->info.rows, (const char*)pGp, false);
pBlock->info.rows++;
}
@@ -1195,24 +1275,18 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
// must check update info first.
bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]);
bool closedWin = isClosed && isSignleIntervalWindow(pInfo) &&
- isDeletedWindow(&win, pBlock->info.groupId, pInfo->sessionSup.pIntervalAggSup);
+ isDeletedWindow(&win, pBlock->info.groupId, pInfo->windowSup.pIntervalAggSup);
if ((update || closedWin) && out) {
- appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, UID_COLUMN_INDEX, &pBlock->info.uid);
+ uint64_t gpId = closedWin&&pInfo->partitionSup.needCalc ?
+ calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId) : 0;
+ appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid,
+ &gpId);
}
}
- if (out) {
+ if (out && pInfo->pUpdateDataRes->info.rows > 0) {
+ pInfo->pUpdateDataRes->info.version = pBlock->info.version;
blockDataUpdateTsWindow(pInfo->pUpdateDataRes, 0);
- pInfo->pUpdateDataRes->info.type = STREAM_CLEAR;
- }
-}
-
-static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t uidColIndex) {
- ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3);
- SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, uidColIndex);
- uint64_t* uidCol = (uint64_t*)pColDataInfo->pData;
- ASSERT(pBlock->info.rows > 0);
- for (int32_t i = 0; i < pBlock->info.rows; i++) {
- uidCol[i] = getGroupId(pOperator, uidCol[i]);
+ pInfo->pUpdateDataRes->info.type = pInfo->partitionSup.needCalc ? STREAM_DELETE_DATA : STREAM_CLEAR;
}
}
@@ -1285,17 +1359,22 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
if (pResult && pResult->info.rows > 0) {
qDebug("queue scan tsdb return %d rows", pResult->info.rows);
+ pTaskInfo->streamInfo.returned = 1;
return pResult;
} else {
- STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
- tsdbReaderClose(pTSInfo->dataReader);
- pTSInfo->dataReader = NULL;
- tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
- qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1);
- if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) {
+ if (!pTaskInfo->streamInfo.returned) {
+ STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
+ tsdbReaderClose(pTSInfo->dataReader);
+ pTSInfo->dataReader = NULL;
+ tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
+ qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1);
+ if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) {
+ return NULL;
+ }
+ ASSERT(pInfo->tqReader->pWalReader->curVersion == pTaskInfo->streamInfo.snapshotVer + 1);
+ } else {
return NULL;
}
- ASSERT(pInfo->tqReader->pWalReader->curVersion == pTaskInfo->streamInfo.snapshotVer + 1);
}
}
@@ -1442,6 +1521,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
default:
break;
}
+ // printDataBlock(pBlock, "stream scan recv");
return pBlock;
} else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
qDebug("scan mode %d", pInfo->scanMode);
@@ -1451,6 +1531,14 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
return pInfo->pRes;
} break;
+ case STREAM_SCAN_FROM_DELETERES: {
+ generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
+ prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
+ pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
+ copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
+ pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA;
+ return pInfo->pDeleteDataRes;
+ } break;
case STREAM_SCAN_FROM_UPDATERES: {
generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
@@ -1466,6 +1554,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->cond.twindows, pInfo->groupId, version);
pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA;
checkUpdateData(pInfo, true, pSDB, false);
+ // printDataBlock(pSDB, "stream scan update");
return pSDB;
}
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
@@ -1474,7 +1563,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
break;
}
- SStreamAggSupporter* pSup = pInfo->sessionSup.pStreamAggSup;
+ SStreamAggSupporter* pSup = pInfo->windowSup.pStreamAggSup;
if (isStateWindow(pInfo) && pSup->pScanBlock->info.rows > 0) {
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
pInfo->updateResIndex = 0;
@@ -1540,7 +1629,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
// record the scan action.
pInfo->numOfExec++;
pOperator->resultInfo.totalRows += pBlockInfo->rows;
- printDataBlock(pInfo->pRes, "stream scan");
+ // printDataBlock(pInfo->pRes, "stream scan");
if (pBlockInfo->rows == 0) {
updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
@@ -1549,19 +1638,20 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
checkUpdateData(pInfo, true, pInfo->pRes, true);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey);
if (pInfo->pUpdateDataRes->info.rows > 0) {
+ pInfo->updateResIndex = 0;
if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) {
- pInfo->updateResIndex = 0;
pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
} else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) {
pInfo->scanMode = STREAM_SCAN_FROM_RES;
return pInfo->pUpdateDataRes;
+ } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) {
+ pInfo->scanMode = STREAM_SCAN_FROM_DELETERES;
}
}
}
qDebug("scan rows: %d", pBlockInfo->rows);
return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
-
} else {
ASSERT(0);
return NULL;
@@ -1825,13 +1915,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
ASSERT(pHandle->tqReader);
pInfo->tqReader = pHandle->tqReader;
}
-
- if (pTSInfo->pdInfo.interval.interval > 0) {
- pInfo->pUpdateInfo = updateInfoInitP(&pTSInfo->pdInfo.interval, pInfo->twAggSup.waterMark);
- } else {
- pInfo->pUpdateInfo = NULL;
- }
-
+
+ pInfo->pUpdateInfo = NULL;
pInfo->pTableScanOp = pTableScanOp;
pInfo->interval = pTSInfo->pdInfo.interval;
@@ -1862,8 +1947,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
pInfo->pCondition = pScanPhyNode->node.pConditions;
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
- pInfo->sessionSup =
- (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN};
+ pInfo->windowSup =
+ (SWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN};
pInfo->groupId = 0;
pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
pInfo->pStreamScanOp = pOperator;
@@ -1872,6 +1957,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX};
pInfo->pUpdateDataRes = createSpecialDataBlock(STREAM_CLEAR);
pInfo->assignBlockUid = pTableScanNode->assignBlockUid;
+ pInfo->partitionSup.needCalc = false;
pOperator->name = "StreamScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 152bd5939d..ab304b8d2d 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -897,7 +897,7 @@ int64_t getWinReskey(void* data, int32_t index) {
int32_t compareWinRes(void* pKey, void* data, int32_t index) {
SArray* res = (SArray*)data;
- SWinKey* pos = taosArrayGetP(res, index);
+ SWinKey* pos = taosArrayGet(res, index);
SResKeyPos* pData = (SResKeyPos*)pKey;
if (*(int64_t*)pData->key == pos->ts) {
if (pData->groupId > pos->groupId) {
@@ -919,10 +919,11 @@ static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
}
void* pIte = NULL;
while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
- SResKeyPos* pResKey = (SResKeyPos*)pIte;
+ SResKeyPos* pResKey = *(SResKeyPos**)pIte;
int32_t index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes);
if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) {
taosArrayRemove(pDelWins, index);
+ delSize = taosArrayGetSize(pDelWins);
}
}
}
@@ -1423,7 +1424,7 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId)
return true;
}
-void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pUpWins, SInterval* pInterval) {
+void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pDelWins, SInterval* pInterval, SHashObj* pUpdatedMap) {
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* tsStarts = (TSKEY*)pStartCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
@@ -1433,9 +1434,12 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock,
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC);
doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
- if (pUpWins) {
- SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
- taosArrayPush(pUpWins, &winRes);
+ SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
+ if (pDelWins) {
+ taosArrayPush(pDelWins, &winRes);
+ }
+ if (pUpdatedMap) {
+ taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
}
}
}
@@ -1446,19 +1450,14 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval*
TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
TSKEY* endTsCols = (TSKEY*)pEndTsCol->pData;
- uint64_t* pGpDatas = NULL;
- if (pBlock->info.type == STREAM_RETRIEVE) {
- SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
- pGpDatas = (uint64_t*)pGpCol->pData;
- }
- int32_t step = 0;
- int32_t startPos = 0;
+ SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+ uint64_t* pGpDatas = (uint64_t*)pGpCol->pData;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
while (win.ekey <= endTsCols[i]) {
- uint64_t winGpId = pGpDatas ? pGpDatas[startPos] : pBlock->info.groupId;
+ uint64_t winGpId = pGpDatas[i];
bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput);
if (pUpWins && res) {
SWinKey winRes = {.ts = win.skey, .groupId = winGpId};
@@ -1571,13 +1570,10 @@ static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlo
return;
}
blockDataEnsureCapacity(pBlock, size - *index);
- SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
- SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+ uint64_t uid = 0;
for (int32_t i = *index; i < size; i++) {
SWinKey* pWin = taosArrayGet(pWins, i);
- colDataAppend(pTsCol, pBlock->info.rows, (const char*)&pWin->ts, false);
- colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&pWin->groupId, false);
- pBlock->info.rows++;
+ appendOneRow(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId);
(*index)++;
}
}
@@ -1596,6 +1592,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
if (pOperator->status == OP_RES_TO_RETURN) {
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows > 0) {
+ printDataBlock(pInfo->pDelRes, "single interval");
return pInfo->pDelRes;
}
@@ -1632,7 +1629,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
continue;
}
if (pBlock->info.type == STREAM_DELETE_DATA) {
- doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
+ doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap);
continue;
} else if (pBlock->info.type == STREAM_GET_ALL) {
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
@@ -1707,6 +1704,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
taosHashCleanup(pUpdatedMap);
doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
if (pInfo->pDelRes->info.rows > 0) {
+ printDataBlock(pInfo->pDelRes, "single interval");
return pInfo->pDelRes;
}
@@ -1828,14 +1826,16 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
return needed;
}
-void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup) {
+void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval, int64_t waterMark) {
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
- // Todo(liuyao) support partition by column
+ initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, waterMark);
return;
}
SStreamScanInfo* pScanInfo = downstream->info;
- pScanInfo->sessionSup.parentType = type;
- pScanInfo->sessionSup.pIntervalAggSup = pSup;
+ pScanInfo->windowSup.parentType = type;
+ pScanInfo->windowSup.pIntervalAggSup = pSup;
+ pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, waterMark);
+ pScanInfo->interval = *pInterval;
}
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
@@ -1921,7 +1921,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) {
- initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup);
+ initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
}
code = appendDownstream(pOperator, &downstream, 1);
@@ -2849,14 +2849,26 @@ _error:
}
void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
- SExecTaskInfo* pTaskInfo) {
+ SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) {
for (int32_t k = 0; k < numOfOutput; ++k) {
if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
- continue;
- }
- int32_t code = TSDB_CODE_SUCCESS;
- if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
- code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
+ if (!pTimeWindowData) {
+ continue;
+ }
+
+ SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);
+ char* p = GET_ROWCELL_INTERBUF(pEntryInfo);
+ SColumnInfoData idata = {0};
+ idata.info.type = TSDB_DATA_TYPE_BIGINT;
+ idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
+ idata.pData = p;
+
+ SScalarParam out = {.columnData = &idata};
+ SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
+ pDestCtx[k].sfp.process(&tw, 1, &out);
+ pEntryInfo->numOfRes = 1;
+ }else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
+ int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
if (code != TSDB_CODE_SUCCESS) {
qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
pTaskInfo->code = code;
@@ -2874,8 +2886,14 @@ bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) {
return p1 != NULL;
}
+STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
+ STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
+ w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
+ return w;
+}
+
static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray,
- int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) {
+ int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pUpdatedMap) {
int32_t size = taosArrayGetSize(pWinArray);
if (!pInfo->pChildren) {
return;
@@ -2883,11 +2901,14 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr
for (int32_t i = 0; i < size; i++) {
SWinKey* pWinRes = taosArrayGet(pWinArray, i);
SResultRow* pCurResult = NULL;
- STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts + 1};
- setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
+ STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval);
+ if (isDeletedWindow(&parentWin, pWinRes->groupId, &pInfo->aggSup) && isCloseWindow(&parentWin, &pInfo->twAggSup)) {
+ continue;
+ }
+ setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &parentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
- bool find = true;
+ int32_t num = 0;
for (int32_t j = 0; j < numOfChildren; j++) {
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j);
SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
@@ -2895,15 +2916,16 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr
if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) {
continue;
}
- find = true;
+ num++;
SResultRow* pChResult = NULL;
- setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId,
+ setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &parentWin, true, &pChResult, pWinRes->groupId,
pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup,
pTaskInfo);
- compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo);
+ updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true);
+ compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
}
- if (find && pUpdated) {
- saveResultRow(pCurResult, pWinRes->groupId, pUpdated);
+ if (num > 1 && pUpdatedMap) {
+ saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap);
setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur);
}
}
@@ -2934,12 +2956,6 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {
static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }
-STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
- STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
- w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
- return w;
-}
-
static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
SHashObj* pUpdatedMap) {
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
@@ -3126,6 +3142,25 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
}
}
+static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pInfo) {
+ int32_t size = taosArrayGetSize(wins);
+ for (int32_t i = 0; i < size; i++) {
+ SWinKey* winKey = taosArrayGet(wins, i);
+ STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
+ if (isCloseWindow(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) {
+ void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
+ if (!chIds) {
+ SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId};
+ // add pull data request
+ savePullWindow(&pull, pInfo->pPullWins);
+ int32_t size = taosArrayGetSize(pInfo->pChildren);
+ addPullWindow(pInfo->pPullDataMap, winKey, size);
+ qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size);
+ }
+ }
+ }
+}
+
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
@@ -3150,12 +3185,20 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
return pInfo->pPullDataRes;
}
+ doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
+ if (pInfo->pDelRes->info.rows != 0) {
+ // process the rest of the data
+ printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
+ return pInfo->pDelRes;
+ }
+
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0) {
pOperator->status = OP_EXEC_DONE;
if (!IS_FINAL_OP(pInfo)) {
// semi interval operator clear disk buffer
clearStreamIntervalOperator(pInfo);
+ qDebug("===stream===clear semi operator");
} else {
freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
}
@@ -3219,23 +3262,28 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
}
removeResults(pUpWins, pUpdatedMap);
copyDataBlock(pInfo->pUpdateRes, pBlock);
- // copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
pInfo->returnUpdate = true;
taosArrayDestroy(pUpWins);
break;
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
- doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
+ SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
+ doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, delWins, &pInfo->interval, pUpdatedMap);
if (IS_FINAL_OP(pInfo)) {
int32_t childIndex = getChildIndex(pBlock);
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
SExprSupp* pChildSup = &pChildOp->exprSupp;
- doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval);
- rebuildIntervalWindow(pInfo, pSup, pInfo->pDelWins, pInfo->binfo.pRes->info.groupId,
- pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated);
+ doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval, NULL);
+ rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId,
+ pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdatedMap);
+ addRetriveWindow(delWins, pInfo);
+ taosArrayAddAll(pInfo->pDelWins, delWins);
+ taosArrayDestroy(delWins);
continue;
}
- removeResults(pInfo->pDelWins, pUpdatedMap);
+ removeResults(delWins, pUpdatedMap);
+ taosArrayAddAll(pInfo->pDelWins, delWins);
+ taosArrayDestroy(delWins);
break;
} else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
@@ -3309,6 +3357,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
return pInfo->pPullDataRes;
}
+ // we should send result first.
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows != 0) {
printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
@@ -3470,7 +3519,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
aggEncodeResultRow, aggDecodeResultRow, NULL);
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
- initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup);
+ initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
}
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -3567,10 +3616,18 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
}
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
- uint16_t type) {
- ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
+ uint16_t type, int32_t tsColIndex) {
+ if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
+ SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
+ pScanInfo->tsColIndex = tsColIndex;
+ }
+
+ if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+ initDownStream(downstream->pDownstream[0], pAggSup, gap, waterMark, type, tsColIndex);
+ return;
+ }
SStreamScanInfo* pScanInfo = downstream->info;
- pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
+ pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
}
@@ -3642,7 +3699,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
pInfo->isFinal = false;
pInfo->pPhyNode = pPhyNode;
pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
- pInfo->returnDelete = false;
pOperator->name = "StreamSessionWindowAggOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
@@ -3653,7 +3709,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
aggEncodeResultRow, aggDecodeResultRow, NULL);
if (downstream) {
- initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType);
+ initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex);
code = appendDownstream(pOperator, &downstream, 1);
}
return pOperator;
@@ -3683,13 +3739,13 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); }
-static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) {
- SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
+static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) {
+ SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
return taosArrayInsert(pWinInfos, index, &win);
}
-static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) {
- SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
+static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) {
+ SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
return taosArrayPush(pWinInfos, &win);
}
@@ -3748,7 +3804,7 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY star
int32_t size = taosArrayGetSize(pWinInfos);
if (size == 0) {
*pIndex = 0;
- return addNewSessionWindow(pWinInfos, startTs);
+ return addNewSessionWindow(pWinInfos, startTs, endTs);
}
// find the first position which is smaller than the key
int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
@@ -3774,10 +3830,10 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY star
if (index == size - 1) {
*pIndex = taosArrayGetSize(pWinInfos);
- return addNewSessionWindow(pWinInfos, startTs);
+ return addNewSessionWindow(pWinInfos, startTs, endTs);
}
*pIndex = index + 1;
- return insertNewSessionWindow(pWinInfos, startTs, index + 1);
+ return insertNewSessionWindow(pWinInfos, startTs, endTs, index + 1);
}
int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,
@@ -3789,7 +3845,7 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TS
if (pWinInfo->win.skey > pStartTs[i]) {
if (pStDeleted && pWinInfo->isOutput) {
SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
- taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey));
+ taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
pWinInfo->isOutput = false;
}
pWinInfo->win.skey = pStartTs[i];
@@ -3904,11 +3960,12 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex,
setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
&pInfo->streamAggSup, pTaskInfo);
pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
- compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo);
+ updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true);
+ compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
if (pWinInfo->isOutput) {
SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
- taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey));
+ taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
pWinInfo->isOutput = false;
}
taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
@@ -4005,10 +4062,11 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc
if (!pCurWin) {
break;
}
+ SResultWindowInfo delWin = *pCurWin;
deleteWindow(pAggSup->pCurWins, winIndex, fp);
if (result) {
- pCurWin->groupId = gpDatas[i];
- taosArrayPush(result, pCurWin);
+ delWin.groupId = gpDatas[i];
+ taosArrayPush(result, &delWin);
}
}
}
@@ -4033,6 +4091,7 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup,
ASSERT(isInWindow(pCurWin, tsCols[i], gap));
doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
if (result) {
+ pCurWin->groupId = gpCols[i];
taosArrayPush(result, pCurWin);
}
}
@@ -4067,10 +4126,18 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It
size_t keyLen = 0;
while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
SWinKey* res = *Ite;
- SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
- colDataAppend(pTsCol, pBlock->info.rows, (const char*)&res->ts, false);
+ SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
+ colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->ts, false);
+ SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
+ colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->ts, false);
+ SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
+ colDataAppendNULL(pUidCol, pBlock->info.rows);
SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false);
+ SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
+ colDataAppendNULL(pCalStCol, pBlock->info.rows);
+ SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
+ colDataAppendNULL(pCalEdCol, pBlock->info.rows);
pBlock->info.rows += 1;
if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
break;
@@ -4081,8 +4148,8 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It
}
}
-static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId,
- int32_t numOfOutput, SOperatorInfo* pOperator) {
+static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray,
+ int32_t numOfOutput, SOperatorInfo* pOperator, SHashObj* pStUpdated, bool needCreate) {
SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -4092,9 +4159,15 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
for (int32_t i = 0; i < size; i++) {
SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
SResultRow* pCurResult = NULL;
+ uint64_t groupId = pParentWin->groupId;
+ int32_t winIndex = 0;
+ if (needCreate) {
+ pParentWin = getSessionTimeWindow(&pInfo->streamAggSup, pParentWin->win.skey, pParentWin->win.ekey, groupId, 0, &winIndex);
+ }
setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
&pInfo->streamAggSup, pTaskInfo);
int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
+ int32_t num = 0;
for (int32_t j = 0; j < numOfChildren; j++) {
SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j);
SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
@@ -4110,15 +4183,24 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
SResultRow* pChResult = NULL;
setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
- compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo);
+ updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pChWin->win, true);
+ compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
+ num++;
continue;
} else if (!pChWin->isClosed) {
break;
}
}
}
+ if (num == 0 && needCreate) {
+ deleteWindow(pInfo->streamAggSup.pCurWins, winIndex, NULL);
+ }
+ if (pStUpdated && num > 0) {
+ SWinKey value = {.ts = pParentWin->win.skey, .groupId = groupId};
+ taosHashPut(pStUpdated, &pParentWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
+ }
SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
ASSERT(size > 0);
setBufPageDirty(bufPage, true);
@@ -4198,7 +4280,46 @@ static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) {
for (int32_t i = 0; i < size; i++) {
SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i);
SWinKey res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId};
- taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey));
+ taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
+ }
+}
+
+static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
+ int32_t size = taosArrayGetSize(pWins);
+ for (int32_t i = 0; i < size; i++) {
+ SResultWindowInfo* pWin = taosArrayGet(pWins, i);
+ taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
+ }
+}
+
+int32_t compareWinKey(void* pKey, void* data, int32_t index) {
+ SArray* res = (SArray*)data;
+ SResKeyPos* pos = taosArrayGetP(res, index);
+ SWinKey* pData = (SWinKey*)pKey;
+ if (pData->ts == *(int64_t*)pos->key) {
+ if (pData->groupId > pos->groupId) {
+ return 1;
+ } else if (pData->groupId < pos->groupId) {
+ return -1;
+ }
+ return 0;
+ } else if (pData->ts > *(int64_t*)pos->key) {
+ return 1;
+ }
+ return -1;
+}
+
+static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) {
+ int32_t size = taosHashGetSize(pStDeleted);
+ if (size == 0) {
+ return;
+ }
+
+ int32_t num = taosArrayGetSize(update);
+ for (int32_t i = 0; i < num; i++) {
+ SResKeyPos* pos = taosArrayGetP(update, i);
+ SWinKey winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId};
+ taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey));
}
}
@@ -4226,7 +4347,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
SOperatorInfo* downstream = pOperator->pDownstream[0];
- SArray* pUpdated = taosArrayInit(16, POINTER_BYTES);
+ SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); // SResKeyPos
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
@@ -4242,9 +4363,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
int32_t childIndex = getChildIndex(pBlock);
SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
- doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX,
- pChildOp->exprSupp.numOfExprs, 0, NULL);
- rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
+ doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX, pChildOp->exprSupp.numOfExprs,
+ 0, NULL);
+ rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, NULL, false);
}
taosArrayDestroy(pWins);
continue;
@@ -4258,9 +4379,10 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
// gap must be 0
doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL);
- rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
+ rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated, true);
}
copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
+ removeSessionResults(pStUpdated, pWins);
taosArrayDestroy(pWins);
continue;
} else if (pBlock->info.type == STREAM_GET_ALL) {
@@ -4303,6 +4425,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
pInfo->ignoreExpiredData, NULL);
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL);
copyUpdateResult(pStUpdated, pUpdated);
+ removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
taosHashCleanup(pStUpdated);
finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
@@ -4333,14 +4456,6 @@ static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
pInfo->streamAggSup.currentPageId = -1;
}
-static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
- int32_t size = taosArrayGetSize(pWins);
- for (int32_t i = 0; i < size; i++) {
- SResultWindowInfo* pWin = taosArrayGet(pWins, i);
- taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
- }
-}
-
static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
@@ -4349,30 +4464,34 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
- } else if (pOperator->status == OP_RES_TO_RETURN) {
+ }
+
+ {
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
if (pBInfo->pRes->info.rows > 0) {
printDataBlock(pBInfo->pRes, "semi session");
return pBInfo->pRes;
}
- // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
- if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) {
- pInfo->returnDelete = true;
+ doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
+ if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, "semi session");
return pInfo->pDelRes;
}
- if (pInfo->pUpdateRes->info.rows > 0) {
+ if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
+ pInfo->returnUpdate = false;
// process the rest of the data
- pOperator->status = OP_OPENED;
printDataBlock(pInfo->pUpdateRes, "semi session");
return pInfo->pUpdateRes;
}
- // semi interval operator clear disk buffer
- clearStreamSessionOperator(pInfo);
- pOperator->status = OP_EXEC_DONE;
- return NULL;
+
+ if (pOperator->status == OP_RES_TO_RETURN) {
+ // semi interval operator clear disk buffer
+ clearStreamSessionOperator(pInfo);
+ pOperator->status = OP_EXEC_DONE;
+ return NULL;
+ }
}
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
@@ -4383,6 +4502,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
clearSpecialDataBlock(pInfo->pUpdateRes);
+ pOperator->status = OP_RES_TO_RETURN;
break;
}
printDataBlock(pBlock, "semi session recv");
@@ -4393,12 +4513,15 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
removeSessionResults(pStUpdated, pWins);
taosArrayDestroy(pWins);
copyDataBlock(pInfo->pUpdateRes, pBlock);
+ pInfo->returnUpdate = true;
break;
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
// gap must be 0
- doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL, NULL);
- copyDataBlock(pInfo->pDelRes, pBlock);
- pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
+ SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
+ doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
+ copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
+ removeSessionResults(pStUpdated, pWins);
+ taosArrayDestroy(pWins);
break;
} else if (pBlock->info.type == STREAM_GET_ALL) {
getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
@@ -4411,18 +4534,15 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
}
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
- doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, false);
+ doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false);
maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
}
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;
- // restore the value
- pOperator->status = OP_RES_TO_RETURN;
- // semi operator
- // closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
- // getResWinForSession);
+
copyUpdateResult(pStUpdated, pUpdated);
+ removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
taosHashCleanup(pStUpdated);
finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
@@ -4436,16 +4556,15 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
return pBInfo->pRes;
}
- // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
- if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) {
- pInfo->returnDelete = true;
+ doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
+ if (pInfo->pDelRes->info.rows > 0) {
printDataBlock(pInfo->pDelRes, "semi session");
return pInfo->pDelRes;
}
- if (pInfo->pUpdateRes->info.rows > 0) {
+ if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
+ pInfo->returnUpdate = false;
// process the rest of the data
- pOperator->status = OP_OPENED;
printDataBlock(pInfo->pUpdateRes, "semi session");
return pInfo->pUpdateRes;
}
@@ -4669,7 +4788,7 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, u
if (pWinInfo->winInfo.win.skey > pTs[i]) {
if (pSeDeleted && pWinInfo->winInfo.isOutput) {
SWinKey res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId};
- taosHashPut(pSeDeleted, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey));
+ taosHashPut(pSeDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
pWinInfo->winInfo.isOutput = false;
}
pWinInfo->winInfo.win.skey = pTs[i];
@@ -4737,8 +4856,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, pSDataBlock->info.rows,
i, &allEqual, pStDeleted);
if (!allEqual) {
- appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, GROUPID_COLUMN_INDEX,
- &groupId);
+ uint64_t uid = 0;
+ appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey,
+ &uid, &groupId);
taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
continue;
@@ -4767,6 +4887,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pOperator->exprSupp;
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
+ int64_t maxTs = INT64_MIN;
if (pOperator->status == OP_RES_TO_RETURN) {
doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
if (pInfo->pDelRes->info.rows > 0) {
@@ -4799,6 +4920,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo);
copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
+ removeSessionResults(pSeUpdated, pWins);
taosArrayDestroy(pWins);
continue;
} else if (pBlock->info.type == STREAM_GET_ALL) {
@@ -4813,8 +4935,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
- pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
+ maxTs = TMAX(maxTs, pBlock->info.window.ekey);
}
+ pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
// restore the value
pOperator->status = OP_RES_TO_RETURN;
@@ -4913,7 +5036,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->info = pInfo;
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
- initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType);
+ initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index b7cd02befd..648ae5a538 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -311,6 +311,22 @@ static int32_t translateInOutStr(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return TSDB_CODE_SUCCESS;
}
+static int32_t translateMinMax(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+ if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+ return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+ }
+
+ uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+ if (!IS_TIMESTAMP_TYPE(paraType) && !IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
+ return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+ } else if (IS_NULL_TYPE(paraType)) {
+ paraType = TSDB_DATA_TYPE_BIGINT;
+ }
+
+ pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType};
+ return TSDB_CODE_SUCCESS;
+}
+
static int32_t translateTrimStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isLtrim) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@@ -698,7 +714,7 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
}
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
- if (!IS_NUMERIC_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+ if (!IS_NUMERIC_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
@@ -713,7 +729,7 @@ static int32_t translateSpreadImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (isPartial) {
- if (!IS_NUMERIC_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+ if (!IS_NUMERIC_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getSpreadInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
@@ -788,7 +804,7 @@ static int32_t translateElapsedImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t
}
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
- if (TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+ if (!IS_TIMESTAMP_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
@@ -1634,7 +1650,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType &&
- TSDB_DATA_TYPE_TIMESTAMP != colType) {
+ !IS_TIMESTAMP_TYPE(colType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
@@ -1660,7 +1676,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
}
uint8_t resType;
- if (IS_SIGNED_NUMERIC_TYPE(colType) || TSDB_DATA_TYPE_BOOL == colType || TSDB_DATA_TYPE_TIMESTAMP == colType) {
+ if (IS_SIGNED_NUMERIC_TYPE(colType) || IS_TIMESTAMP_TYPE(colType) || TSDB_DATA_TYPE_BOOL == colType) {
resType = TSDB_DATA_TYPE_BIGINT;
} else {
resType = TSDB_DATA_TYPE_DOUBLE;
@@ -1825,7 +1841,7 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
// param0
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
- if (!IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+ if (!IS_INTEGER_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
@@ -1878,7 +1894,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
- if ((!IS_STR_DATA_TYPE(para1Type) && !IS_INTEGER_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) ||
+ if ((!IS_STR_DATA_TYPE(para1Type) && !IS_INTEGER_TYPE(para1Type) && !IS_TIMESTAMP_TYPE(para1Type)) ||
!IS_INTEGER_TYPE(para2Type)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
@@ -1911,7 +1927,7 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
for (int32_t i = 0; i < 2; ++i) {
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type;
- if (!IS_STR_DATA_TYPE(paraType) && !IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+ if (!IS_STR_DATA_TYPE(paraType) && !IS_INTEGER_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
}
@@ -2060,7 +2076,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "min",
.type = FUNCTION_TYPE_MIN,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
- .translateFunc = translateInOutNum,
+ .translateFunc = translateMinMax,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getMinmaxFuncEnv,
.initFunc = minmaxFunctionSetup,
@@ -2075,7 +2091,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "max",
.type = FUNCTION_TYPE_MAX,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
- .translateFunc = translateInOutNum,
+ .translateFunc = translateMinMax,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getMinmaxFuncEnv,
.initFunc = minmaxFunctionSetup,
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 84d3a04807..7160541c13 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -1204,7 +1204,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
} else {
- if (IS_SIGNED_NUMERIC_TYPE(type)) {
+ if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type)) {
int64_t prev = 0;
GET_TYPED_DATA(prev, int64_t, type, &pBuf->v);
@@ -1216,7 +1216,6 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
-
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
uint64_t prev = 0;
GET_TYPED_DATA(prev, uint64_t, type, &pBuf->v);
@@ -1264,7 +1263,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows;
- if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
+ if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) {
int8_t* pData = (int8_t*)pCol->pData;
int8_t* val = (int8_t*)&pBuf->v;
@@ -1358,7 +1357,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
numOfElems += 1;
}
- } else if (type == TSDB_DATA_TYPE_BIGINT) {
+ } else if (type == TSDB_DATA_TYPE_BIGINT ||
+ type == TSDB_DATA_TYPE_TIMESTAMP) {
int64_t* pData = (int64_t*)pCol->pData;
int64_t* val = (int64_t*)&pBuf->v;
diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c
index 83bccbffb4..eb0b604d37 100644
--- a/source/libs/nodes/src/nodesCloneFuncs.c
+++ b/source/libs/nodes/src/nodesCloneFuncs.c
@@ -777,6 +777,7 @@ SNode* nodesCloneNode(const SNode* pNode) {
code = physiSessionCopy((const SSessionWinodwPhysiNode*)pNode, (SSessionWinodwPhysiNode*)pDst);
break;
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+ case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
code = physiPartitionCopy((const SPartitionPhysiNode*)pNode, (SPartitionPhysiNode*)pDst);
break;
default:
diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c
index 822bdec365..8976daadbd 100644
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@@ -265,6 +265,8 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiStreamStateWindow";
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return "PhysiPartition";
+ case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
+ return "PhysiStreamPartition";
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return "PhysiIndefRowsFunc";
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
@@ -4485,6 +4487,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
return physiStateWindowNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+ case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
return physiPartitionNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return physiIndefRowsFuncNodeToJson(pObj, pJson);
@@ -4632,6 +4635,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
return jsonToPhysiStateWindowNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+ case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
return jsonToPhysiPartitionNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return jsonToPhysiIndefRowsFuncNode(pJson, pObj);
diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c
index 2e23998aad..728e173ff8 100644
--- a/source/libs/nodes/src/nodesTraverseFuncs.c
+++ b/source/libs/nodes/src/nodesTraverseFuncs.c
@@ -537,7 +537,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
}
break;
}
- case QUERY_NODE_PHYSICAL_PLAN_PARTITION: {
+ case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+ case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: {
SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)pNode;
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index d13057a93e..61b2ad954f 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -322,6 +322,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SStreamStateWinodwPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return makeNode(type, sizeof(SPartitionPhysiNode));
+ case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
+ return makeNode(type, sizeof(SStreamPartitionPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
return makeNode(type, sizeof(SIndefRowsFuncPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
@@ -951,7 +953,8 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pPhyNode->pStateKey);
break;
}
- case QUERY_NODE_PHYSICAL_PLAN_PARTITION: {
+ case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+ case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: {
SPartitionPhysiNode* pPhyNode = (SPartitionPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pExprs);
diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index 4d0b0bbb25..f39fed4677 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -795,6 +795,20 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode*
return (SNode*)setOp;
}
+static void updateWalOptionsDefault(SDatabaseOptions* pOptions) {
+ if (!pOptions->walRetentionPeriodIsSet) {
+ pOptions->walRetentionPeriod =
+ pOptions->replica > 1 ? TSDB_REPS_DEF_DB_WAL_RET_PERIOD : TSDB_REP_DEF_DB_WAL_RET_PERIOD;
+ }
+ if (!pOptions->walRetentionSizeIsSet) {
+ pOptions->walRetentionSize = pOptions->replica > 1 ? TSDB_REPS_DEF_DB_WAL_RET_SIZE : TSDB_REP_DEF_DB_WAL_RET_SIZE;
+ }
+ if (!pOptions->walRollPeriodIsSet) {
+ pOptions->walRollPeriod =
+ pOptions->replica > 1 ? TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD : TSDB_REP_DEF_DB_WAL_ROLL_PERIOD;
+ }
+}
+
SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
CHECK_PARSER_STATUS(pCxt);
SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
@@ -819,9 +833,7 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
pOptions->numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
pOptions->singleStable = TSDB_DEFAULT_DB_SINGLE_STABLE;
pOptions->schemaless = TSDB_DEFAULT_DB_SCHEMALESS;
- pOptions->walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD;
- pOptions->walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE;
- pOptions->walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD;
+ updateWalOptionsDefault(pOptions);
pOptions->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
return (SNode*)pOptions;
}
@@ -859,78 +871,83 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) {
CHECK_PARSER_STATUS(pCxt);
+ SDatabaseOptions* pDbOptions = (SDatabaseOptions*)pOptions;
switch (type) {
case DB_OPTION_BUFFER:
- ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_CACHEMODEL:
- COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->cacheModelStr, (SToken*)pVal);
+ COPY_STRING_FORM_STR_TOKEN(pDbOptions->cacheModelStr, (SToken*)pVal);
break;
case DB_OPTION_CACHESIZE:
- ((SDatabaseOptions*)pOptions)->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_COMP:
- ((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_DAYS: {
SToken* pToken = pVal;
if (TK_NK_INTEGER == pToken->type) {
- ((SDatabaseOptions*)pOptions)->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440;
+ pDbOptions->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440;
} else {
- ((SDatabaseOptions*)pOptions)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken);
+ pDbOptions->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken);
}
break;
}
case DB_OPTION_FSYNC:
- ((SDatabaseOptions*)pOptions)->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_MAXROWS:
- ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_MINROWS:
- ((SDatabaseOptions*)pOptions)->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_KEEP:
- ((SDatabaseOptions*)pOptions)->pKeep = pVal;
+ pDbOptions->pKeep = pVal;
break;
case DB_OPTION_PAGES:
- ((SDatabaseOptions*)pOptions)->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_PAGESIZE:
- ((SDatabaseOptions*)pOptions)->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_PRECISION:
- COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->precisionStr, (SToken*)pVal);
+ COPY_STRING_FORM_STR_TOKEN(pDbOptions->precisionStr, (SToken*)pVal);
break;
case DB_OPTION_REPLICA:
- ((SDatabaseOptions*)pOptions)->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+ updateWalOptionsDefault(pDbOptions);
break;
case DB_OPTION_STRICT:
- COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->strictStr, (SToken*)pVal);
+ COPY_STRING_FORM_STR_TOKEN(pDbOptions->strictStr, (SToken*)pVal);
break;
case DB_OPTION_WAL:
- ((SDatabaseOptions*)pOptions)->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_VGROUPS:
- ((SDatabaseOptions*)pOptions)->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_SINGLE_STABLE:
- ((SDatabaseOptions*)pOptions)->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_RETENTIONS:
- ((SDatabaseOptions*)pOptions)->pRetentions = pVal;
+ pDbOptions->pRetentions = pVal;
break;
case DB_OPTION_WAL_RETENTION_PERIOD:
- ((SDatabaseOptions*)pOptions)->walRetentionPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->walRetentionPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->walRetentionPeriodIsSet = true;
break;
case DB_OPTION_WAL_RETENTION_SIZE:
- ((SDatabaseOptions*)pOptions)->walRetentionSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->walRetentionSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->walRetentionSizeIsSet = true;
break;
case DB_OPTION_WAL_ROLL_PERIOD:
- ((SDatabaseOptions*)pOptions)->walRollPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->walRollPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->walRollPeriodIsSet = true;
break;
case DB_OPTION_WAL_SEGMENT_SIZE:
- ((SDatabaseOptions*)pOptions)->walSegmentSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+ pDbOptions->walSegmentSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
default:
break;
@@ -1251,7 +1268,8 @@ SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
static bool needDbShowStmt(ENodeType type) {
return QUERY_NODE_SHOW_TABLES_STMT == type || QUERY_NODE_SHOW_STABLES_STMT == type ||
- QUERY_NODE_SHOW_VGROUPS_STMT == type;
+ QUERY_NODE_SHOW_VGROUPS_STMT == type || QUERY_NODE_SHOW_INDEXES_STMT == type ||
+ QUERY_NODE_SHOW_TAGS_STMT == type;
}
SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type) {
@@ -1264,7 +1282,7 @@ SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type) {
SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName,
EOperatorType tableCondType) {
CHECK_PARSER_STATUS(pCxt);
- if (needDbShowStmt(type) && NULL == pDbName && NULL == pCxt->pQueryCxt->db) {
+ if (needDbShowStmt(type) && NULL == pDbName) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "db not specified");
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
return NULL;
diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c
index 4dcd2bba5a..aa9ead2e27 100644
--- a/source/libs/parser/src/parAstParser.c
+++ b/source/libs/parser/src/parAstParser.c
@@ -374,8 +374,13 @@ static int32_t collectMetaKeyFromShowIndexes(SCollectMetaKeyCxt* pCxt, SShowStmt
}
static int32_t collectMetaKeyFromShowStables(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
- return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES,
- pCxt->pMetaCache);
+ int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES,
+ pCxt->pMetaCache);
+ if (TSDB_CODE_SUCCESS == code) {
+ code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser,
+ ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ, pCxt->pMetaCache);
+ }
+ return code;
}
static int32_t collectMetaKeyFromShowStreams(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
@@ -387,11 +392,11 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt*
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES,
pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
- if (NULL != pStmt->pDbName) {
- code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
- } else {
- code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, pCxt->pMetaCache);
- }
+ code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser,
+ ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ, pCxt->pMetaCache);
}
return code;
}
@@ -400,11 +405,7 @@ static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* p
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TAGS,
pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
- if (NULL != pStmt->pDbName) {
- code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
- } else {
- code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, pCxt->pMetaCache);
- }
+ code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
}
return code;
}
diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c
index d9a5761d99..9d73be7454 100644
--- a/source/libs/parser/src/parAuthenticator.c
+++ b/source/libs/parser/src/parAuthenticator.c
@@ -96,6 +96,10 @@ static int32_t authInsert(SAuthCxt* pCxt, SInsertStmt* pInsert) {
return code;
}
+static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) {
+ return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ);
+}
+
static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) {
return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_READ);
}
@@ -127,6 +131,9 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_SHOW_VNODES_STMT:
case QUERY_NODE_SHOW_SCORES_STMT:
return !pCxt->pParseCxt->enableSysInfo ? TSDB_CODE_PAR_PERMISSION_DENIED : TSDB_CODE_SUCCESS;
+ case QUERY_NODE_SHOW_TABLES_STMT:
+ case QUERY_NODE_SHOW_STABLES_STMT:
+ return authShowTables(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
return authShowCreateTable(pCxt, (SShowCreateTableStmt*)pStmt);
diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index 049d1ef545..bd8491a59b 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -786,6 +786,7 @@ static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTa
if (sname) pTbReq->ctb.name = strdup(sname);
pTbReq->ctb.pTag = (uint8_t*)pTag;
pTbReq->ctb.tagName = taosArrayDup(tagName);
+ pTbReq->ttl = TSDB_DEFAULT_TABLE_TTL;
pTbReq->commentLen = -1;
return;
@@ -1121,6 +1122,40 @@ static int32_t ignoreAutoCreateTableClause(SInsertParseContext* pCxt) {
return code;
}
+static int32_t parseTableOptions(SInsertParseContext* pCxt) {
+ do {
+ int32_t index = 0;
+ SToken sToken;
+ NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index);
+ if (TK_TTL == sToken.type) {
+ pCxt->pSql += index;
+ NEXT_TOKEN(pCxt->pSql, sToken);
+ if (TK_NK_INTEGER != sToken.type) {
+ return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z);
+ }
+ pCxt->createTblReq.ttl = taosStr2Int32(sToken.z, NULL, 10);
+ } else if (TK_COMMENT == sToken.type) {
+ pCxt->pSql += index;
+ NEXT_TOKEN(pCxt->pSql, sToken);
+ if (TK_NK_STRING != sToken.type) {
+ return buildSyntaxErrMsg(&pCxt->msg, "Invalid option comment", sToken.z);
+ }
+ if (sToken.n >= TSDB_TB_COMMENT_LEN) {
+ return buildSyntaxErrMsg(&pCxt->msg, "comment too long", sToken.z);
+ }
+ int32_t len = trimString(sToken.z, sToken.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN);
+ pCxt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len);
+ if (NULL == pCxt->createTblReq.comment) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+ pCxt->createTblReq.commentLen = len;
+ } else {
+ break;
+ }
+ } while (1);
+ return TSDB_CODE_SUCCESS;
+}
+
// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* tbFName) {
int32_t len = strlen(tbFName);
@@ -1172,7 +1207,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName*
return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z);
}
- return TSDB_CODE_SUCCESS;
+ return parseTableOptions(pCxt);
}
static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, bool* gotRow,
@@ -1672,7 +1707,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache
if (pContext->pStmtCb) {
context.pVgroupsHashObj = NULL;
context.pTableBlockHashObj = NULL;
- }
+ }
destroyInsertParseContext(&context);
return code;
}
@@ -1700,6 +1735,21 @@ static int32_t skipValuesClause(SInsertParseSyntaxCxt* pCxt) {
static int32_t skipTagsClause(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); }
+static int32_t skipTableOptions(SInsertParseSyntaxCxt* pCxt) {
+ do {
+ int32_t index = 0;
+ SToken sToken;
+ NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index);
+ if (TK_TTL == sToken.type || TK_COMMENT == sToken.type) {
+ pCxt->pSql += index;
+ NEXT_TOKEN(pCxt->pSql, sToken);
+ } else {
+ break;
+ }
+ } while (1);
+ return TSDB_CODE_SUCCESS;
+}
+
// pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...)
static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) {
SToken sToken;
@@ -1718,6 +1768,7 @@ static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) {
return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z);
}
CHECK_CODE(skipTagsClause(pCxt));
+ CHECK_CODE(skipTableOptions(pCxt));
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp
index 68c4ac3706..8455da12d7 100644
--- a/source/libs/parser/test/parInitialCTest.cpp
+++ b/source/libs/parser/test/parInitialCTest.cpp
@@ -111,9 +111,9 @@ TEST_F(ParserInitialCTest, createDatabase) {
expect.numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
expect.numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE;
expect.schemaless = TSDB_DEFAULT_DB_SCHEMALESS;
- expect.walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD;
- expect.walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE;
- expect.walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD;
+ expect.walRetentionPeriod = TSDB_REP_DEF_DB_WAL_RET_PERIOD;
+ expect.walRetentionSize = TSDB_REP_DEF_DB_WAL_RET_SIZE;
+ expect.walRollPeriod = TSDB_REP_DEF_DB_WAL_ROLL_PERIOD;
expect.walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
};
@@ -266,6 +266,14 @@ TEST_F(ParserInitialCTest, createDatabase) {
"DURATION 100m "
"KEEP 1440m,300h,400d ");
clearCreateDbReq();
+
+ setCreateDbReqFunc("wxy_db", 1);
+ setDbReplicaFunc(3);
+ setDbWalRetentionPeriod(TSDB_REPS_DEF_DB_WAL_RET_PERIOD);
+ setDbWalRetentionSize(TSDB_REPS_DEF_DB_WAL_RET_SIZE);
+ setDbWalRollPeriod(TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD);
+ run("CREATE DATABASE IF NOT EXISTS wxy_db REPLICA 3");
+ clearCreateDbReq();
}
TEST_F(ParserInitialCTest, createDatabaseSemanticCheck) {
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index c2f1d71b18..b160f45479 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -16,6 +16,7 @@
#include "filter.h"
#include "functionMgt.h"
#include "planInt.h"
+#include "tglobal.h"
#include "ttime.h"
#define OPTIMIZE_FLAG_MASK(n) (1 << n)
@@ -2410,7 +2411,7 @@ static const SOptimizeRule optimizeRuleSet[] = {
static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule));
static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
- if (0 == (qDebugFlag & DEBUG_DEBUG)) {
+ if (!tsQueryPlannerTrace) {
return;
}
char* pStr = NULL;
diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c
index cafae18dbe..0cbb833a4d 100644
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@@ -1324,7 +1324,8 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) {
SPartitionPhysiNode* pPart =
- (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION);
+ (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode,
+ pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION : QUERY_NODE_PHYSICAL_PLAN_PARTITION);
if (NULL == pPart) {
return TSDB_CODE_OUT_OF_MEMORY;
}
diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index c582994b7c..beb938b161 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -1427,7 +1427,7 @@ static const SSplitRule splitRuleSet[] = {
static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule));
static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
- if (0 == (qDebugFlag & DEBUG_DEBUG)) {
+ if (!tsQueryPlannerTrace) {
return;
}
char* pStr = NULL;
diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c
index c1296982e0..baa1d1074c 100644
--- a/source/libs/planner/src/planner.c
+++ b/source/libs/planner/src/planner.c
@@ -17,9 +17,10 @@
#include "planInt.h"
#include "scalar.h"
+#include "tglobal.h"
static void dumpQueryPlan(SQueryPlan* pPlan) {
- if (0 == (qDebugFlag & DEBUG_DEBUG)) {
+ if (!tsQueryPlannerTrace) {
return;
}
char* pStr = NULL;
diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c
index d053662bd3..332f7ad2fd 100644
--- a/source/libs/stream/src/streamUpdate.c
+++ b/source/libs/stream/src/streamUpdate.c
@@ -170,8 +170,17 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) {
if (ts < maxTs - pInfo->watermark) {
// this window has been closed.
if (pInfo->pCloseWinSBF) {
- return tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY));
+ res = tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY));
+ if (res == TSDB_CODE_SUCCESS) {
+ return false;
+ } else {
+ qDebug("===stream===Update close window sbf. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
+ maxTs, *pMapMaxTs, ts);
+ return true;
+ }
}
+ qDebug("===stream===Update close window. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
+ maxTs, *pMapMaxTs, ts);
return true;
}
@@ -193,7 +202,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) {
}
if (ts < pInfo->minTS) {
- qDebug("===stream===Update. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
+ qDebug("===stream===Update min ts. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
maxTs, *pMapMaxTs, ts);
return true;
} else if (res == TSDB_CODE_SUCCESS) {
diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c
index 5489a107e7..0be3392a9a 100644
--- a/source/libs/sync/src/syncSnapshot.c
+++ b/source/libs/sync/src/syncSnapshot.c
@@ -583,7 +583,7 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
&(pReceiver->snapshot));
if (code != 0) {
syncNodeErrorLog(pReceiver->pSyncNode, "snapshot stop writer true error");
- ASSERT(0);
+ // ASSERT(0);
return -1;
}
pReceiver->pWriter = NULL;
diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c
index af15c377fb..c3c8131cbb 100644
--- a/source/libs/sync/src/syncTimeout.c
+++ b/source/libs/sync/src/syncTimeout.c
@@ -91,16 +91,16 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) {
} else if (pMsg->timeoutType == SYNC_TIMEOUT_ELECTION) {
if (atomic_load_64(&ths->electTimerLogicClockUser) <= pMsg->logicClock) {
++(ths->electTimerCounter);
- sInfo("vgId:%d, sync timeout, type:election count:%d, electTimerLogicClockUser:%ld", ths->vgId,
- ths->electTimerCounter, ths->electTimerLogicClockUser);
+ sTrace("vgId:%d, sync timer, type:election count:%d, electTimerLogicClockUser:%ld", ths->vgId,
+ ths->electTimerCounter, ths->electTimerLogicClockUser);
syncNodeElect(ths);
}
} else if (pMsg->timeoutType == SYNC_TIMEOUT_HEARTBEAT) {
if (atomic_load_64(&ths->heartbeatTimerLogicClockUser) <= pMsg->logicClock) {
++(ths->heartbeatTimerCounter);
- sInfo("vgId:%d, sync timeout, type:replicate count:%d, heartbeatTimerLogicClockUser:%ld", ths->vgId,
- ths->heartbeatTimerCounter, ths->heartbeatTimerLogicClockUser);
+ sTrace("vgId:%d, sync timer, type:replicate count:%d, heartbeatTimerLogicClockUser:%ld", ths->vgId,
+ ths->heartbeatTimerCounter, ths->heartbeatTimerLogicClockUser);
syncNodeReplicate(ths, true);
}
} else {
diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c
index 30aaa01dae..3dfb1458ad 100644
--- a/source/os/src/osDir.c
+++ b/source/os/src/osDir.c
@@ -160,6 +160,66 @@ int32_t taosMulMkDir(const char *dirname) {
return code;
}
+int32_t taosMulModeMkDir(const char *dirname, int mode) {
+ if (dirname == NULL) return -1;
+ char temp[1024];
+ char *pos = temp;
+ int32_t code = 0;
+#ifdef WINDOWS
+ taosRealPath(dirname, temp, sizeof(temp));
+ if (temp[1] == ':') pos += 3;
+#else
+ strcpy(temp, dirname);
+#endif
+
+ if (taosDirExist(temp)) {
+ chmod(temp, mode);
+ return code;
+ }
+
+ if (strncmp(temp, TD_DIRSEP, 1) == 0) {
+ pos += 1;
+ } else if (strncmp(temp, "." TD_DIRSEP, 2) == 0) {
+ pos += 2;
+ }
+
+ for (; *pos != '\0'; pos++) {
+ if (*pos == TD_DIRSEP[0]) {
+ *pos = '\0';
+#ifdef WINDOWS
+ code = _mkdir(temp, mode);
+#else
+ code = mkdir(temp, mode);
+#endif
+ if (code < 0 && errno != EEXIST) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return code;
+ }
+ *pos = TD_DIRSEP[0];
+ }
+ }
+
+ if (*(pos - 1) != TD_DIRSEP[0]) {
+#ifdef WINDOWS
+ code = _mkdir(temp, mode);
+#else
+ code = mkdir(temp, mode);
+#endif
+ if (code < 0 && errno != EEXIST) {
+ terrno = TAOS_SYSTEM_ERROR(errno);
+ return code;
+ }
+ }
+
+ if (code < 0 && errno == EEXIST) {
+ chmod(temp, mode);
+ return 0;
+ }
+
+ chmod(temp, mode);
+ return code;
+}
+
void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
TdDirPtr pDir = taosOpenDir(dirname);
if (pDir == NULL) return;
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index b382257972..044cdc86b4 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -622,7 +622,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REGEX_MATCH, "Rsma regex match")
//index
TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding")
-TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Invalid index file")
+TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_INVALID_FILE, "Index file is invalid")
//tmq
TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message")
diff --git a/tests/pytest/tools/taosdumpTest2.py b/tests/pytest/tools/taosdumpTest2.py
index a8117ec04c..7071adb3a9 100644
--- a/tests/pytest/tools/taosdumpTest2.py
+++ b/tests/pytest/tools/taosdumpTest2.py
@@ -11,15 +11,19 @@
# -*- coding: utf-8 -*-
+from logging.config import dictConfig
import sys
import os
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
+import string
+import random
class TDTestCase:
+
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
@@ -47,12 +51,19 @@ class TDTestCase:
return ""
return paths[0]
+ def generateString(self, length):
+ chars = string.ascii_uppercase + string.ascii_lowercase
+ v = ""
+ for i in range(length):
+ v += random.choice(chars)
+ return v
+
def run(self):
if not os.path.exists("./taosdumptest/tmp"):
os.makedirs("./taosdumptest/tmp")
else:
- os.system("rm -rf ./taosdumptest/tmp")
- os.makedirs("./taosdumptest/tmp")
+ print("directory exists")
+ os.system("rm -rf ./taosdumptest/tmp/*")
tdSql.prepare()
@@ -76,17 +87,19 @@ class TDTestCase:
tdLog.info("taosdump found in %s" % binPath)
os.system("rm ./taosdumptest/tmp/*.sql")
+ os.system("rm ./taosdumptest/tmp/*.avro*")
+ os.system("rm -rf ./taosdumptest/taosdump.*")
os.system(
- "%s --databases db -o ./taosdumptest/tmp -B 32766 -L 1048576" %
+ "%s --databases db -o ./taosdumptest/tmp " %
binPath)
tdSql.execute("drop database db")
- tdSql.query("select * from information_schema.ins_databases")
+ tdSql.query("show databases")
tdSql.checkRows(2)
- os.system("%s -i ./taosdumptest/tmp" % binPath)
+ os.system("%s -i ./taosdumptest/tmp -y" % binPath)
- tdSql.query("select * from information_schema.ins_databases")
+ tdSql.query("show databases")
tdSql.checkRows(3)
tdSql.checkData(2, 0, 'db')
@@ -105,17 +118,17 @@ class TDTestCase:
"create table stb(ts timestamp, c1 binary(16374), c2 binary(16374), c3 binary(16374)) tags(t1 nchar(256))")
tdSql.execute(
"insert into t1 using stb tags('t1') values(now, '%s', '%s', '%s')" %
- ("16374",
- "16374",
- "16374"))
+ (self.generateString(16374),
+ self.generateString(16374),
+ self.generateString(16374)))
-# sys.exit(0)
os.system("rm ./taosdumptest/tmp/*.sql")
os.system("rm ./taosdumptest/tmp/*.avro*")
+ os.system("rm -rf ./taosdumptest/tmp/taosdump.*")
os.system("%s -D test -o ./taosdumptest/tmp -y" % binPath)
tdSql.execute("drop database test")
- tdSql.query("select * from information_schema.ins_databases")
+ tdSql.query("show databases")
tdSql.checkRows(3)
os.system("%s -i ./taosdumptest/tmp -y" % binPath)
diff --git a/tests/script/tsim/stable/alter_comment.sim b/tests/script/tsim/stable/alter_comment.sim
index beb049985c..7c2d6edfcb 100644
--- a/tests/script/tsim/stable/alter_comment.sim
+++ b/tests/script/tsim/stable/alter_comment.sim
@@ -95,7 +95,7 @@ sql_error alter table db.stb add tag t1 int
sql_error alter table db.stb add tag t2 int
sql_error alter table db.stb add tag t3 int
sql alter table db.stb add tag t4 bigint
-sql alter table db.stb add tag c1 int
+sql alter table db.stb add tag c1 int
sql alter table db.stb add tag t5 binary(12)
sql select * from information_schema.ins_stables where db_name = 'db'
diff --git a/tests/script/tsim/stable/alter_count.sim b/tests/script/tsim/stable/alter_count.sim
index 83ea4b14fa..4a2aeca029 100644
--- a/tests/script/tsim/stable/alter_count.sim
+++ b/tests/script/tsim/stable/alter_count.sim
@@ -5,8 +5,8 @@ print ========= start dnode1 as master
system sh/exec.sh -n dnode1 -s start
sql connect
-print ======== step1
-sql create database d1 replica 1 duration 7 keep 50
+print ======== step1
+sql create database d1 replica 1 duration 7 keep 50
sql use d1
sql create table tb (ts timestamp, a int)
sql insert into tb values(now-28d, -28)
@@ -83,7 +83,7 @@ if $data00 != 3 then
endi
print ======== step8
-# sql alter table tb(ts timestamp, a int, b smallint, c tinyint, d int, e bigint, f float, g double, h binary(10) )
+# sql alter table tb(ts timestamp, a int, b smallint, c tinyint, d int, e bigint, f float, g double, h binary(10) )
sql alter table tb add column h binary(10)
sql insert into tb values(now-7d, -7, 18, 0, 0, 0, 0, 0, '0')
sql insert into tb values(now-6d, -6, 19, 1, 1, 1, 1, 1, '1')
@@ -260,4 +260,4 @@ if $data00 != 31 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/alter_import.sim b/tests/script/tsim/stable/alter_import.sim
index b968eb6a12..7431ea698a 100644
--- a/tests/script/tsim/stable/alter_import.sim
+++ b/tests/script/tsim/stable/alter_import.sim
@@ -5,7 +5,7 @@ print ========= start dnode1 as master
system sh/exec.sh -n dnode1 -s start
sql connect
-print ======== step1
+print ======== step1
sql create database d1 replica 1 duration 7 keep 50
sql use d1
sql create table tb (ts timestamp, a int)
@@ -42,4 +42,4 @@ if $data00 != 6 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/alter_insert1.sim b/tests/script/tsim/stable/alter_insert1.sim
index bcea0b48c4..0e5617e92d 100644
--- a/tests/script/tsim/stable/alter_insert1.sim
+++ b/tests/script/tsim/stable/alter_insert1.sim
@@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
-print ======== step1
+print ======== step1
sql create database d3
sql use d3
sql create table tb (ts timestamp, a int)
@@ -1137,4 +1137,4 @@ if $data79 != null then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/alter_insert2.sim b/tests/script/tsim/stable/alter_insert2.sim
index faae24d32a..a6046f3dda 100644
--- a/tests/script/tsim/stable/alter_insert2.sim
+++ b/tests/script/tsim/stable/alter_insert2.sim
@@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
-print ======== step1
+print ======== step1
sql create database d4
sql use d4
sql create table tb (ts timestamp, a int, b smallint, c tinyint, d int, e bigint, f float, g double, h binary(10))
@@ -662,4 +662,4 @@ if $data62 != null then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/alter_metrics.sim b/tests/script/tsim/stable/alter_metrics.sim
index e32250de13..203f41e18b 100644
--- a/tests/script/tsim/stable/alter_metrics.sim
+++ b/tests/script/tsim/stable/alter_metrics.sim
@@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
-print ======== step1
+print ======== step1
sql create database d2
sql use d2
sql create table mt (ts timestamp, a int) TAGS (t int)
@@ -757,8 +757,8 @@ endi
print ======= over
sql drop database d2
sql select * from information_schema.ins_databases
-if $rows != 2 then
+if $rows != 2 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/column_add.sim b/tests/script/tsim/stable/column_add.sim
index c0f3b4f490..05189f6c7d 100644
--- a/tests/script/tsim/stable/column_add.sim
+++ b/tests/script/tsim/stable/column_add.sim
@@ -116,7 +116,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 2 then
return -1
-endi
+endi
if $data[0][3] != NULL then
return -1
endi
@@ -153,7 +153,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 2 then
return -1
-endi
+endi
if $data[0][3] != NULL then
return -1
endi
@@ -299,4 +299,4 @@ if $rows != 10 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/column_modify.sim b/tests/script/tsim/stable/column_modify.sim
index e2752ccf95..43284ba829 100644
--- a/tests/script/tsim/stable/column_modify.sim
+++ b/tests/script/tsim/stable/column_modify.sim
@@ -31,7 +31,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 1234 then
return -1
-endi
+endi
if $data[0][3] != 101 then
return -1
endi
@@ -92,7 +92,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 1234 then
return -1
-endi
+endi
if $data[0][3] != 101 then
return -1
endi
@@ -106,4 +106,4 @@ if $data[1][3] != 101 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/disk.sim b/tests/script/tsim/stable/disk.sim
index e0e51b2625..8edd0a845e 100644
--- a/tests/script/tsim/stable/disk.sim
+++ b/tests/script/tsim/stable/disk.sim
@@ -188,4 +188,4 @@ if $rows != 2 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/refcount.sim b/tests/script/tsim/stable/refcount.sim
index a83c0ca53f..35d8767efd 100644
--- a/tests/script/tsim/stable/refcount.sim
+++ b/tests/script/tsim/stable/refcount.sim
@@ -123,4 +123,4 @@ if $rows != 2 then
endi
print =============== step6
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/tag_add.sim b/tests/script/tsim/stable/tag_add.sim
index 7ee9aee974..4f5f0e7452 100644
--- a/tests/script/tsim/stable/tag_add.sim
+++ b/tests/script/tsim/stable/tag_add.sim
@@ -139,7 +139,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 2 then
return -1
-endi
+endi
if $data[0][3] != 101 then
return -1
endi
@@ -170,7 +170,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 2 then
return -1
-endi
+endi
if $data[0][3] != 101 then
return -1
endi
@@ -190,4 +190,4 @@ if $rows != 7 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/tag_drop.sim b/tests/script/tsim/stable/tag_drop.sim
index 7902358817..b457bf195b 100644
--- a/tests/script/tsim/stable/tag_drop.sim
+++ b/tests/script/tsim/stable/tag_drop.sim
@@ -165,7 +165,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 2 then
return -1
-endi
+endi
if $data[0][3] != 101 then
return -1
endi
@@ -196,7 +196,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 2 then
return -1
-endi
+endi
if $data[0][3] != 201 then
return -1
endi
@@ -229,7 +229,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 2 then
return -1
-endi
+endi
if $data[0][3] != 201 then
return -1
endi
@@ -261,7 +261,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 2 then
return -1
-endi
+endi
if $data[0][3] != 301 then
return -1
endi
@@ -323,7 +323,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 2 then
return -1
-endi
+endi
if $data[0][3] != 302 then
return -1
endi
@@ -334,4 +334,4 @@ if $data[0][5] != 304 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim
index f44142fbbf..de2a87d6c4 100644
--- a/tests/script/tsim/stable/tag_filter.sim
+++ b/tests/script/tsim/stable/tag_filter.sim
@@ -27,47 +27,47 @@ sql create table db.ctb6 using db.stb tags(6, "102")
sql insert into db.ctb6 values(now, 6, "2")
sql select * from db.stb where t1 = 1
-if $rows != 1 then
+if $rows != 1 then
return -1
endi
-sql select * from db.stb where t1 < 1
-if $rows != 0 then
+sql select * from db.stb where t1 < 1
+if $rows != 0 then
return -=1
endi
-sql select * from db.stb where t1 < 2
-if $rows != 1 then
+sql select * from db.stb where t1 < 2
+if $rows != 1 then
return -1
endi
-sql select * from db.stb where t1 <= 2
-if $rows != 2 then
+sql select * from db.stb where t1 <= 2
+if $rows != 2 then
return -1
endi
-sql select * from db.stb where t1 >= 1
-if $rows != 6 then
+sql select * from db.stb where t1 >= 1
+if $rows != 6 then
return -1
endi
-sql select * from db.stb where t1 > 1
-if $rows != 5 then
+sql select * from db.stb where t1 > 1
+if $rows != 5 then
return -1
endi
-sql select * from db.stb where t1 between 1 and 1
-if $rows != 1 then
+sql select * from db.stb where t1 between 1 and 1
+if $rows != 1 then
return -1
endi
-sql select * from db.stb where t1 between 1 and 6
-if $rows != 6 then
+sql select * from db.stb where t1 between 1 and 6
+if $rows != 6 then
return -1
endi
-sql select * from db.stb where t1 between 1 and 7
-if $rows != 6 then
+sql select * from db.stb where t1 between 1 and 7
+if $rows != 6 then
return -1
endi
@@ -88,25 +88,25 @@ sql insert into db.ctbBin2 values(now, 3, "2")
sql create table db.ctbBin3 using db.stbBin tags("d")
sql insert into db.ctbBin3 values(now, 4, "2")
-sql select * from db.stbBin where t1 = "a"
-if $rows != 1 then
+sql select * from db.stbBin where t1 = "a"
+if $rows != 1 then
return -1
endi
-sql select * from db.stbBin where t1 < "a"
-if $rows != 0 then
+sql select * from db.stbBin where t1 < "a"
+if $rows != 0 then
return -=1
endi
-sql select * from db.stbBin where t1 < "b"
-if $rows != 1 then
+sql select * from db.stbBin where t1 < "b"
+if $rows != 1 then
return -1
endi
-sql select * from db.stbBin where t1 between "a" and "e"
-if $rows != 4 then
+sql select * from db.stbBin where t1 between "a" and "e"
+if $rows != 4 then
return -1
endi
@@ -127,25 +127,25 @@ sql insert into db.ctbNc2 values(now, 3, "2")
sql create table db.ctbNc3 using db.stbNc tags("d")
sql insert into db.ctbNc3 values(now, 4, "2")
-sql select * from db.stbNc where t1 = "a"
-if $rows != 1 then
+sql select * from db.stbNc where t1 = "a"
+if $rows != 1 then
return -1
endi
-sql select * from db.stbNc where t1 < "a"
-if $rows != 0 then
+sql select * from db.stbNc where t1 < "a"
+if $rows != 0 then
return -=1
endi
-sql select * from db.stbNc where t1 < "b"
-if $rows != 1 then
+sql select * from db.stbNc where t1 < "b"
+if $rows != 1 then
return -1
endi
-sql select * from db.stbNc where t1 between "a" and "e"
-if $rows != 4 then
+sql select * from db.stbNc where t1 between "a" and "e"
+if $rows != 4 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/tag_modify.sim b/tests/script/tsim/stable/tag_modify.sim
index 909ed79359..53e7227d1b 100644
--- a/tests/script/tsim/stable/tag_modify.sim
+++ b/tests/script/tsim/stable/tag_modify.sim
@@ -28,7 +28,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 1234 then
return -1
-endi
+endi
if $data[0][3] != 101 then
return -1
endi
@@ -55,7 +55,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 1234 then
return -1
-endi
+endi
if $data[0][3] != 101 then
return -1
endi
@@ -120,4 +120,4 @@ if $data[4][2] != 5 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stable/tag_rename.sim b/tests/script/tsim/stable/tag_rename.sim
index 5bdfa24990..c85ed183de 100644
--- a/tests/script/tsim/stable/tag_rename.sim
+++ b/tests/script/tsim/stable/tag_rename.sim
@@ -28,7 +28,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 1234 then
return -1
-endi
+endi
if $data[0][3] != 101 then
return -1
endi
@@ -52,7 +52,7 @@ if $data[0][1] != 1 then
endi
if $data[0][2] != 1234 then
return -1
-endi
+endi
if $data[0][3] != 101 then
return -1
endi
@@ -117,4 +117,4 @@ if $data[4][2] != 4 then
return -1
endi
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim
index 5392979c0a..d9777d5133 100644
--- a/tests/script/tsim/stream/basic1.sim
+++ b/tests/script/tsim/stream/basic1.sim
@@ -462,10 +462,10 @@ if $data25 != 3 then
return -1
endi
-sql create database test2 vgroups 1
-sql select * from information_schema.ins_databases
+sql create database test2 vgroups 1;
+sql select * from information_schema.ins_databases;
-sql use test2
+sql use test2;
sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
diff --git a/tests/script/tsim/stream/partitionbyColumn0.sim b/tests/script/tsim/stream/partitionbyColumn0.sim
new file mode 100644
index 0000000000..d91d4b7bf0
--- /dev/null
+++ b/tests/script/tsim/stream/partitionbyColumn0.sim
@@ -0,0 +1,570 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop0:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop0
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop0
+endi
+
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+
+loop1:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+
+$loop_count = 0
+
+loop2:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop2
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop2
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791213001,2,2,3,1.0);
+sql insert into t1 values(1648791213002,2,2,3,1.0);
+sql insert into t1 values(1648791213002,1,2,3,1.0);
+
+$loop_count = 0
+
+loop3:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop3
+endi
+
+if $data12 != 2 then
+ print =====data12=$data12
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop4
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop4
+endi
+
+if $data11 != 1 then
+ print =====data11=$data11
+ goto loop4
+endi
+
+if $data12 != 2 then
+ print =====data12=$data12
+ goto loop4
+endi
+
+if $data21 != 2 then
+ print =====data21=$data21
+ goto loop4
+endi
+
+if $data22 != 1 then
+ print =====data22=$data22
+ goto loop4
+endi
+
+if $data31 != 1 then
+ print =====data31=$data31
+ goto loop4
+endi
+
+if $data32 != 2 then
+ print =====data32=$data32
+ goto loop4
+endi
+
+if $data41 != 1 then
+ print =====data41=$data41
+ goto loop4
+endi
+
+if $data42 != 3 then
+ print =====data42=$data42
+ goto loop4
+endi
+
+sql drop stream if exists streams1;
+sql drop database if exists test1;
+sql create database test1 vgroups 1;
+sql use test1;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams1 trigger at_once into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,1,2,1,1.0);
+sql insert into t1 values(1648791213001,2,1,2,2.0);
+sql insert into t1 values(1648791213001,1,2,3,2.0);
+
+$loop_count = 0
+
+loop5:
+sleep 100
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+sql insert into t1 values(1648791223000,1,2,4,2.0);
+sql insert into t1 values(1648791223001,1,2,5,2.0);
+sql insert into t1 values(1648791223002,1,2,5,2.0);
+sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
+
+$loop_count = 0
+
+loop6:
+sleep 100
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 6 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 1 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+if $data21 != 1 then
+ print =====data21=$data21
+ goto loop6
+endi
+
+if $data22 != 7 then
+ print =====data22=$data22
+ goto loop6
+endi
+
+if $data31 != 2 then
+ print =====data31=$data31
+ goto loop6
+endi
+
+if $data32 != 5 then
+ print =====data32=$data32
+ goto loop6
+endi
+
+sql drop stream if exists streams2;
+sql drop database if exists test2;
+sql create database test2 vgroups 4;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop7
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop7
+endi
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+sql insert into t2 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+
+loop8:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop8
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop8
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+
+$loop_count = 0
+
+loop9:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop9
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop9
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791213001,2,2,3,1.0);
+sql insert into t1 values(1648791213002,2,2,3,1.0);
+sql insert into t1 values(1648791213002,1,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213001,2,2,3,1.0);
+sql insert into t2 values(1648791213002,2,2,3,1.0);
+sql insert into t2 values(1648791213002,1,2,3,1.0);
+
+$loop_count = 0
+
+loop10:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop10
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop10
+endi
+
+if $data11 != 4 thenloop4
+ print =====data11=$data11
+ goto loop10
+endi
+
+if $data12 != 2 then
+ print =====data12=$data12
+ goto loop10
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+$loop_count = 0
+
+loop11:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop11
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop11
+endi
+
+if $data11 != 4 then
+ print =====data11=$data11
+ goto loop11
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop11
+endi
+
+if $data21 != 2 then
+ print =====data21=$data21
+ goto loop11
+endi
+
+if $data22 != 2 then
+ print =====data22=$data22
+ goto loop11
+endi
+
+if $data31 != 2 then
+ print =====data31=$data31
+ goto loop11
+endi
+
+if $data32 != 3 then
+ print =====data32=$data32
+ goto loop11
+endi
+
+if $data41 != 4 then
+ print =====data41=$data41
+ goto loop11
+endi
+
+if $data42 != 1 then
+ print =====data42=$data42
+ goto loop11
+endi
+
+sql drop stream if exists streams4;
+sql drop database if exists test4;
+sql create database test4 vgroups 4;
+sql use test4;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create table t3 using st tags(2,2,2);
+sql create table t4 using st tags(2,2,2);
+sql create stream streams4 trigger at_once into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s);
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+sql insert into t3 values(1648791213000,2,2,3,1.0);
+sql insert into t4 values(1648791213000,2,2,3,1.0);
+sql insert into t4 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+
+loop13:
+sleep 100
+sql select * from test.streamt4 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop13
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop13
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop13
+endi
+
+if $data11 != 3 then
+ print =====data11=$data11
+ goto loop13
+endi
+
+if $data12 != 2 then
+ print =====data12=$data12
+ goto loop13
+endi
+
+sql insert into t4 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791233000,2,2,3,1.0);
+
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+
+loop14:
+sleep 100
+sql select * from test.streamt4 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+ return -1
+endi
+
+if $rows != 3 then
+ print =====rows=$rows
+ goto loop14
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop14
+endi
+
+if $data11 != 3 then
+ print =====data11=$data11
+ goto loop14
+endi
+
+if $data21 != 1 then
+ print =====data21=$data21
+ goto loop14
+endi
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/partitionbyColumn1.sim b/tests/script/tsim/stream/partitionbyColumn1.sim
new file mode 100644
index 0000000000..7f5c53ebe3
--- /dev/null
+++ b/tests/script/tsim/stream/partitionbyColumn1.sim
@@ -0,0 +1,546 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a session(ts, 5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop0:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop0
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop0
+endi
+
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+
+loop1:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+
+loop2:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop2
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop2
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791213001,2,2,3,1.0);
+sql insert into t1 values(1648791213002,2,2,3,1.0);
+sql insert into t1 values(1648791213002,1,2,3,1.0);
+
+loop3:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+if $data11 != 1 then
+ print =====data11=$data11
+ goto loop3
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+loop4:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop4
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop4
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop4
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop4
+endi
+
+if $data21 != 2 then
+ print =====data21=$data21
+ goto loop4
+endi
+
+if $data22 != 1 then
+ print =====data22=$data22
+ goto loop4
+endi
+
+if $data31 != 1 then
+ print =====data31=$data31
+ goto loop4
+endi
+
+if $data32 != 2 then
+ print =====data32=$data32
+ goto loop4
+endi
+
+if $data41 != 1 then
+ print =====data41=$data41
+ goto loop4
+endi
+
+if $data42 != 3 then
+ print =====data42=$data42
+ goto loop4
+endi
+
+sql drop database if exists test1;
+sql create database test1 vgroups 1;
+sql use test1;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams1 trigger at_once into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b session(ts, 5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,1,2,1,1.0);
+sql insert into t1 values(1648791213001,2,1,2,2.0);
+sql insert into t1 values(1648791213001,1,2,3,2.0);
+
+$loop_count = 0
+
+loop5:
+sleep 300
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+sql insert into t1 values(1648791223000,1,2,4,2.0);
+sql insert into t1 values(1648791223001,1,2,5,2.0);
+sql insert into t1 values(1648791223002,1,2,5,2.0);
+sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
+
+loop6:
+sleep 300
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 1 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 6 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+if $data21 != 2 then
+ print =====data21=$data21
+ goto loop6
+endi
+
+if $data22 != 5 then
+ print =====data22=$data22
+ goto loop6
+endi
+
+if $data31 != 1 then
+ print =====data31=$data31
+ goto loop6
+endi
+
+if $data32 != 7 then
+ print =====data32=$data32
+ goto loop6
+endi
+
+sql drop database if exists test2;
+sql create database test2 vgroups 4;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop7
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop7
+endi
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+sql insert into t2 values(1648791213000,1,2,3,1.0);
+
+loop8:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop8
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop8
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+loop9:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop9
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop9
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791213001,2,2,3,1.0);
+sql insert into t1 values(1648791213002,2,2,3,1.0);
+sql insert into t1 values(1648791213002,1,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213001,2,2,3,1.0);
+sql insert into t2 values(1648791213002,2,2,3,1.0);
+sql insert into t2 values(1648791213002,1,2,3,1.0);
+
+loop10:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 4 then
+ print =====data01=$data01
+ goto loop10
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop10
+endi
+
+if $data11 != 2 thenloop4
+ print =====data11=$data11
+ goto loop3
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop10
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+loop11:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop11
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop11
+endi
+
+if $data11 != 4 then
+ print =====data11=$data11
+ goto loop11
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop11
+endi
+
+if $data21 != 4 then
+ print =====data21=$data21
+ goto loop11
+endi
+
+if $data22 != 1 then
+ print =====data22=$data22
+ goto loop11
+endi
+
+if $data31 != 2 then
+ print =====data31=$data31
+ goto loop11
+endi
+
+if $data32 != 2 then
+ print =====data32=$data32
+ goto loop11
+endi
+
+if $data41 != 2 then
+ print =====data41=$data41
+ goto loop11
+endi
+
+if $data42 != 3 then
+ print =====data42=$data42
+ goto loop11
+endi
+
+sql drop database if exists test4;
+sql create database test4 vgroups 4;
+sql use test4;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create table t3 using st tags(2,2,2);
+sql create table t4 using st tags(2,2,2);
+sql create stream streams4 trigger at_once into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s);
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+sql insert into t3 values(1648791213000,2,2,3,1.0);
+sql insert into t4 values(1648791213000,2,2,3,1.0);
+sql insert into t4 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+
+loop13:
+sleep 300
+sql select * from test.streamt4 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop14
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop13
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop13
+endi
+
+if $data11 != 3 then
+ print =====data11=$data11
+ goto loop11
+endi
+
+if $data12 != 2 then
+ print =====data12=$data12
+ goto loop11
+endi
+
+sql insert into t4 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791233000,2,2,3,1.0);
+
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+
+loop14:
+sleep 300
+sql select * from test.streamt4 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 3 then
+ print =====rows=$rows
+ goto loop14
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop14
+endi
+
+if $data11 != 3 then
+ print =====data11=$data11
+ goto loop14
+endi
+
+if $data21 != 1 then
+ print =====data21=$data21
+ goto loop14
+endi
+
+system sh/stop_dnodes.sh
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/partitionbyColumn2.sim b/tests/script/tsim/stream/partitionbyColumn2.sim
new file mode 100644
index 0000000000..3d9acbcac5
--- /dev/null
+++ b/tests/script/tsim/stream/partitionbyColumn2.sim
@@ -0,0 +1,269 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a state_window(b);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop0:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop0
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop0
+endi
+
+
+sql insert into t1 values(1648791213000,1,1,3,1.0);
+
+loop1:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,2,1,3,1.0);
+
+loop2:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop2
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop2
+endi
+
+sql insert into t1 values(1648791213000,2,1,3,1.0);
+sql insert into t1 values(1648791213001,2,1,3,1.0);
+sql insert into t1 values(1648791213002,2,1,3,1.0);
+sql insert into t1 values(1648791213002,1,1,3,1.0);
+
+loop3:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+if $data11 != 1 then
+ print =====data11=$data11
+ goto loop3
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,1,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+loop4:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop4
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop4
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop4
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop4
+endi
+
+if $data21 != 2 then
+ print =====data21=$data21
+ goto loop4
+endi
+
+if $data22 != 1 then
+ print =====data22=$data22
+ goto loop4
+endi
+
+if $data31 != 1 then
+ print =====data31=$data31
+ goto loop4
+endi
+
+if $data32 != 2 then
+ print =====data32=$data32
+ goto loop4
+endi
+
+if $data41 != 1 then
+ print =====data41=$data41
+ goto loop4
+endi
+
+if $data42 != 3 then
+ print =====data42=$data42
+ goto loop4
+endi
+
+sql drop database if exists test1;
+sql create database test1 vgroups 1;
+sql use test1;
+sql create table t1(ts timestamp, a int, b int , c int, d int);
+sql create stream streams1 trigger at_once into streamt1 as select _wstart c1, count(*) c2, max(d) c3, _group_key(a+b) c4 from t1 partition by a+b state_window(c);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,1,2,1,1);
+sql insert into t1 values(1648791213001,2,1,1,2);
+sql insert into t1 values(1648791213001,1,2,1,3);
+
+$loop_count = 0
+
+loop5:
+sleep 300
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+sql insert into t1 values(1648791223000,1,2,2,4);
+sql insert into t1 values(1648791223001,1,2,2,5);
+sql insert into t1 values(1648791223002,1,2,2,6);
+sql insert into t1 values(1648791213001,1,1,1,7) (1648791223002,1,1,2,8);
+
+loop6:
+sleep 300
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 1 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 7 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+if $data21 != 2 then
+ print =====data21=$data21
+ goto loop6
+endi
+
+if $data22 != 5 then
+ print =====data22=$data22
+ goto loop6
+endi
+
+if $data31 != 1 then
+ print =====data31=$data31
+ goto loop6
+endi
+
+if $data32 != 8 then
+ print =====data32=$data32
+ goto loop6
+endi
+
+system sh/stop_dnodes.sh
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+#goto looptest
diff --git a/tests/script/tsim/sync/create-mnode.sim b/tests/script/tsim/sync/create-mnode.sim
new file mode 100644
index 0000000000..cfaafc8208
--- /dev/null
+++ b/tests/script/tsim/sync/create-mnode.sim
@@ -0,0 +1,20 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+
+sql connect
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+
+sql create mnode on dnode 2
+sql create mnode on dnode 3
diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py
index 3be59f0adf..a20b7b17bc 100644
--- a/tests/system-test/0-others/user_control.py
+++ b/tests/system-test/0-others/user_control.py
@@ -282,12 +282,12 @@ class TDTestCase:
use.error(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())")
elif check_priv == PRIVILEGES_WRITE:
use.query(f"use {DBNAME}")
- use.query(f"show {DBNAME}.tables")
+ use.error(f"show {DBNAME}.tables")
use.error(f"select * from {DBNAME}.{CTBNAME}")
use.query(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())")
elif check_priv is None:
use.error(f"use {DBNAME}")
- # use.error(f"show {DBNAME}.tables")
+ use.error(f"show {DBNAME}.tables")
use.error(f"show tables")
use.error(f"select * from {DBNAME}.{CTBNAME}")
use.error(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())")
diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py
index 1cb08aeee8..169b1c2c38 100644
--- a/tests/system-test/2-query/max.py
+++ b/tests/system-test/2-query/max.py
@@ -26,7 +26,7 @@ class TDTestCase:
% (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
intData.append(i + 1)
floatData.append(i + 0.1)
- for i in ['ts','col11','col12','col13']:
+ for i in ['col11','col12','col13']:
for j in ['stb','stb_1']:
tdSql.error(f'select max({i} from {dbname}.{j} )')
@@ -37,6 +37,20 @@ class TDTestCase:
tdSql.checkData(0, 0, np.max(intData))
elif i>=9:
tdSql.checkData(0, 0, np.max(floatData))
+
+ tdSql.query(f"select max(now()) from {dbname}.stb_1")
+ tdSql.checkRows(1)
+
+ tdSql.query(f"select last(ts) from {dbname}.stb_1")
+ lastTs = tdSql.getData(0, 0)
+ tdSql.query(f"select max(ts) from {dbname}.stb_1")
+ tdSql.checkData(0, 0, lastTs)
+
+ tdSql.query(f"select last(ts) from {dbname}.stb")
+ lastTs = tdSql.getData(0, 0)
+ tdSql.query(f"select max(ts) from {dbname}.stb")
+ tdSql.checkData(0, 0, lastTs)
+
tdSql.query(f"select max(col1) from {dbname}.stb_1 where col2<=5")
tdSql.checkData(0,0,5)
tdSql.query(f"select max(col1) from {dbname}.stb where col2<=5")
@@ -53,7 +67,7 @@ class TDTestCase:
% (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
intData.append(i + 1)
floatData.append(i + 0.1)
- for i in ['ts','col11','col12','col13']:
+ for i in ['col11','col12','col13']:
for j in ['ntb']:
tdSql.error(f'select max({i} from {dbname}.{j} )')
for i in range(1,11):
@@ -63,6 +77,15 @@ class TDTestCase:
tdSql.checkData(0, 0, np.max(intData))
elif i>=9:
tdSql.checkData(0, 0, np.max(floatData))
+
+ tdSql.query(f"select max(now()) from {dbname}.ntb")
+ tdSql.checkRows(1)
+
+ tdSql.query(f"select last(ts) from {dbname}.ntb")
+ lastTs = tdSql.getData(0, 0)
+ tdSql.query(f"select max(ts) from {dbname}.ntb")
+ tdSql.checkData(0, 0, lastTs)
+
tdSql.query(f"select max(col1) from {dbname}.ntb where col2<=5")
tdSql.checkData(0,0,5)
diff --git a/tests/system-test/2-query/min.py b/tests/system-test/2-query/min.py
index c47fa49237..3d46b7b222 100644
--- a/tests/system-test/2-query/min.py
+++ b/tests/system-test/2-query/min.py
@@ -37,13 +37,11 @@ class TDTestCase:
floatData.append(i + 0.1)
# max verifacation
- tdSql.error(f"select min(ts) from {dbname}.stb_1")
tdSql.error(f"select min(col7) from {dbname}.stb_1")
tdSql.error(f"select min(col8) from {dbname}.stb_1")
tdSql.error(f"select min(col9) from {dbname}.stb_1")
tdSql.error(f"select min(a) from {dbname}.stb_1")
tdSql.query(f"select min(1) from {dbname}.stb_1")
- tdSql.error(f"select min(now()) from {dbname}.stb_1")
tdSql.error(f"select min(count(c1),count(c2)) from {dbname}.stb_1")
tdSql.query(f"select min(col1) from {dbname}.stb_1")
@@ -69,14 +67,25 @@ class TDTestCase:
tdSql.query(f"select min(col1) from {dbname}.stb_1 where col2>=5")
tdSql.checkData(0,0,5)
+ tdSql.query(f"select min(now()) from {dbname}.stb_1")
+ tdSql.checkRows(1)
+
+ tdSql.query(f"select first(ts) from {dbname}.stb_1")
+ firstTs = tdSql.getData(0, 0)
+ tdSql.query(f"select min(ts) from {dbname}.stb_1")
+ tdSql.checkData(0, 0, firstTs)
+
+ tdSql.query(f"select first(ts) from {dbname}.stb_1")
+ firstTs = tdSql.getData(0, 0)
+ tdSql.query(f"select min(ts) from {dbname}.stb_1")
+ tdSql.checkData(0, 0, firstTs)
+
- tdSql.error(f"select min(ts) from {dbname}.stb_1")
tdSql.error(f"select min(col7) from {dbname}.stb_1")
tdSql.error(f"select min(col8) from {dbname}.stb_1")
tdSql.error(f"select min(col9) from {dbname}.stb_1")
tdSql.error(f"select min(a) from {dbname}.stb_1")
tdSql.query(f"select min(1) from {dbname}.stb_1")
- tdSql.error(f"select min(now()) from {dbname}.stb_1")
tdSql.error(f"select min(count(c1),count(c2)) from {dbname}.stb_1")
tdSql.query(f"select min(col1) from {dbname}.stb")
@@ -102,13 +111,24 @@ class TDTestCase:
tdSql.query(f"select min(col1) from {dbname}.stb where col2>=5")
tdSql.checkData(0,0,5)
- tdSql.error(f"select min(ts) from {dbname}.ntb")
+ tdSql.query(f"select min(now()) from {dbname}.stb_1")
+ tdSql.checkRows(1)
+
+ tdSql.query(f"select first(ts) from {dbname}.stb_1")
+ firstTs = tdSql.getData(0, 0)
+ tdSql.query(f"select min(ts) from {dbname}.stb_1")
+ tdSql.checkData(0, 0, firstTs)
+
+ tdSql.query(f"select first(ts) from {dbname}.stb_1")
+ firstTs = tdSql.getData(0, 0)
+ tdSql.query(f"select min(ts) from {dbname}.stb_1")
+ tdSql.checkData(0, 0, firstTs)
+
tdSql.error(f"select min(col7) from {dbname}.ntb")
tdSql.error(f"select min(col8) from {dbname}.ntb")
tdSql.error(f"select min(col9) from {dbname}.ntb")
tdSql.error(f"select min(a) from {dbname}.ntb")
tdSql.query(f"select min(1) from {dbname}.ntb")
- tdSql.error(f"select min(now()) from {dbname}.ntb")
tdSql.error(f"select min(count(c1),count(c2)) from {dbname}.ntb")
tdSql.query(f"select min(col1) from {dbname}.ntb")
@@ -134,6 +154,19 @@ class TDTestCase:
tdSql.query(f"select min(col1) from {dbname}.ntb where col2>=5")
tdSql.checkData(0,0,5)
+ tdSql.query(f"select min(now()) from {dbname}.stb_1")
+ tdSql.checkRows(1)
+
+ tdSql.query(f"select first(ts) from {dbname}.stb_1")
+ firstTs = tdSql.getData(0, 0)
+ tdSql.query(f"select min(ts) from {dbname}.stb_1")
+ tdSql.checkData(0, 0, firstTs)
+
+ tdSql.query(f"select first(ts) from {dbname}.stb_1")
+ firstTs = tdSql.getData(0, 0)
+ tdSql.query(f"select min(ts) from {dbname}.stb_1")
+ tdSql.checkData(0, 0, firstTs)
+
def stop(self):
tdSql.close()
diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py
index 6f8183bf06..c0f33d9204 100644
--- a/tests/system-test/7-tmq/tmqShow.py
+++ b/tests/system-test/7-tmq/tmqShow.py
@@ -19,6 +19,11 @@ class TDTestCase:
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
+ def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,offset=1,cdbName='cdb'):
+ sql = "insert into %s.consumeinfo values "%cdbName
+ sql += "(now+%ds, %d, '%s', '%s', %d, %d, %d)"%(offset,consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit)
+ tdLog.info("consume info sql: %s"%sql)
+ tdSql.query(sql)
def tmqCase1(self):
tdLog.printNoPrefix("======== test case 1: ")
@@ -95,19 +100,23 @@ class TDTestCase:
ifcheckdata = 0
ifManualCommit = 0
keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[0]
- tmqCom.insertConsumerInfo(consumerIdList[0], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+ tsOffset=1
+ self.insertConsumerInfo(consumerIdList[0], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit,tsOffset)
topicList = topicNameList[1]
keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[1]
- tmqCom.insertConsumerInfo(consumerIdList[1], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+ tsOffset=2
+ self.insertConsumerInfo(consumerIdList[1], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit,tsOffset)
topicList = topicNameList[2]
keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[2]
- tmqCom.insertConsumerInfo(consumerIdList[2], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+ tsOffset=3
+ self.insertConsumerInfo(consumerIdList[2], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit,tsOffset)
topicList = topicNameList[3]
keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[3]
- tmqCom.insertConsumerInfo(consumerIdList[3], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+ tsOffset=4
+ self.insertConsumerInfo(consumerIdList[3], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit,tsOffset)
tdLog.info("start consume processor")
tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 335247c16d..4305ceff56 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -314,8 +314,8 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
-#python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
-#python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
+python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
+python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py
python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py
@@ -514,3 +514,4 @@ python3 ./test.py -f 2-query/last_row.py -Q 3
python3 ./test.py -f 2-query/tsbsQuery.py -Q 3
python3 ./test.py -f 2-query/sml.py -Q 3
python3 ./test.py -f 2-query/interp.py -Q 3
+