Merge branch '3.0' into feature/3.0_interval_hash_optimize

.gitignore

@@ -1,5 +1,6 @@
 build/
 compile_commands.json
+CMakeSettings.json
 .cache
 .ycm_extra_conf.py
 .tasks

CMakeSettings.json

@@ -1,25 +0,0 @@
-{
-  "configurations": [
-    {
-      "name": "WSL-GCC-Debug",
-      "generator": "Unix Makefiles",
-      "configurationType": "Debug",
-      "buildRoot": "${projectDir}\\build\\",
-      "installRoot": "${projectDir}\\build\\",
-      "cmakeExecutable": "/usr/bin/cmake",
-      "cmakeCommandArgs": "",
-      "buildCommandArgs": "",
-      "ctestCommandArgs": "",
-      "inheritEnvironments": [ "linux_x64" ],
-      "wslPath": "${defaultWSLPath}",
-      "addressSanitizerRuntimeFlags": "detect_leaks=0",
-      "variables": [
-        {
-          "name": "CMAKE_INSTALL_PREFIX",
-          "value": "/mnt/d/TDengine/TDengine/build",
-          "type": "PATH"
-        }
-      ]
-    }
-  ]
-}

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 9cb965f
+        GIT_TAG d2b5dec
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/en/07-develop/01-connect/index.md

@@ -1,6 +1,7 @@
 ---
-title: Connect
+sidebar_label: Connect
-description: "This document explains how to establish connections to TDengine and how to install and use TDengine connectors."
+title: Connect to TDengine
+description: "How to establish connections to TDengine and how to install and use TDengine connectors."
 ---
 
 import Tabs from "@theme/Tabs";

docs/en/12-taos-sql/02-database.md

@@ -71,9 +71,9 @@ database_option: {
 - SINGLE_STABLE: specifies whether the database can contain more than one supertable.
   - 0: The database can contain multiple supertables.
   - 1: The database can contain only one supertable.
-- WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. Enter a time in seconds. The default value is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted.
+- WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. Enter a time in seconds. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is 4 days.
-- WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted.
+- WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. Enter a size in KB. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is -1.
-- WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk.
+- WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value of single copy is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk. The default values of multiple copy is 1 day.
 - WAL_SEGMENT_SIZE: specifies the maximum size of a WAL file. After the current WAL file reaches this size, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk.
 
 ### Example Statement

docs/en/12-taos-sql/10-function.md

@@ -613,6 +613,7 @@ SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHER
 **Explanations**：
 - _P_ is in range [0,100], when _P_ is 0, the result is same as using function MIN; when _P_ is 100, the result is same as function MAX.
 - `algo_type` can only be input as `default` or `t-digest` Enter `default` to use a histogram-based algorithm. Enter `t-digest` to use the t-digest algorithm to calculate the approximation of the quantile. `default` is used by default.
+- The approximation result of `t-digest` algorithm is sensitive to input data order. For example, when querying STable with different input data order there might be minor differences in calculated results.
 
 ### AVG
 

docs/en/12-taos-sql/14-stream.md

@@ -50,7 +50,7 @@ SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVA
 ## Delete a Stream
 
 ```sql
-DROP STREAM [IF NOT EXISTS] stream_name
+DROP STREAM [IF EXISTS] stream_name
 ```
 
 This statement deletes the stream processing service only. The data generated by the stream is retained.

docs/en/12-taos-sql/24-show.md

@@ -5,16 +5,6 @@ title: SHOW Statement for Metadata
 
 `SHOW` command can be used to get brief system information. To get details about metatadata, information, and status in the system, please use `select` to query the tables in database `INFORMATION_SCHEMA`. 
 
-## SHOW ACCOUNTS
-
-```sql
-SHOW ACCOUNTS;
-```
-
-Shows information about tenants on the system.
-
-Note: TDengine Enterprise Edition only.
-
 ## SHOW APPS
 
 ```sql

docs/en/14-reference/03-connector/05-go.mdx

@@ -7,7 +7,7 @@ title: TDengine Go Connector
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-import Preparition from "./_preparition.mdx"
+import Preparition from "./_preparation.mdx"
 import GoInsert from "../../07-develop/03-insert-data/_go_sql.mdx"
 import GoInfluxLine from "../../07-develop/03-insert-data/_go_line.mdx"
 import GoOpenTSDBTelnet from "../../07-develop/03-insert-data/_go_opts_telnet.mdx"

docs/en/14-reference/03-connector/06-rust.mdx

@@ -7,7 +7,7 @@ title: TDengine Rust Connector
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-import Preparition from "./_preparition.mdx"
+import Preparition from "./_preparation.mdx"
 import RustInsert from "../../07-develop/03-insert-data/_rust_sql.mdx"
 import RustBind from "../../07-develop/03-insert-data/_rust_stmt.mdx"
 import RustQuery from "../../07-develop/04-query-data/_rust.mdx"

docs/en/14-reference/03-connector/08-node.mdx

@@ -7,7 +7,7 @@ title: TDengine Node.js Connector
 import Tabs from "@theme/Tabs";
 import TabItem from "@theme/TabItem";
 
-import Preparition from "./_preparition.mdx";
+import Preparition from "./_preparation.mdx";
 import NodeInsert from "../../07-develop/03-insert-data/_js_sql.mdx";
 import NodeInfluxLine from "../../07-develop/03-insert-data/_js_line.mdx";
 import NodeOpenTSDBTelnet from "../../07-develop/03-insert-data/_js_opts_telnet.mdx";

docs/en/14-reference/03-connector/09-csharp.mdx

@@ -7,7 +7,7 @@ title: C# Connector
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-import Preparition from "./_preparition.mdx"
+import Preparition from "./_preparation.mdx"
 import CSInsert from "../../07-develop/03-insert-data/_cs_sql.mdx"
 import CSInfluxLine from "../../07-develop/03-insert-data/_cs_line.mdx"
 import CSOpenTSDBTelnet from "../../07-develop/03-insert-data/_cs_opts_telnet.mdx"

docs/en/14-reference/03-connector/_preparition.mdx

@@ -1,10 +0,0 @@
-- 已安装客户端驱动（使用原生连接必须安装，使用 REST 连接无需安装）
-
-:::info
-
-由于 TDengine 的客户端驱动使用 C 语言编写，使用原生连接时需要加载系统对应安装在本地的客户端驱动共享库文件，通常包含在 TDengine 安装包。TDengine Linux 服务端安装包附带了 TDengine 客户端，也可以单独安装 [Linux 客户端](/get-started/) 。在 Windows 环境开发时需要安装 TDengine 对应的 [Windows 客户端](https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client) 。
-
-- libtaos.so: 在 Linux 系统中成功安装 TDengine 后，依赖的 Linux 版客户端驱动 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。
-- taos.dll: 在 Windows 系统中安装完客户端之后，依赖的 Windows 版客户端驱动 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。
-
-:::

docs/zh/02-intro.md

@@ -26,7 +26,7 @@ TDengine 的主要功能如下：
      - [Icinga2](../third-party/icinga2)
      - [TCollector](../third-party/tcollector)
      - [EMQX](../third-party/emq-broker)
-     - [HiveMQ](../third-party/hive-mq-broker) ；
+     - [HiveMQ](../third-party/hive-mq-broker)
 2. 查询数据，支持
    - [标准 SQL](../taos-sql)，含嵌套查询
    - [时序数据特色函数](../taos-sql/function/#time-series-extensions)
@@ -85,14 +85,14 @@ TDengine 的主要功能如下：
 
 ![TDengine Database 技术生态图](eco_system.webp)
 
-<center><figcaption>图 1. TDengine技术生态图</figcaption></center>
+<center><figcaption>图 1. TDengine 技术生态图</figcaption></center>
 </figure>
 
 上图中，左侧是各种数据采集或消息队列，包括 OPC-UA、MQTT、Telegraf、也包括 Kafka，他们的数据将被源源不断的写入到 TDengine。右侧则是可视化、BI 工具、组态软件、应用程序。下侧则是 TDengine 自身提供的命令行程序（CLI）以及可视化管理工具。
 
 ## 典型适用场景
 
-作为一个高性能、分布式、支持 SQL 的时序数据库（Database)，TDengine 的典型适用场景包括但不限于 IoT、工业互联网、车联网、IT 运维、能源、金融证券等领域。需要指出的是，TDengine 是针对时序数据场景设计的专用数据库和专用大数据处理工具，因其充分利用了时序大数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。下面本文将对适用场景做更多详细的分析。
+作为一个高性能、分布式、支持 SQL 的时序数据库（Database），TDengine 的典型适用场景包括但不限于 IoT、工业互联网、车联网、IT 运维、能源、金融证券等领域。需要指出的是，TDengine 是针对时序数据场景设计的专用数据库和专用大数据处理工具，因其充分利用了时序大数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。下面本文将对适用场景做更多详细的分析。
 
 ### 数据源特点和需求
 

docs/zh/08-connector/09-csharp.mdx

@@ -169,9 +169,9 @@ namespace TDengineExample
 
 ### 第三方驱动
 
-`Maikebing.Data.Taos` 是一个 TDengine 的 ADO.NET 连接器，支持 Linux，Windows 平台。该连接器由社区贡献者`麦壳饼@@maikebing` 提供，具体请参考:
+[`IoTSharp.Data.Taos`](https://github.com/IoTSharp/EntityFrameworkCore.Taos) 是一个 TDengine 的 ADO.NET 连接器,其中包含了用于EntityFrameworkCore 的提供程序 IoTSharp.EntityFrameworkCore.Taos 和健康检查组件 IoTSharp.HealthChecks.Taos ，支持 Linux，Windows 平台。该连接器由社区贡献者`麦壳饼@@maikebing` 提供，具体请参考:
 
-* 接口下载:<https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos>
+* 接口下载: <https://github.com/IoTSharp/EntityFrameworkCore.Taos>
 * 用法说明:<https://www.taosdata.com/blog/2020/11/02/1901.html>
 
 ## 常见问题

docs/zh/12-taos-sql/02-database.md

@@ -71,9 +71,9 @@ database_option: {
 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。
   - 0：表示可以创建多张超级表。
   - 1：表示只可以创建一张超级表。
-- WAL_RETENTION_PERIOD：wal 文件的额外保留策略，用于数据订阅。wal 的保存时长，单位为 s。默认为 0，即落盘后立即删除。-1 表示不删除。
+- WAL_RETENTION_PERIOD：wal 文件的额外保留策略，用于数据订阅。wal 的保存时长，单位为 s。单副本默认为 0，即落盘后立即删除。-1 表示不删除。多副本默认为 4 天。
-- WAL_RETENTION_SIZE：wal 文件的额外保留策略，用于数据订阅。wal 的保存的最大上限，单位为 KB。默认为 0，即落盘后立即删除。-1 表示不删除。
+- WAL_RETENTION_SIZE：wal 文件的额外保留策略，用于数据订阅。wal 的保存的最大上限，单位为 KB。单副本默认为 0，即落盘后立即删除。多副本默认为-1，表示不删除。
-- WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当 wal 文件创建并写入后，经过该时间，会自动创建一个新的 wal 文件。默认为 0，即仅在落盘时创建新文件。
+- WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当 wal 文件创建并写入后，经过该时间，会自动创建一个新的 wal 文件。单副本默认为 0，即仅在落盘时创建新文件。多副本默认为 1 天。
 - WAL_SEGMENT_SIZE：wal 单个文件大小，单位为 KB。当前写入文件大小超过上限后会自动创建一个新的 wal 文件。默认为 0，即仅在落盘时创建新文件。
 
 ### 创建数据库示例

docs/zh/12-taos-sql/10-function.md

@@ -614,6 +614,7 @@ SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHER
 **说明**：
 - P值范围是[0,100]，当为0时等同于MIN，为100时等同于MAX。
 - algo_type 取值为 "default" 或 "t-digest"。 输入为 "default" 时函数使用基于直方图算法进行计算。输入为 "t-digest" 时使用t-digest算法计算分位数的近似结果。如果不指定 algo_type 则使用 "default" 算法。
+- "t-digest"算法的近似结果对于输入数据顺序敏感，对超级表查询时不同的输入排序结果可能会有微小的误差。
 
 ### AVG
 
@@ -917,7 +918,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回数据类型**：同应用的字段。
 
-**适用数据类型**：数值类型。
+**适用数据类型**：数值类型，时间戳类型。
 
 **适用于**：表和超级表。
 
@@ -932,7 +933,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause];
 
 **返回数据类型**：同应用的字段。
 
-**适用数据类型**：数值类型。
+**适用数据类型**：数值类型，时间戳类型。
 
 **适用于**：表和超级表。
 

docs/zh/12-taos-sql/14-stream.md

@@ -58,7 +58,7 @@ SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVA
 ## 删除流式计算
 
 ```sql
-DROP STREAM [IF NOT EXISTS] stream_name;
+DROP STREAM [IF EXISTS] stream_name;
 ```
 
 仅删除流式计算任务，由流式计算写入的数据不会被删除。

docs/zh/12-taos-sql/24-show.md

@@ -6,16 +6,6 @@ description: SHOW 命令的完整列表
 
 SHOW 命令可以用来获取简要的系统信息。若想获取系统中详细的各种元数据、系统信息和状态，请使用 select 语句查询 INFORMATION_SCHEMA 数据库中的表。
 
-## SHOW ACCOUNTS
-
-```sql
-SHOW ACCOUNTS;
-```
-
-显示当前系统中所有租户的信息。
-
-注：企业版独有
-
 ## SHOW APPS
 
 ```sql

docs/zh/14-reference/05-taosbenchmark.md

@@ -405,37 +405,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
 
 订阅子表或者普通表的配置参数在 `specified_table_query` 中设置。
 
-- **threads** : 执行 SQL 的线程数，默认为 1。
+- **threads/concurrent** : 执行 SQL 的线程数，默认为 1。
-
-- **interval** : 执行订阅的时间间隔，单位为秒，默认为 0。
-
-- **restart** : "yes" 表示开始新的订阅，"no" 表示继续之前的订阅，默认值为 "no"。
-
-- **keepProgress** : "yes" 表示保留订阅进度，"no" 表示不保留，默认值为 "no"。
-
-- **resubAfterConsume** : "yes" 表示取消之前的订阅然后再次订阅， "no" 表示继续之前的订阅，默认值为 "no"。
 
 - **sqls** ：
   - **sql** : 执行的 SQL 命令，必填。
-  - **result** : 保存查询结果的文件，未指定则不保存。
-
-#### 订阅超级表的配置参数
-
-订阅超级表的配置参数在 `super_table_query` 中设置。
-
-- **stblname** : 要订阅的超级表名称，必填。
-
-- **threads** : 执行 SQL 的线程数，默认为 1。
-
-- **interval** : 执行订阅的时间间隔，单位为秒，默认为 0。
-
-- **restart** : "yes" 表示开始新的订阅，"no" 表示继续之前的订阅，默认值为 "no"。
-
-- **keepProgress** : "yes" 表示保留订阅进度，"no" 表示不保留，默认值为 "no"。
-
-- **resubAfterConsume** : "yes" 表示取消之前的订阅然后再次订阅， "no" 表示继续之前的订阅，默认值为 "no"。
-
-- **sqls** ：
-  - **sql** : 执行的 SQL 命令，必填；对于超级表的查询 SQL，在 SQL 命令中保留 "xxxx"，程序会自动将其替换为超级表的所有子表名。
-    替换为超级表中所有的子表名。
-  - **result** : 保存查询结果的文件，未指定则不保存。

include/common/tglobal.h

@@ -89,11 +89,12 @@ extern uint16_t tsTelemPort;
 
 // query buffer management
 extern int32_t tsQueryBufferSize;  // maximum allowed usage buffer size in MB for each data node during query processing
-extern int64_t tsQueryBufferSizeBytes;   // maximum allowed usage buffer size in byte for each data node
+extern int64_t tsQueryBufferSizeBytes;  // maximum allowed usage buffer size in byte for each data node
 
 // query client
 extern int32_t tsQueryPolicy;
 extern int32_t tsQuerySmaOptimize;
+extern bool    tsQueryPlannerTrace;
 
 // client
 extern int32_t tsMinSlidingTime;
@@ -147,7 +148,7 @@ struct SConfig *taosGetCfg();
 void    taosSetAllDebugFlag(int32_t flag, bool rewrite);
 void    taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite);
 int32_t taosSetCfg(SConfig *pCfg, char *name);
-void    taosLocalCfgForbiddenToChange(char* name, bool* forbidden);
+void    taosLocalCfgForbiddenToChange(char *name, bool *forbidden);
 
 #ifdef __cplusplus
 }

include/common/tmsg.h

@@ -2994,6 +2994,7 @@ typedef struct {
 
 int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const STaosxRsp* pRsp);
 int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, STaosxRsp* pRsp);
+void    tDeleteSTaosxRsp(STaosxRsp* pRsp);
 
 typedef struct {
   SMqRspHead head;

include/common/ttypes.h

@@ -49,9 +49,6 @@ typedef struct {
 #define varDataCopy(dst, v)    memcpy((dst), (void *)(v), varDataTLen(v))
 #define varDataLenByData(v)    (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
-#define IS_VAR_DATA_TYPE(t) \
-  (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
-#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
 
 #define varDataNetLen(v)  (htons(((VarDataLenT *)(v))[0]))
 #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
@@ -268,11 +265,16 @@ typedef struct {
 #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT)
 #define IS_FLOAT_TYPE(_t)            ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE)
 #define IS_INTEGER_TYPE(_t)          ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)))
+#define IS_TIMESTAMP_TYPE(_t)        ((_t) == TSDB_DATA_TYPE_TIMESTAMP)
 
 #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)))
 #define IS_MATHABLE_TYPE(_t) \
   (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
 
+#define IS_VAR_DATA_TYPE(t) \
+  (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
+#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
+
 #define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
 #define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
 #define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)

include/libs/nodes/cmdnodes.h

@@ -78,6 +78,9 @@ typedef struct SDatabaseOptions {
   int32_t     walRetentionSize;
   int32_t     walRollPeriod;
   int32_t     walSegmentSize;
+  bool        walRetentionPeriodIsSet;
+  bool        walRetentionSizeIsSet;
+  bool        walRollPeriodIsSet;
 } SDatabaseOptions;
 
 typedef struct SCreateDatabaseStmt {

include/libs/nodes/nodes.h

@@ -244,6 +244,7 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE,
   QUERY_NODE_PHYSICAL_PLAN_PARTITION,
+  QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION,
   QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC,
   QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC,
   QUERY_NODE_PHYSICAL_PLAN_DISPATCH,

include/libs/nodes/plannodes.h

@@ -488,6 +488,8 @@ typedef struct SPartitionPhysiNode {
   SNodeList* pTargets;
 } SPartitionPhysiNode;
 
+typedef SPartitionPhysiNode SStreamPartitionPhysiNode;
+
 typedef struct SDataSinkNode {
   ENodeType           type;
   SDataBlockDescNode* pInputDataBlockDesc;

include/os/osDir.h

@@ -56,6 +56,7 @@ void    taosRemoveDir(const char *dirname);
 bool    taosDirExist(const char *dirname);
 int32_t taosMkDir(const char *dirname);
 int32_t taosMulMkDir(const char *dirname);
+int32_t taosMulModeMkDir(const char *dirname, int mode);
 void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
 int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen);

include/util/tdef.h

@@ -360,14 +360,17 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_SCHEMALESS_OFF          0
 #define TSDB_DEFAULT_DB_SCHEMALESS      TSDB_DB_SCHEMALESS_OFF
 
-#define TSDB_DB_MIN_WAL_RETENTION_PERIOD     -1
+#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1
-#define TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD (24 * 60 * 60 * 4)
+#define TSDB_REP_DEF_DB_WAL_RET_PERIOD   0
-#define TSDB_DB_MIN_WAL_RETENTION_SIZE       -1
+#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD  (24 * 60 * 60 * 4)
-#define TSDB_DEFAULT_DB_WAL_RETENTION_SIZE   -1
+#define TSDB_DB_MIN_WAL_RETENTION_SIZE   -1
-#define TSDB_DB_MIN_WAL_ROLL_PERIOD          0
+#define TSDB_REP_DEF_DB_WAL_RET_SIZE     0
-#define TSDB_DEFAULT_DB_WAL_ROLL_PERIOD      (24 * 60 * 60 * 1)
+#define TSDB_REPS_DEF_DB_WAL_RET_SIZE    -1
-#define TSDB_DB_MIN_WAL_SEGMENT_SIZE         0
+#define TSDB_DB_MIN_WAL_ROLL_PERIOD      0
-#define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE     0
+#define TSDB_REP_DEF_DB_WAL_ROLL_PERIOD  0
+#define TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD (24 * 60 * 60 * 1)
+#define TSDB_DB_MIN_WAL_SEGMENT_SIZE     0
+#define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE 0
 
 #define TSDB_MIN_ROLLUP_MAX_DELAY     1  // unit millisecond
 #define TSDB_MAX_ROLLUP_MAX_DELAY     (15 * 60 * 1000)
@@ -386,7 +389,7 @@ typedef enum ELogicConditionType {
 
 #define TSDB_DEFAULT_EXPLAIN_VERBOSE false
 
-#define TSDB_EXPLAIN_RESULT_ROW_SIZE    (16*1024)
+#define TSDB_EXPLAIN_RESULT_ROW_SIZE    (16 * 1024)
 #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
 
 #define TSDB_MAX_FIELD_LEN             16384

packaging/deb/makedeb.sh

@@ -45,6 +45,7 @@ mkdir -p ${pkg_dir}${install_home_path}/include
 mkdir -p ${pkg_dir}${install_home_path}/script
 
 cp ${compile_dir}/../packaging/cfg/taos.cfg         ${pkg_dir}${install_home_path}/cfg
+cp ${compile_dir}/../packaging/cfg/taosd.service    ${pkg_dir}${install_home_path}/cfg
 if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then
     cp ${compile_dir}/test/cfg/taosadapter.toml		${pkg_dir}${install_home_path}/cfg || :
 fi

source/client/src/tmq.c

@@ -1773,7 +1773,7 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) {
     }
     return TMQ_RES_TABLE_META;
   } else if (TD_RES_TMQ_TAOSX(res)) {
-    return TMQ_RES_TAOSX;
+    return TMQ_RES_DATA;
   } else {
     return TMQ_RES_INVALID;
   }

source/common/src/tglobal.c

@@ -91,6 +91,7 @@ bool tsSmlDataFormat =
 // query
 int32_t tsQueryPolicy = 1;
 int32_t tsQuerySmaOptimize = 0;
+bool    tsQueryPlannerTrace = false;
 
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,
@@ -286,6 +287,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1;
+  if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1;
   if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
   if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
   if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
@@ -429,9 +431,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
 static int32_t taosUpdateServerCfg(SConfig *pCfg) {
   SConfigItem *pItem;
-  ECfgSrcType stype;
+  ECfgSrcType  stype;
-  int32_t numOfCores;
+  int32_t      numOfCores;
-  int64_t totalMemoryKB;
+  int64_t      totalMemoryKB;
 
   pItem = cfgGetItem(tsCfg, "numOfCores");
   if (pItem == NULL) {
@@ -572,7 +574,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
   return 0;
 }
 
-
 static void taosSetClientLogCfg(SConfig *pCfg) {
   SConfigItem *pItem = cfgGetItem(pCfg, "logDir");
   tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX);
@@ -643,6 +644,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
   tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
   tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32;
+  tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
   return 0;
 }
 
@@ -972,6 +974,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
         tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32;
       } else if (strcasecmp("qDebugFlag", name) == 0) {
         qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
+      } else if (strcasecmp("queryPlannerTrace", name) == 0) {
+        tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval;
       }
       break;
     }
@@ -1129,7 +1133,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
 
   taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false);
 
-  if (taosMulMkDir(tsLogDir) != 0) {
+  if (taosMulModeMkDir(tsLogDir, 0777) != 0) {
     uError("failed to create dir:%s since %s", tsLogDir, terrstr());
     cfgCleanup(pCfg);
     return -1;
@@ -1241,16 +1245,14 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
   }
 
   const char *options[] = {
-      "dDebugFlag",   "vDebugFlag",  "mDebugFlag",   "wDebugFlag",   "sDebugFlag",   "tsdbDebugFlag",
+      "dDebugFlag",   "vDebugFlag",   "mDebugFlag",   "wDebugFlag",    "sDebugFlag",   "tsdbDebugFlag", "tqDebugFlag",
-      "tqDebugFlag",  "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag",
+      "fsDebugFlag",  "udfDebugFlag", "smaDebugFlag", "idxDebugFlag",  "tdbDebugFlag", "tmrDebugFlag",  "uDebugFlag",
-      "tmrDebugFlag", "uDebugFlag",  "smaDebugFlag", "rpcDebugFlag", "qDebugFlag",   "metaDebugFlag",
+      "smaDebugFlag", "rpcDebugFlag", "qDebugFlag",   "metaDebugFlag", "jniDebugFlag",
-      "jniDebugFlag",
   };
   int32_t *optionVars[] = {
-      &dDebugFlag,   &vDebugFlag,  &mDebugFlag,   &wDebugFlag,   &sDebugFlag,   &tsdbDebugFlag,
+      &dDebugFlag,   &vDebugFlag,   &mDebugFlag,   &wDebugFlag,    &sDebugFlag,   &tsdbDebugFlag, &tqDebugFlag,
-      &tqDebugFlag,  &fsDebugFlag, &udfDebugFlag, &smaDebugFlag, &idxDebugFlag, &tdbDebugFlag,
+      &fsDebugFlag,  &udfDebugFlag, &smaDebugFlag, &idxDebugFlag,  &tdbDebugFlag, &tmrDebugFlag,  &uDebugFlag,
-      &tmrDebugFlag, &uDebugFlag,  &smaDebugFlag, &rpcDebugFlag, &qDebugFlag,   &metaDebugFlag,
+      &smaDebugFlag, &rpcDebugFlag, &qDebugFlag,   &metaDebugFlag, &jniDebugFlag,
-      &jniDebugFlag,
   };
 
   int32_t optionSize = tListLen(options);

source/common/src/tmsg.c

@@ -5984,6 +5984,17 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) {
   }
   return 0;
 }
+
+void tDeleteSTaosxRsp(STaosxRsp *pRsp) {
+  taosArrayDestroy(pRsp->blockDataLen);
+  taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
+  taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);
+  taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
+
+  taosArrayDestroy(pRsp->createTableLen);
+  taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree);
+}
+
 int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) {
   if (tEncodeI64(pEncoder, pReq->uid) < 0) return -1;
   if (tEncodeI64(pEncoder, pReq->ts) < 0) return -1;

source/dnode/mnode/impl/src/mndDb.c

@@ -358,10 +358,10 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0;
   if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF;
   if (pCfg->walRetentionPeriod < 0 && pCfg->walRetentionPeriod != -1)
-    pCfg->walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD;
+    pCfg->walRetentionPeriod = TSDB_REPS_DEF_DB_WAL_RET_PERIOD;
   if (pCfg->walRetentionSize < 0 && pCfg->walRetentionSize != -1)
-    pCfg->walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE;
+    pCfg->walRetentionSize = TSDB_REPS_DEF_DB_WAL_RET_SIZE;
-  if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD;
+  if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD;
   if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
 }
 

source/dnode/vnode/src/inc/tq.h

@@ -140,11 +140,12 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle);
 int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
 
 // tqRead
-int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
+int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
+int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset);
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);
 
 // tqExec
-int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp* pRsp);
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp);
 int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp);
 
 // tqMeta

source/dnode/vnode/src/tq/tq.c

@@ -196,6 +196,66 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
   return 0;
 }
 
+int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) {
+  ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
+  ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
+
+  if (pRsp->withSchema) {
+    ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum);
+  } else {
+    ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0);
+  }
+
+  if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
+    if (pRsp->blockNum > 0) {
+      ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);
+    } else {
+      ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version);
+    }
+  }
+
+  int32_t len = 0;
+  int32_t code = 0;
+  tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code);
+  if (code < 0) {
+    return -1;
+  }
+  int32_t tlen = sizeof(SMqRspHead) + len;
+  void*   buf = rpcMallocCont(tlen);
+  if (buf == NULL) {
+    return -1;
+  }
+
+  ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__TAOSX_RSP;
+  ((SMqRspHead*)buf)->epoch = pReq->epoch;
+  ((SMqRspHead*)buf)->consumerId = pReq->consumerId;
+
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, abuf, len);
+  tEncodeSTaosxRsp(&encoder, pRsp);
+  tEncoderClear(&encoder);
+
+  SRpcMsg rsp = {
+      .info = pMsg->info,
+      .pCont = buf,
+      .contLen = tlen,
+      .code = 0,
+  };
+  tmsgSendRsp(&rsp);
+
+  char buf1[80] = {0};
+  char buf2[80] = {0};
+  tFormatOffset(buf1, 80, &pRsp->reqOffset);
+  tFormatOffset(buf2, 80, &pRsp->rspOffset);
+  tqDebug("taosx rsp, vgId:%d, from consumer:%" PRId64
+          ", (epoch %d) send rsp, block num: %d, reqOffset:%s, rspOffset:%s",
+          TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
+
+  return 0;
+}
+
 static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) {
   return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG &&
          pLeft->val.version <= pRight->val.version;
@@ -303,6 +363,22 @@ static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t su
   return 0;
 }
 
+static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) {
+  pRsp->reqOffset = pReq->reqOffset;
+
+  pRsp->withTbName = 1;
+  pRsp->withSchema = 1;
+  pRsp->blockData = taosArrayInit(0, sizeof(void*));
+  pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t));
+  pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
+  pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
+
+  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
+    return -1;
+  }
+  return 0;
+}
+
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   SMqPollReq*  pReq = pMsg->pCont;
   int64_t      consumerId = pReq->consumerId;
@@ -341,9 +417,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   tqDebug("tmq poll: consumer %" PRId64 " (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId,
           pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf);
 
-  SMqDataRsp dataRsp = {0};
-  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
-
   // 2.reset offset if needed
   if (reqOffset.type > 0) {
     fetchOffsetNew = reqOffset;
@@ -367,6 +440,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
           tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal));
         }
       } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
+        SMqDataRsp dataRsp = {0};
+        tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+
         tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
         tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, offset reset to %" PRId64, consumerId,
                 pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version);
@@ -380,16 +456,38 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
                 " in vg %d, subkey %s, reset none failed",
                 pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey);
         terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
-        code = -1;
+        return -1;
-        tDeleteSMqDataRsp(&dataRsp);
-        return code;
       }
     }
   }
 
-  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN || fetchOffsetNew.type != TMQ_OFFSET__LOG) {
+  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    SMqMetaRsp metaRsp = {0};
+    SMqDataRsp dataRsp = {0};
-    tqScan(pTq, pHandle, &dataRsp, &metaRsp, &fetchOffsetNew);
+    tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+    tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew);
+
+    if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+      code = -1;
+    }
+
+    tqDebug("tmq poll: consumer %ld, subkey %s, vg %d, send data blockNum:%d, offset type:%d, uid:%ld, version:%ld",
+            consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type,
+            dataRsp.rspOffset.uid, dataRsp.rspOffset.version);
+
+    tDeleteSMqDataRsp(&dataRsp);
+    return code;
+  }
+
+  // for taosx
+  ASSERT(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN);
+
+  SMqMetaRsp metaRsp = {0};
+
+  STaosxRsp taosxRsp = {0};
+  tqInitTaosxRsp(&taosxRsp, pReq);
+
+  if (fetchOffsetNew.type != TMQ_OFFSET__LOG) {
+    tqScan(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew);
 
     if (metaRsp.metaRspLen > 0) {
       if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
@@ -399,29 +497,30 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
               pHandle->subKey, TD_VID(pTq->pVnode), metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
               metaRsp.rspOffset.version);
       taosMemoryFree(metaRsp.metaRsp);
-      goto OVER;
+      tDeleteSTaosxRsp(&taosxRsp);
+      return code;
     }
 
-    if (dataRsp.blockNum > 0) {
+    if (taosxRsp.blockNum > 0) {
-      if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+      if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
         code = -1;
       }
-      goto OVER;
+      tDeleteSTaosxRsp(&taosxRsp);
+      return code;
     } else {
-      fetchOffsetNew = dataRsp.rspOffset;
+      fetchOffsetNew = taosxRsp.rspOffset;
     }
 
-    tqDebug("tmq poll: consumer %ld, subkey %s, vg %d, send data blockNum:%d, offset type:%d,uid:%ld,version:%ld",
+    tqDebug("taosx poll: consumer %ld, subkey %s, vg %d, send data blockNum:%d, offset type:%d,uid:%ld,version:%ld",
-            consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type,
+            consumerId, pHandle->subKey, TD_VID(pTq->pVnode), taosxRsp.blockNum, taosxRsp.rspOffset.type,
-            dataRsp.rspOffset.uid, dataRsp.rspOffset.version);
+            taosxRsp.rspOffset.uid, taosxRsp.rspOffset.version);
   }
 
-  if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) {
+  if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
     int64_t fetchVer = fetchOffsetNew.version + 1;
     pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
     if (pCkHead == NULL) {
-      code = -1;
+      return -1;
-      goto OVER;
     }
 
     walSetReaderCapacity(pHandle->pWalReader, 2048);
@@ -436,13 +535,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       }
 
       if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) {
-        // TODO add push mgr
+        tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
-
+        if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
-        tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
-        if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
           code = -1;
         }
-        goto OVER;
+        tDeleteSTaosxRsp(&taosxRsp);
+        if (pCkHead) taosMemoryFree(pCkHead);
+        return code;
       }
 
       SWalCont* pHead = &pCkHead->head;
@@ -453,17 +552,19 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       if (pHead->msgType == TDMT_VND_SUBMIT) {
         SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
 
-        if (tqLogScanExec(pTq, pHandle, pCont, &dataRsp) < 0) {
+        if (tqTaosxScanLog(pTq, pHandle, pCont, &taosxRsp) < 0) {
           /*ASSERT(0);*/
         }
         // TODO batch optimization:
         // TODO continue scan until meeting batch requirement
-        if (dataRsp.blockNum > 0 /* threshold */) {
+        if (taosxRsp.blockNum > 0 /* threshold */) {
-          tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
+          tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
-          if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+          if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) {
             code = -1;
           }
-          goto OVER;
+          tDeleteSTaosxRsp(&taosxRsp);
+          if (pCkHead) taosMemoryFree(pCkHead);
+          return code;
         } else {
           fetchVer++;
         }
@@ -472,30 +573,22 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
         ASSERT(pHandle->fetchMeta);
         ASSERT(IS_META_MSG(pHead->msgType));
         tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType);
-        SMqMetaRsp metaRsp = {0};
         tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer);
         metaRsp.resMsgType = pHead->msgType;
         metaRsp.metaRspLen = pHead->bodyLen;
         metaRsp.metaRsp = pHead->body;
         if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
           code = -1;
-          goto OVER;
+          taosMemoryFree(pCkHead);
+          return code;
         }
         code = 0;
-        goto OVER;
+        if (pCkHead) taosMemoryFree(pCkHead);
+        return code;
       }
     }
   }
-
+  return 0;
-  // send empty to client
-  if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
-    code = -1;
-  }
-
-OVER:
-  if (pCkHead) taosMemoryFree(pCkHead);
-  tDeleteSMqDataRsp(&dataRsp);
-  return code;
 }
 
 int32_t tqProcessVgDeleteReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
@@ -602,10 +695,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe
       pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode);
       pHandle->execHandle.execDb.pFilterOutTbUid =
           taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
-      buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext **)(&handle.sContext));
+      buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
+                       (SSnapContext**)(&handle.sContext));
 
-      pHandle->execHandle.task =
+      pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL);
-          qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL);
     } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
       pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
 

source/dnode/vnode/src/tq/tqExec.c

@@ -60,7 +60,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) {
   return 0;
 }
 
-int64_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
+int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
   const STqExecHandle* pExec = &pHandle->execHandle;
   ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
 
@@ -89,18 +89,41 @@ int64_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
     if (qExecTask(task, &pDataBlock, &ts) < 0) {
       ASSERT(0);
     }
-    tqDebug("tmq task execute end, get %p", pDataBlock);
+    tqDebug("tmq task executed, get %p", pDataBlock);
 
-    if (pDataBlock) {
+    if (pDataBlock == NULL) {
-      tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
+      break;
-      pRsp->blockNum++;
+    }
+
+    tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
+    pRsp->blockNum++;
+
+    if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+      rowCnt += pDataBlock->info.rows;
+      if (rowCnt >= 4096) break;
     }
   }
 
+  if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) {
+    ASSERT(0);
+    return -1;
+  }
+  ASSERT(pRsp->rspOffset.type != 0);
+
+  if (pRsp->withTbName) {
+    if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) {
+      int64_t uid = pExec->pExecReader->msgIter.uid;
+      tqAddTbNameToRsp(pTq, uid, pRsp);
+    } else {
+      pRsp->withTbName = false;
+    }
+  }
+  ASSERT(pRsp->withSchema == false);
+
   return 0;
 }
 
-int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
+int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
   const STqExecHandle* pExec = &pHandle->execHandle;
   qTaskInfo_t          task = pExec->task;
 
@@ -134,7 +157,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
         int64_t uid = 0;
         if (pOffset->type == TMQ_OFFSET__LOG) {
           uid = pExec->pExecReader->msgIter.uid;
-          if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) {
+          if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp) < 0) {
             continue;
           }
         } else {
@@ -144,18 +167,14 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
       }
       if (pRsp->withSchema) {
         if (pOffset->type == TMQ_OFFSET__LOG) {
-          tqAddBlockSchemaToRsp(pExec, pRsp);
+          tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
         } else {
           SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
           taosArrayPush(pRsp->blockSchema, &pSW);
         }
       }
 
-      if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+      tqAddBlockDataToRsp(pDataBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pDataBlock->pDataBlock));
-        tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
-      } else {
-        tqAddBlockDataToRsp(pDataBlock, pRsp, taosArrayGetSize(pDataBlock->pDataBlock));
-      }
       pRsp->blockNum++;
       if (pOffset->type == TMQ_OFFSET__LOG) {
         continue;
@@ -165,34 +184,32 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
       }
     }
 
-    if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN) {
+    if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-      if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+      if (qStreamExtractPrepareUid(task) != 0) {
-        if (qStreamExtractPrepareUid(task) != 0) {
-          continue;
-        }
-        tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode),
-                pHandle->snapshotVer + 1);
-        break;
-      }
-
-      if (pRsp->blockNum > 0) {
-        tqDebug("tmqsnap task exec exited, get data");
-        break;
-      }
-
-      SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
-      if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-        tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts);
-        qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
-        tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META;
-        tqDebug("tmqsnap task exec change to get data");
         continue;
       }
-
+      tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode),
-      *pMetaRsp = *tmp;
+              pHandle->snapshotVer + 1);
-      tqDebug("tmqsnap task exec exited, get meta");
+      break;
     }
 
+    if (pRsp->blockNum > 0) {
+      tqDebug("tmqsnap task exec exited, get data");
+      break;
+    }
+
+    SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
+    if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
+      tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts);
+      qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
+      tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META;
+      tqDebug("tmqsnap task exec change to get data");
+      continue;
+    }
+
+    *pMetaRsp = *tmp;
+    tqDebug("tmqsnap task exec exited, get meta");
+
     tqDebug("task exec exited");
     break;
   }
@@ -205,12 +222,11 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
   return 0;
 }
 
-int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp* pRsp) {
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp) {
   STqExecHandle* pExec = &pHandle->execHandle;
   ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN);
 
   if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
-    pRsp->withSchema = 1;
     STqReader* pReader = pExec->pExecReader;
     tqReaderSetDataMsg(pReader, pReq, 0);
     while (tqNextDataBlock(pReader)) {
@@ -220,18 +236,17 @@ int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp
       }
       if (pRsp->withTbName) {
         int64_t uid = pExec->pExecReader->msgIter.uid;
-        if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) {
+        if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp) < 0) {
           blockDataFreeRes(&block);
           continue;
         }
       }
-      tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock));
+      tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock));
       blockDataFreeRes(&block);
-      tqAddBlockSchemaToRsp(pExec, pRsp);
+      tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
       pRsp->blockNum++;
     }
   } else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
-    pRsp->withSchema = 1;
     STqReader* pReader = pExec->pExecReader;
     tqReaderSetDataMsg(pReader, pReq, 0);
     while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
@@ -241,24 +256,25 @@ int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp
       }
       if (pRsp->withTbName) {
         int64_t uid = pExec->pExecReader->msgIter.uid;
-        if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) {
+        if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp) < 0) {
           blockDataFreeRes(&block);
           continue;
         }
       }
-      tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock));
+      tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock));
       blockDataFreeRes(&block);
-      tqAddBlockSchemaToRsp(pExec, pRsp);
+      tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);
       pRsp->blockNum++;
     }
-#if 0
+#if 1
     if (pHandle->fetchMeta && pRsp->blockNum) {
       SSubmitMsgIter iter = {0};
       tInitSubmitMsgIter(pReq, &iter);
       STaosxRsp* pXrsp = (STaosxRsp*)pRsp;
       while (1) {
         SSubmitBlk* pBlk = NULL;
-        if (tGetSubmitMsgNext(&iter, &pBlk) < 0) return -1;
+        if (tGetSubmitMsgNext(&iter, &pBlk) < 0) break;
+        if (pBlk == NULL) break;
         if (pBlk->schemaLen > 0) {
           if (pXrsp->createTableNum == 0) {
             pXrsp->createTableLen = taosArrayInit(0, sizeof(int32_t));

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -2039,7 +2039,7 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
   size += pBlockCol->szOffset;
 
   // value
-  if (pColData->flag != (HAS_NULL | HAS_NONE)) {
+  if ((pColData->flag != (HAS_NULL | HAS_NONE)) && pColData->nData) {
     code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, ppOut, nOut + size,
                         &pBlockCol->szValue, ppBuf);
     if (code) goto _exit;

source/libs/executor/inc/executorimpl.h

@@ -143,6 +143,7 @@ typedef struct {
   STqOffsetVal   prepareStatus; // for tmq
   STqOffsetVal   lastStatus;    // for tmq
   SMqMetaRsp     metaRsp;       // for tmq fetching meta
+  int8_t  returned;
   int64_t snapshotVer;
 
   SSchemaWrapper *schema;
@@ -409,6 +410,7 @@ typedef enum EStreamScanMode {
   STREAM_SCAN_FROM_READERHANDLE = 1,
   STREAM_SCAN_FROM_RES,
   STREAM_SCAN_FROM_UPDATERES,
+  STREAM_SCAN_FROM_DELETERES,
   STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
   STREAM_SCAN_FROM_DATAREADER_RANGE,
 } EStreamScanMode;
@@ -437,12 +439,24 @@ typedef struct SStreamAggSupporter {
   SSDataBlock*   pScanBlock;
 } SStreamAggSupporter;
 
-typedef struct SessionWindowSupporter {
+typedef struct SWindowSupporter {
   SStreamAggSupporter* pStreamAggSup;
   int64_t              gap;
   uint16_t             parentType;
   SAggSupporter*       pIntervalAggSup;
-} SessionWindowSupporter;
+} SWindowSupporter;
+
+typedef struct SPartitionBySupporter {
+  SArray*         pGroupCols;     // group by columns, SArray<SColumn>
+  SArray*         pGroupColVals;  // current group column values, SArray<SGroupKeys>
+  char*           keyBuf;         // group by keys for hash
+  bool            needCalc;       // partition by column
+} SPartitionBySupporter;
+
+typedef struct SPartitionDataInfo {
+  uint64_t        groupId;
+  SArray*         rowIds;
+} SPartitionDataInfo;
 
 typedef struct STimeWindowSupp {
   int8_t           calTrigger;
@@ -477,7 +491,9 @@ typedef struct SStreamScanInfo {
   SOperatorInfo*         pStreamScanOp;
   SOperatorInfo*         pTableScanOp;
   SArray*                childIds;
-  SessionWindowSupporter sessionSup;
+  SWindowSupporter       windowSup;
+  SPartitionBySupporter  partitionSup;
+  SExprSupp*             pPartScalarSup;
   bool                   assignBlockUid;  // assign block uid to groupId, temporarily used for generating rollup SMA.
   int32_t                scanWinIndex;    // for state operator
   int32_t                pullDataResIndex;
@@ -690,7 +706,6 @@ typedef struct SPartitionOperatorInfo {
   SArray*        sortedGroupArray;   // SDataGroupInfo sorted by group id
   int32_t        groupIndex;  // group index
   int32_t        pageIndex;   // page index of current group
-  SSDataBlock*   pUpdateRes;
   SExprSupp      scalarSup;
 } SPartitionOperatorInfo;
 
@@ -742,8 +757,8 @@ typedef struct SStreamSessionAggOperatorInfo {
   SSDataBlock*         pWinBlock;       // window result
   SqlFunctionCtx*      pDummyCtx;       // for combine
   SSDataBlock*         pDelRes;         // delete result
-  bool                 returnDelete;
   SSDataBlock*         pUpdateRes;      // update window
+  bool                 returnUpdate;
   SHashObj*            pStDeleted;
   void*                pDelIterator;
   SArray*              pChildren;       // cache for children's result; final stream operator
@@ -752,6 +767,16 @@ typedef struct SStreamSessionAggOperatorInfo {
   bool                 ignoreExpiredData;
 } SStreamSessionAggOperatorInfo;
 
+typedef struct SStreamPartitionOperatorInfo {
+  SOptrBasicInfo        binfo;
+  SPartitionBySupporter partitionSup;
+  SExprSupp             scalarSup;
+  SHashObj*             pPartitions;
+  void*                 parIte;
+  SSDataBlock*          pInputDataBlock;
+  int32_t               tsColIndex;
+} SStreamPartitionOperatorInfo;
+
 typedef struct STimeSliceOperatorInfo {
   SSDataBlock*            pRes;
   STimeWindow             win;
@@ -953,6 +978,9 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
+
+SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
+
 SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode,
                                            SExecTaskInfo* pTaskInfo);
@@ -1021,8 +1049,9 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx);
 bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
 bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
 bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
-void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, int32_t uidCol, uint64_t* pID);
+void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp);
 void printDataBlock(SSDataBlock* pBlock, const char* flag);
+uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
 
 int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition,
                                        SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset,

source/libs/executor/src/executor.c

@@ -745,6 +745,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
   SOperatorInfo* pOperator = pTaskInfo->pRoot;
   ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
   pTaskInfo->streamInfo.prepareStatus = *pOffset;
+  pTaskInfo->streamInfo.returned = 0;
   if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) {
     return 0;
   }

source/libs/executor/src/executorimpl.c

@@ -4150,6 +4150,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children);
   } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) {
     pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo);
+  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION == type) {
+    pOptr = createStreamPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE == type) {
     SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode;
     pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo);

source/libs/executor/src/groupoperator.c

@@ -830,3 +830,205 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo,
   setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset);
   return TSDB_CODE_SUCCESS;
 }
+
+uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId) {
+  if (pExprSup->pExprInfo != NULL) {
+    int32_t code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+    if (code != TSDB_CODE_SUCCESS) {
+      qError("calaculate group id error, code:%d", code);
+    }
+  }
+  recordNewGroupKeys(pParSup->pGroupCols, pParSup->pGroupColVals, pBlock, rowId);
+  int32_t len = buildGroupKeys(pParSup->keyBuf, pParSup->pGroupColVals);
+  uint64_t groupId = calcGroupId(pParSup->keyBuf, len);
+  return groupId;
+}
+
+static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) {
+  return pInfo->parIte != NULL;
+}
+
+static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
+  SStreamPartitionOperatorInfo* pInfo = pOperator->info;
+  SSDataBlock* pDest = pInfo->binfo.pRes;
+  ASSERT(hasRemainPartion(pInfo));
+  SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte;
+  blockDataCleanup(pDest);
+  int32_t rows = taosArrayGetSize(pParInfo->rowIds);
+  SSDataBlock* pSrc = pInfo->pInputDataBlock;
+  for (int32_t i = 0; i < rows; i++) {
+    int32_t rowIndex = *(int32_t*)taosArrayGet(pParInfo->rowIds, i);
+    for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; j++) {
+      int32_t slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId;
+      SColumnInfoData* pSrcCol = taosArrayGet(pSrc->pDataBlock, slotId);
+      SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, j);
+      bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL);
+      char* pSrcData = colDataGetData(pSrcCol, rowIndex);
+      colDataAppend(pDestCol, pDest->info.rows, pSrcData, isNull);
+    }
+    pDest->info.rows++;
+  }
+  blockDataUpdateTsWindow(pDest, pInfo->tsColIndex);
+  pDest->info.groupId = pParInfo->groupId;
+  pOperator->resultInfo.totalRows += pDest->info.rows;
+  pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte);
+  ASSERT(pDest->info.rows > 0);
+  printDataBlock(pDest, "stream partitionby");
+  return pDest;
+}
+
+static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) {
+  pInfo->pInputDataBlock = pBlock;
+  for (int32_t i = 0; i < pBlock->info.rows; ++i) {
+    recordNewGroupKeys(pInfo->partitionSup.pGroupCols, pInfo->partitionSup.pGroupColVals, pBlock, i);
+    int32_t keyLen = buildGroupKeys(pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupColVals);
+    SPartitionDataInfo* pParData = 
+        (SPartitionDataInfo*) taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen);
+    if (pParData) {
+      taosArrayPush(pParData->rowIds, &i);
+    } else {
+      SPartitionDataInfo newParData = {0};
+      newParData.groupId = calcGroupId(pInfo->partitionSup.keyBuf, keyLen);
+      newParData.rowIds = taosArrayInit(64, sizeof(int32_t));
+      taosArrayPush(newParData.rowIds, &i);
+      taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData,
+          sizeof(SPartitionDataInfo));
+    }
+  }
+}
+
+static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
+  if (pOperator->status == OP_EXEC_DONE) {
+    return NULL;
+  }
+
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SStreamPartitionOperatorInfo* pInfo = pOperator->info;
+  if (hasRemainPartion(pInfo)) {
+    return buildStreamPartitionResult(pOperator);
+  }
+
+  int64_t st = taosGetTimestampUs();
+  SOperatorInfo* downstream = pOperator->pDownstream[0];
+  {
+    pInfo->pInputDataBlock = NULL;
+    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
+    if (pBlock == NULL) {
+      doSetOperatorCompleted(pOperator);
+      return NULL;
+    }
+    printDataBlock(pBlock, "stream partitionby recv");
+    switch (pBlock->info.type) {
+      case STREAM_NORMAL:
+      case STREAM_PULL_DATA:
+      case STREAM_INVALID:
+        pInfo->binfo.pRes->info.type = pBlock->info.type;
+        break;
+      default:
+        return pBlock;
+    }
+
+    // there is an scalar expression that needs to be calculated right before apply the group aggregation.
+    if (pInfo->scalarSup.pExprInfo != NULL) {
+      pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock,
+          pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL);
+      if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
+        longjmp(pTaskInfo->env, pTaskInfo->code);
+      }
+    }
+    taosHashClear(pInfo->pPartitions);
+    doStreamHashPartitionImpl(pInfo, pBlock);
+  }
+  pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
+  
+  pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL);
+  return buildStreamPartitionResult(pOperator);
+}
+
+static void destroyStreamPartitionOperatorInfo(void* param) {
+  SStreamPartitionOperatorInfo* pInfo = (SStreamPartitionOperatorInfo*)param;
+  cleanupBasicInfo(&pInfo->binfo);
+  taosArrayDestroy(pInfo->partitionSup.pGroupCols);
+
+  for(int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++){
+    SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->partitionSup.pGroupColVals, i);
+    taosMemoryFree(key.pData);
+  }
+  taosArrayDestroy(pInfo->partitionSup.pGroupColVals);
+
+  taosMemoryFree(pInfo->partitionSup.keyBuf);
+  cleanupExprSupp(&pInfo->scalarSup);
+  taosMemoryFreeClear(param);
+}
+
+void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) {
+  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+    return;
+  }
+  SStreamScanInfo* pScanInfo = downstream->info;
+  pScanInfo->partitionSup = *pParSup;
+  pScanInfo->pPartScalarSup = pExpr;
+}
+
+SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) {
+  SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo));
+  SOperatorInfo*                pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  if (pInfo == NULL || pOperator == NULL) {
+    goto _error;
+  }
+  int32_t code = TSDB_CODE_SUCCESS;
+  pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys);
+
+  if (pPartNode->pExprs != NULL) {
+    int32_t num = 0;
+    SExprInfo* pCalExprInfo = createExprInfo(pPartNode->pExprs, NULL, &num);
+    code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _error;
+    }
+  }
+
+  int32_t keyLen = 0;
+  code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+  pInfo->partitionSup.needCalc = true;
+
+  SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc);
+  if (!pResBlock) {
+    goto _error;
+  }
+  blockDataEnsureCapacity(pResBlock, 4096);
+  pInfo->binfo.pRes       = pResBlock;
+  pInfo->parIte           = NULL;
+  pInfo->pInputDataBlock  = NULL;
+  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+  pInfo->pPartitions      = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
+  pInfo->tsColIndex       = 0;
+
+  int32_t numOfCols = 0;
+  SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols);
+
+  pOperator->name         = "StreamPartitionOperator";
+  pOperator->blocking     = false;
+  pOperator->status       = OP_NOT_OPENED;
+  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION;
+  pOperator->exprSupp.numOfExprs   = numOfCols;
+  pOperator->exprSupp.pExprInfo    = pExprInfo;
+  pOperator->info         = pInfo;
+  pOperator->pTaskInfo    = pTaskInfo;
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, destroyStreamPartitionOperatorInfo,
+                                         NULL, NULL, NULL);
+
+  initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup);
+  code = appendDownstream(pOperator, &downstream, 1);
+  return pOperator;
+
+  _error:
+  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+  taosMemoryFreeClear(pInfo);
+  taosMemoryFreeClear(pOperator);
+  return NULL;
+}
+

source/libs/executor/src/scanoperator.c

@@ -920,49 +920,28 @@ static void doClearBufferedBlocks(SStreamScanInfo* pInfo) {
 }
 
 static bool isSessionWindow(SStreamScanInfo* pInfo) {
-  return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
+  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
-         pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
+         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
 }
 
 static bool isStateWindow(SStreamScanInfo* pInfo) {
-  return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
+  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
 }
 
 static bool isIntervalWindow(SStreamScanInfo* pInfo) {
-  return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL ||
+  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL ||
-         pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
+         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
-         pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
+         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
 }
 
 static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) {
-  return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
+  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
 }
 
 static bool isSlidingWindow(SStreamScanInfo* pInfo) {
   return isIntervalWindow(pInfo) && pInfo->interval.interval != pInfo->interval.sliding;
 }
 
-static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) {
-  uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t));
-  if (groupId) {
-    return *groupId;
-  }
-  return 0;
-  /* Todo(liuyao) for partition by column
-  recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, rowId);
-  int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals);
-  uint64_t resId = 0;
-  uint64_t* groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len);
-  if (groupId) {
-    return *groupId;
-  } else if (len != 0) {
-    resId = calcGroupId(pTableScanInfo->keyBuf, len);
-    taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &resId, sizeof(uint64_t));
-  }
-  return resId;
-  */
-}
-
 static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
   SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
   uint64_t*        groupCol = (uint64_t*)pColInfo->pData;
@@ -976,6 +955,62 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
   pTableScanInfo->currentGroupId = -1;
 }
 
+static void freeArray(void* array) {
+  taosArrayDestroy(array);
+}
+
+static void resetTableScanOperator(SOperatorInfo* pTableScanOp) {
+  STableScanInfo* pTableScanInfo = pTableScanOp->info;
+  pTableScanInfo->cond.startVersion = -1;
+  pTableScanInfo->cond.endVersion = -1;
+  SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList;
+  SArray* allTbls = pTableScanOp->pTaskInfo->tableqinfoList.pTableList;
+  taosArrayClearP(gpTbls, freeArray);
+  taosArrayPush(gpTbls, &allTbls);
+  STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX};
+  resetTableScanInfo(pTableScanOp->info, &win);
+}
+
+static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, int64_t maxVersion) {
+  SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList;
+  taosArrayClear(gpTbls);
+  STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0};
+  SArray* tbls = taosArrayInit(1, sizeof(STableKeyInfo));
+  taosArrayPush(tbls, &tblInfo);
+  taosArrayPush(gpTbls, &tbls);
+
+  STimeWindow win = {.skey = startTs, .ekey = endTs};
+  STableScanInfo* pTableScanInfo = pTableScanOp->info;
+  pTableScanInfo->cond.startVersion = -1;
+  pTableScanInfo->cond.endVersion = maxVersion;
+  resetTableScanInfo(pTableScanOp->info, &win);
+  SSDataBlock* pRes = doTableScan(pTableScanOp);
+  resetTableScanOperator(pTableScanOp);
+  return pRes;
+}
+
+static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
+  SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uid, ts, ts, maxVersion);
+  if (!pPreRes || pPreRes->info.rows == 0) {
+    return 0;
+  }
+  ASSERT(pPreRes->info.rows == 1);
+  return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0);
+}
+
+static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) {
+  if (pInfo->partitionSup.needCalc) {
+    return getGroupIdByCol(pInfo, uid, ts, maxVersion);
+  }
+
+  SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map;
+  uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t));
+  if (groupId) {
+    return *groupId;
+  }
+  return 0;
+}
+
 static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
   if ((*pRowIndex) == pBlock->info.rows) {
     return false;
@@ -987,6 +1022,9 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
   SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
   TSKEY*           endData = (TSKEY*)pEndTsCol->pData;
   STimeWindow      win = {.skey = startData[*pRowIndex], .ekey = endData[*pRowIndex]};
+  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+  uint64_t*        gpData = (uint64_t*)pGpCol->pData;
+  uint64_t         groupId = gpData[*pRowIndex];
 
   SColumnInfoData* pCalStartTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
   TSKEY*           calStartData = (TSKEY*)pCalStartTsCol->pData;
@@ -1001,11 +1039,11 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
   (*pRowIndex)++;
 
   for (; *pRowIndex < pBlock->info.rows; (*pRowIndex)++) {
-    if (win.skey == startData[*pRowIndex]) {
+    if (win.skey == startData[*pRowIndex] && groupId == gpData[*pRowIndex]) {
       win.ekey = TMAX(win.ekey, endData[*pRowIndex]);
       continue;
     }
-    if (win.skey == endData[*pRowIndex]) {
+    if (win.skey == endData[*pRowIndex] && groupId == gpData[*pRowIndex]) {
       win.skey = TMIN(win.skey, startData[*pRowIndex]);
       continue;
     }
@@ -1020,15 +1058,19 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
 }
 
 static STimeWindow getSlidingWindow(TSKEY* tsCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo,
-                                    int32_t* pRowIndex) {
+                                    int32_t* pRowIndex, bool hasGroup) {
   SResultRowInfo dumyInfo;
   dumyInfo.cur.pageId = -1;
   STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC);
   STimeWindow endWin = win;
   STimeWindow preWin = win;
   while (1) {
-    (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey, binarySearchForKey, NULL,
+    if (hasGroup) {
-                                             TSDB_ORDER_ASC);
+      (*pRowIndex) += 1;
+    } else {
+      (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey,
+          binarySearchForKey, NULL, TSDB_ORDER_ASC);
+    }
     do {
       preWin = endWin;
       getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC);
@@ -1060,7 +1102,26 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
       return NULL;
     }
 
-    if (pResult->info.groupId == pInfo->groupId) {
+    if (pInfo->partitionSup.needCalc) {
+      SSDataBlock* tmpBlock = createOneDataBlock(pResult, true);
+      blockDataCleanup(pResult);
+      for (int32_t i = 0; i < tmpBlock->info.rows; i++) {
+        if (calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, tmpBlock, i) == pInfo->groupId) {
+          for (int32_t j = 0; j < pInfo->pTableScanOp->exprSupp.numOfExprs; j++) {
+            SColumnInfoData* pSrcCol = taosArrayGet(tmpBlock->pDataBlock, j);
+            SColumnInfoData* pDestCol = taosArrayGet(pResult->pDataBlock, j);
+            bool isNull = colDataIsNull(pSrcCol, tmpBlock->info.rows, i, NULL);
+            char* pSrcData = colDataGetData(pSrcCol, i);
+            colDataAppend(pDestCol, pResult->info.rows, pSrcData, isNull);
+          }
+          pResult->info.rows++;
+        }
+      }
+      if (pResult->info.rows > 0) {
+        pResult->info.calWin = pInfo->updateWin;
+        return pResult;
+      }
+    } else if (pResult->info.groupId == pInfo->groupId) {
       pResult->info.calWin = pInfo->updateWin;
       return pResult;
     }
@@ -1091,17 +1152,18 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr
   SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
   SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
   int32_t          dummy = 0;
+  int64_t version = pSrcBlock->info.version - 1;
   for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
-    uint64_t groupId = getGroupId(pInfo->pTableScanOp, uidCol[i]);
+    uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version);
     // gap must be 0.
     SResultWindowInfo* pStartWin =
-        getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy);
+        getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy);
     if (!pStartWin) {
       // window has been closed.
       continue;
     }
     SResultWindowInfo* pEndWin =
-        getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy);
+        getCurSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy);
     ASSERT(pEndWin);
     TSKEY ts = INT64_MIN;
     colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false);
@@ -1121,34 +1183,49 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
   if (rows == 0) {
     return TSDB_CODE_SUCCESS;
   }
-  int32_t code = blockDataEnsureCapacity(pDestBlock, rows);
+  int32_t code = blockDataEnsureCapacity(pDestBlock, rows * 2);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
 
-  SColumnInfoData* pTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pSrcTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
-  SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
+  SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
-  uint64_t*        uidCol = (uint64_t*)pUidCol->pData;
+  uint64_t*        srcUidData = (uint64_t*)pSrcUidCol->pData;
-  ASSERT(pTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
+  SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX);
-  TSKEY*           tsCol = (TSKEY*)pTsCol->pData;
+  uint64_t*        srcGp = (uint64_t*)pSrcGpCol->pData;
+  ASSERT(pSrcTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
+  TSKEY*           tsCol = (TSKEY*)pSrcTsCol->pData;
   SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
   SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  SColumnInfoData* pDeUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX);
   SColumnInfoData* pGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
   SColumnInfoData* pCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
   SColumnInfoData* pCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
-  uint64_t         groupId = getGroupId(pInfo->pTableScanOp, uidCol[0]);
+  int64_t version = pSrcBlock->info.version - 1;
   for (int32_t i = 0; i < rows;) {
-    colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(tsCol + i), false);
+    uint64_t srcUid = srcUidData[i];
-    STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i);
+    uint64_t groupId = getGroupIdByData(pInfo, srcUid, tsCol[i], version);
-    colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(tsCol + i - 1), false);
+    uint64_t srcGpId = srcGp[i];
-
+    TSKEY calStartTs = tsCol[i];
+    colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
+    STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i, pInfo->partitionSup.needCalc);
+    TSKEY calEndTs = tsCol[i - 1];
+    colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
+    colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false);
     colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
     colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
     colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false);
     pDestBlock->info.rows++;
+    if (pInfo->partitionSup.needCalc && srcGpId != 0 && groupId != srcGpId) {
+      colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
+      colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
+      colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false);
+      colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
+      colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
+      colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&srcGpId), false);
+      pDestBlock->info.rows++;
+    }
   }
-  // all rows have same group id
-  pDestBlock->info.groupId = groupId;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1160,17 +1237,20 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
     code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock);
   }
   pDestBlock->info.type = STREAM_CLEAR;
+  pDestBlock->info.version = pSrcBlock->info.version;
   blockDataUpdateTsWindow(pDestBlock, 0);
   return code;
 }
 
-void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, int32_t uidCol, uint64_t* pID) {
+void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp) {
   SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
   SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
-  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, uidCol);
+  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
+  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
   colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)pStartTs, false);
   colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false);
-  colDataAppend(pUidCol, pBlock->info.rows, (const char*)pID, false);
+  colDataAppend(pUidCol, pBlock->info.rows, (const char*)pUid, false);
+  colDataAppend(pGpCol, pBlock->info.rows, (const char*)pGp, false);
   pBlock->info.rows++;
 }
 
@@ -1195,24 +1275,18 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
     // must check update info first.
     bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]);
     bool closedWin = isClosed && isSignleIntervalWindow(pInfo) &&
-                     isDeletedWindow(&win, pBlock->info.groupId, pInfo->sessionSup.pIntervalAggSup);
+                     isDeletedWindow(&win, pBlock->info.groupId, pInfo->windowSup.pIntervalAggSup);
     if ((update || closedWin) && out) {
-      appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, UID_COLUMN_INDEX, &pBlock->info.uid);
+      uint64_t gpId = closedWin&&pInfo->partitionSup.needCalc ?
+          calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId) : 0;
+      appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid,
+          &gpId);
     }
   }
-  if (out) {
+  if (out && pInfo->pUpdateDataRes->info.rows > 0) {
+    pInfo->pUpdateDataRes->info.version = pBlock->info.version;
     blockDataUpdateTsWindow(pInfo->pUpdateDataRes, 0);
-    pInfo->pUpdateDataRes->info.type = STREAM_CLEAR;
+    pInfo->pUpdateDataRes->info.type = pInfo->partitionSup.needCalc ? STREAM_DELETE_DATA : STREAM_CLEAR;
-  }
-}
-
-static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t uidColIndex) {
-  ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3);
-  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, uidColIndex);
-  uint64_t*        uidCol = (uint64_t*)pColDataInfo->pData;
-  ASSERT(pBlock->info.rows > 0);
-  for (int32_t i = 0; i < pBlock->info.rows; i++) {
-    uidCol[i] = getGroupId(pOperator, uidCol[i]);
   }
 }
 
@@ -1285,17 +1359,22 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
     SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
     if (pResult && pResult->info.rows > 0) {
       qDebug("queue scan tsdb return %d rows", pResult->info.rows);
+      pTaskInfo->streamInfo.returned = 1;
       return pResult;
     } else {
-      STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
+      if (!pTaskInfo->streamInfo.returned) {
-      tsdbReaderClose(pTSInfo->dataReader);
+        STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
-      pTSInfo->dataReader = NULL;
+        tsdbReaderClose(pTSInfo->dataReader);
-      tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
+        pTSInfo->dataReader = NULL;
-      qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1);
+        tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer);
-      if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) {
+        qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1);
+        if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) {
+          return NULL;
+        }
+        ASSERT(pInfo->tqReader->pWalReader->curVersion == pTaskInfo->streamInfo.snapshotVer + 1);
+      } else {
         return NULL;
       }
-      ASSERT(pInfo->tqReader->pWalReader->curVersion == pTaskInfo->streamInfo.snapshotVer + 1);
     }
   }
 
@@ -1442,6 +1521,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
       default:
         break;
     }
+    // printDataBlock(pBlock, "stream scan recv");
     return pBlock;
   } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
     qDebug("scan mode %d", pInfo->scanMode);
@@ -1451,6 +1531,14 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
         pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
         return pInfo->pRes;
       } break;
+      case STREAM_SCAN_FROM_DELETERES: {
+        generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
+        prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
+        pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
+        copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
+        pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA;
+        return pInfo->pDeleteDataRes;
+      } break;
       case STREAM_SCAN_FROM_UPDATERES: {
         generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
         prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
@@ -1466,6 +1554,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
           updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->cond.twindows, pInfo->groupId, version);
           pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA;
           checkUpdateData(pInfo, true, pSDB, false);
+          // printDataBlock(pSDB, "stream scan update");
           return pSDB;
         }
         pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
@@ -1474,7 +1563,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
         break;
     }
 
-    SStreamAggSupporter* pSup = pInfo->sessionSup.pStreamAggSup;
+    SStreamAggSupporter* pSup = pInfo->windowSup.pStreamAggSup;
     if (isStateWindow(pInfo) && pSup->pScanBlock->info.rows > 0) {
       pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
       pInfo->updateResIndex = 0;
@@ -1540,7 +1629,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
     // record the scan action.
     pInfo->numOfExec++;
     pOperator->resultInfo.totalRows += pBlockInfo->rows;
-    printDataBlock(pInfo->pRes, "stream scan");
+    // printDataBlock(pInfo->pRes, "stream scan");
 
     if (pBlockInfo->rows == 0) {
       updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
@@ -1549,19 +1638,20 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
       checkUpdateData(pInfo, true, pInfo->pRes, true);
       pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey);
       if (pInfo->pUpdateDataRes->info.rows > 0) {
+        pInfo->updateResIndex = 0;
         if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) {
-          pInfo->updateResIndex = 0;
           pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
         } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) {
           pInfo->scanMode = STREAM_SCAN_FROM_RES;
           return pInfo->pUpdateDataRes;
+        } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) {
+          pInfo->scanMode = STREAM_SCAN_FROM_DELETERES;
         }
       }
     }
 
     qDebug("scan rows: %d", pBlockInfo->rows);
     return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
-
   } else {
     ASSERT(0);
     return NULL;
@@ -1825,13 +1915,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
       ASSERT(pHandle->tqReader);
       pInfo->tqReader = pHandle->tqReader;
     }
-
+    
-    if (pTSInfo->pdInfo.interval.interval > 0) {
+    pInfo->pUpdateInfo = NULL;
-      pInfo->pUpdateInfo = updateInfoInitP(&pTSInfo->pdInfo.interval, pInfo->twAggSup.waterMark);
-    } else {
-      pInfo->pUpdateInfo = NULL;
-    }
-
     pInfo->pTableScanOp = pTableScanOp;
     pInfo->interval = pTSInfo->pdInfo.interval;
 
@@ -1862,8 +1947,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
   pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
   pInfo->pCondition = pScanPhyNode->node.pConditions;
   pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
-  pInfo->sessionSup =
+  pInfo->windowSup =
-      (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN};
+      (SWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN};
   pInfo->groupId = 0;
   pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
   pInfo->pStreamScanOp = pOperator;
@@ -1872,6 +1957,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
   pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX};
   pInfo->pUpdateDataRes = createSpecialDataBlock(STREAM_CLEAR);
   pInfo->assignBlockUid = pTableScanNode->assignBlockUid;
+  pInfo->partitionSup.needCalc = false;
 
   pOperator->name = "StreamScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;

source/libs/executor/src/timewindowoperator.c

@@ -897,7 +897,7 @@ int64_t getWinReskey(void* data, int32_t index) {
 
 int32_t compareWinRes(void* pKey, void* data, int32_t index) {
   SArray*     res = (SArray*)data;
-  SWinKey*    pos = taosArrayGetP(res, index);
+  SWinKey*    pos = taosArrayGet(res, index);
   SResKeyPos* pData = (SResKeyPos*)pKey;
   if (*(int64_t*)pData->key == pos->ts) {
     if (pData->groupId > pos->groupId) {
@@ -919,10 +919,11 @@ static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
   }
   void* pIte = NULL;
   while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
-    SResKeyPos* pResKey = (SResKeyPos*)pIte;
+    SResKeyPos* pResKey = *(SResKeyPos**)pIte;
     int32_t     index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes);
     if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) {
       taosArrayRemove(pDelWins, index);
+      delSize = taosArrayGetSize(pDelWins);
     }
   }
 }
@@ -1423,7 +1424,7 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId)
   return true;
 }
 
-void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pUpWins, SInterval* pInterval) {
+void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pDelWins, SInterval* pInterval, SHashObj* pUpdatedMap) {
   SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
   TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
   SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
@@ -1433,9 +1434,12 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock,
     dumyInfo.cur.pageId = -1;
     STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC);
     doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
-    if (pUpWins) {
+    SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
-      SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
+    if (pDelWins) {
-      taosArrayPush(pUpWins, &winRes);
+      taosArrayPush(pDelWins, &winRes);
+    }
+    if (pUpdatedMap) {
+      taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
     }
   }
 }
@@ -1446,19 +1450,14 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval*
   TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
   SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
   TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
-  uint64_t*        pGpDatas = NULL;
+  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
-  if (pBlock->info.type == STREAM_RETRIEVE) {
+  uint64_t*        pGpDatas = (uint64_t*)pGpCol->pData;
-    SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
-    pGpDatas = (uint64_t*)pGpCol->pData;
-  }
-  int32_t step = 0;
-  int32_t startPos = 0;
   for (int32_t i = 0; i < pBlock->info.rows; i++) {
     SResultRowInfo dumyInfo;
     dumyInfo.cur.pageId = -1;
     STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
     while (win.ekey <= endTsCols[i]) {
-      uint64_t winGpId = pGpDatas ? pGpDatas[startPos] : pBlock->info.groupId;
+      uint64_t winGpId = pGpDatas[i];
       bool     res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput);
       if (pUpWins && res) {
         SWinKey winRes = {.ts = win.skey, .groupId = winGpId};
@@ -1571,13 +1570,10 @@ static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlo
     return;
   }
   blockDataEnsureCapacity(pBlock, size - *index);
-  SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  uint64_t uid = 0;
-  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
   for (int32_t i = *index; i < size; i++) {
     SWinKey* pWin = taosArrayGet(pWins, i);
-    colDataAppend(pTsCol, pBlock->info.rows, (const char*)&pWin->ts, false);
+    appendOneRow(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId);
-    colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&pWin->groupId, false);
-    pBlock->info.rows++;
     (*index)++;
   }
 }
@@ -1596,6 +1592,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_RES_TO_RETURN) {
     doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
     if (pInfo->pDelRes->info.rows > 0) {
+      printDataBlock(pInfo->pDelRes, "single interval");
       return pInfo->pDelRes;
     }
 
@@ -1632,7 +1629,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
       continue;
     }
     if (pBlock->info.type == STREAM_DELETE_DATA) {
-      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
+      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap);
       continue;
     } else if (pBlock->info.type == STREAM_GET_ALL) {
       getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
@@ -1707,6 +1704,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   taosHashCleanup(pUpdatedMap);
   doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
   if (pInfo->pDelRes->info.rows > 0) {
+    printDataBlock(pInfo->pDelRes, "single interval");
     return pInfo->pDelRes;
   }
 
@@ -1828,14 +1826,16 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
   return needed;
 }
 
-void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup) {
+void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval, int64_t waterMark) {
   if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
-    // Todo(liuyao) support partition by column
+    initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, waterMark);
     return;
   }
   SStreamScanInfo* pScanInfo = downstream->info;
-  pScanInfo->sessionSup.parentType = type;
+  pScanInfo->windowSup.parentType = type;
-  pScanInfo->sessionSup.pIntervalAggSup = pSup;
+  pScanInfo->windowSup.pIntervalAggSup = pSup;
+  pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, waterMark);
+  pScanInfo->interval = *pInterval;
 }
 
 void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
@@ -1921,7 +1921,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
                                          destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
 
   if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) {
-    initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup);
+    initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
   }
 
   code = appendDownstream(pOperator, &downstream, 1);
@@ -2849,14 +2849,26 @@ _error:
 }
 
 void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
-                      SExecTaskInfo* pTaskInfo) {
+                      SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) {
   for (int32_t k = 0; k < numOfOutput; ++k) {
     if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {
-      continue;
+      if (!pTimeWindowData) {
-    }
+        continue;
-    int32_t code = TSDB_CODE_SUCCESS;
+      }
-    if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
+
-      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
+      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);
+      char* p = GET_ROWCELL_INTERBUF(pEntryInfo);
+      SColumnInfoData idata = {0};
+      idata.info.type = TSDB_DATA_TYPE_BIGINT;
+      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
+      idata.pData = p;
+
+      SScalarParam out = {.columnData = &idata};
+      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};
+      pDestCtx[k].sfp.process(&tw, 1, &out);
+      pEntryInfo->numOfRes = 1;
+    }else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {
+      int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);
       if (code != TSDB_CODE_SUCCESS) {
         qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code));
         pTaskInfo->code = code;
@@ -2874,8 +2886,14 @@ bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) {
   return p1 != NULL;
 }
 
+STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
+  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
+  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
+  return w;
+}
+
 static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray,
-                                  int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) {
+                                  int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pUpdatedMap) {
   int32_t size = taosArrayGetSize(pWinArray);
   if (!pInfo->pChildren) {
     return;
@@ -2883,11 +2901,14 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr
   for (int32_t i = 0; i < size; i++) {
     SWinKey*    pWinRes = taosArrayGet(pWinArray, i);
     SResultRow* pCurResult = NULL;
-    STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts + 1};
+    STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval);
-    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
+    if (isDeletedWindow(&parentWin, pWinRes->groupId, &pInfo->aggSup) && isCloseWindow(&parentWin, &pInfo->twAggSup)) {
+      continue;
+    }
+    setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &parentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx,
                            numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
     int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
-    bool    find = true;
+    int32_t num = 0;
     for (int32_t j = 0; j < numOfChildren; j++) {
       SOperatorInfo*            pChildOp = taosArrayGetP(pInfo->pChildren, j);
       SIntervalAggOperatorInfo* pChInfo = pChildOp->info;
@@ -2895,15 +2916,16 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr
       if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) {
         continue;
       }
-      find = true;
+      num++;
       SResultRow* pChResult = NULL;
-      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId,
+      setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &parentWin, true, &pChResult, pWinRes->groupId,
                              pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup,
                              pTaskInfo);
-      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo);
+      updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true);
+      compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
     }
-    if (find && pUpdated) {
+    if (num > 1 && pUpdatedMap) {
-      saveResultRow(pCurResult, pWinRes->groupId, pUpdated);
+      saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap);
       setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur);
     }
   }
@@ -2934,12 +2956,6 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {
 
 static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }
 
-STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {
-  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};
-  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
-  return w;
-}
-
 static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
                            SHashObj* pUpdatedMap) {
   SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
@@ -3126,6 +3142,25 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
   }
 }
 
+static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pInfo) {
+  int32_t size = taosArrayGetSize(wins);
+  for (int32_t i = 0; i < size; i++) {
+    SWinKey* winKey = taosArrayGet(wins, i);
+    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
+    if (isCloseWindow(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) {
+      void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
+      if (!chIds) {
+        SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId};
+        // add pull data request
+        savePullWindow(&pull, pInfo->pPullWins);
+        int32_t size = taosArrayGetSize(pInfo->pChildren);
+        addPullWindow(pInfo->pPullDataMap, winKey, size);
+        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size);
+      }
+    }
+  }
+}
+
 static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
 
@@ -3150,12 +3185,20 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       return pInfo->pPullDataRes;
     }
 
+    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
+    if (pInfo->pDelRes->info.rows != 0) {
+      // process the rest of the data
+      printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
+      return pInfo->pDelRes;
+    }
+
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
     if (pInfo->binfo.pRes->info.rows == 0) {
       pOperator->status = OP_EXEC_DONE;
       if (!IS_FINAL_OP(pInfo)) {
         // semi interval operator clear disk buffer
         clearStreamIntervalOperator(pInfo);
+        qDebug("===stream===clear semi operator");
       } else {
         freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
       }
@@ -3219,23 +3262,28 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       }
       removeResults(pUpWins, pUpdatedMap);
       copyDataBlock(pInfo->pUpdateRes, pBlock);
-      // copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
       pInfo->returnUpdate = true;
       taosArrayDestroy(pUpWins);
       break;
     } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
-      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval);
+      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
+      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, delWins, &pInfo->interval, pUpdatedMap);
       if (IS_FINAL_OP(pInfo)) {
         int32_t                           childIndex = getChildIndex(pBlock);
         SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
         SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info;
         SExprSupp*                        pChildSup = &pChildOp->exprSupp;
-        doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval);
+        doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval, NULL);
-        rebuildIntervalWindow(pInfo, pSup, pInfo->pDelWins, pInfo->binfo.pRes->info.groupId,
+        rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId,
-                              pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated);
+                              pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdatedMap);
+        addRetriveWindow(delWins, pInfo);
+        taosArrayAddAll(pInfo->pDelWins, delWins);
+        taosArrayDestroy(delWins);
         continue;
       }
-      removeResults(pInfo->pDelWins, pUpdatedMap);
+      removeResults(delWins, pUpdatedMap);
+      taosArrayAddAll(pInfo->pDelWins, delWins);
+      taosArrayDestroy(delWins);
       break;
     } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) {
       getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
@@ -3309,6 +3357,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
     return pInfo->pPullDataRes;
   }
 
+  // we should send result first.
   doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
   if (pInfo->binfo.pRes->info.rows != 0) {
     printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
@@ -3470,7 +3519,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
       createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                           aggEncodeResultRow, aggDecodeResultRow, NULL);
   if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
-    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup);
+    initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
   }
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
@@ -3567,10 +3616,18 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
 }
 
 void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
-                    uint16_t type) {
+                    uint16_t type, int32_t tsColIndex) {
-  ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
+  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
+    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
+    pScanInfo->tsColIndex = tsColIndex;
+  }
+
+  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+    initDownStream(downstream->pDownstream[0], pAggSup, gap, waterMark, type, tsColIndex);
+    return;
+  }
   SStreamScanInfo* pScanInfo = downstream->info;
-  pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
+  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};
   pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark);
 }
 
@@ -3642,7 +3699,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->isFinal = false;
   pInfo->pPhyNode = pPhyNode;
   pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
-  pInfo->returnDelete = false;
 
   pOperator->name = "StreamSessionWindowAggOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
@@ -3653,7 +3709,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
       createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo,
                           aggEncodeResultRow, aggDecodeResultRow, NULL);
   if (downstream) {
-    initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType);
+    initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex);
     code = appendDownstream(pOperator, &downstream, 1);
   }
   return pOperator;
@@ -3683,13 +3739,13 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {
 
 bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); }
 
-static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) {
+static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) {
-  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
+  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
   return taosArrayInsert(pWinInfos, index, &win);
 }
 
-static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) {
+static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) {
-  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false};
+  SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false};
   return taosArrayPush(pWinInfos, &win);
 }
 
@@ -3748,7 +3804,7 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY star
   int32_t size = taosArrayGetSize(pWinInfos);
   if (size == 0) {
     *pIndex = 0;
-    return addNewSessionWindow(pWinInfos, startTs);
+    return addNewSessionWindow(pWinInfos, startTs, endTs);
   }
   // find the first position which is smaller than the key
   int32_t            index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey);
@@ -3774,10 +3830,10 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY star
 
   if (index == size - 1) {
     *pIndex = taosArrayGetSize(pWinInfos);
-    return addNewSessionWindow(pWinInfos, startTs);
+    return addNewSessionWindow(pWinInfos, startTs, endTs);
   }
   *pIndex = index + 1;
-  return insertNewSessionWindow(pWinInfos, startTs, index + 1);
+  return insertNewSessionWindow(pWinInfos, startTs, endTs, index + 1);
 }
 
 int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId,
@@ -3789,7 +3845,7 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TS
     if (pWinInfo->win.skey > pStartTs[i]) {
       if (pStDeleted && pWinInfo->isOutput) {
         SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
-        taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey));
+        taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
         pWinInfo->isOutput = false;
       }
       pWinInfo->win.skey = pStartTs[i];
@@ -3904,11 +3960,12 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex,
     setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
                        &pInfo->streamAggSup, pTaskInfo);
     pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey);
-    compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo);
+    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true);
+    compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
     taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition));
     if (pWinInfo->isOutput) {
       SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId};
-      taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey));
+      taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
       pWinInfo->isOutput = false;
     }
     taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
@@ -4005,10 +4062,11 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc
       if (!pCurWin) {
         break;
       }
+      SResultWindowInfo delWin = *pCurWin;
       deleteWindow(pAggSup->pCurWins, winIndex, fp);
       if (result) {
-        pCurWin->groupId = gpDatas[i];
+        delWin.groupId = gpDatas[i];
-        taosArrayPush(result, pCurWin);
+        taosArrayPush(result, &delWin);
       }
     }
   }
@@ -4033,6 +4091,7 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup,
     ASSERT(isInWindow(pCurWin, tsCols[i], gap));
     doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
     if (result) {
+      pCurWin->groupId = gpCols[i];
       taosArrayPush(result, pCurWin);
     }
   }
@@ -4067,10 +4126,18 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It
   size_t keyLen = 0;
   while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
     SWinKey*         res = *Ite;
-    SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
+    SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
-    colDataAppend(pTsCol, pBlock->info.rows, (const char*)&res->ts, false);
+    colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->ts, false);
+    SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
+    colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->ts, false);
+    SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
+    colDataAppendNULL(pUidCol, pBlock->info.rows);
     SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
     colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false);
+    SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
+    colDataAppendNULL(pCalStCol, pBlock->info.rows);
+    SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
+    colDataAppendNULL(pCalEdCol, pBlock->info.rows);
     pBlock->info.rows += 1;
     if (pBlock->info.rows + 1 >= pBlock->info.capacity) {
       break;
@@ -4081,8 +4148,8 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It
   }
 }
 
-static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId,
+static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray,
-                              int32_t numOfOutput, SOperatorInfo* pOperator) {
+                              int32_t numOfOutput, SOperatorInfo* pOperator, SHashObj* pStUpdated, bool needCreate) {
   SExprSupp*     pSup = &pOperator->exprSupp;
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
 
@@ -4092,9 +4159,15 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
   for (int32_t i = 0; i < size; i++) {
     SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i);
     SResultRow*        pCurResult = NULL;
+    uint64_t           groupId = pParentWin->groupId;
+    int32_t            winIndex = 0;
+    if (needCreate) {
+      pParentWin = getSessionTimeWindow(&pInfo->streamAggSup, pParentWin->win.skey, pParentWin->win.ekey, groupId, 0, &winIndex);
+    }
     setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset,
                        &pInfo->streamAggSup, pTaskInfo);
     int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren);
+    int32_t num = 0;
     for (int32_t j = 0; j < numOfChildren; j++) {
       SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);
       SStreamSessionAggOperatorInfo* pChInfo = pChild->info;
@@ -4110,15 +4183,24 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
           SResultRow* pChResult = NULL;
           setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
                              pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
-          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo);
+          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pChWin->win, true);
+          compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
           SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
           releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
+          num++;
           continue;
         } else if (!pChWin->isClosed) {
           break;
         }
       }
     }
+    if (num == 0 && needCreate) {
+      deleteWindow(pInfo->streamAggSup.pCurWins, winIndex, NULL);
+    }
+    if (pStUpdated && num > 0) {
+      SWinKey value = {.ts = pParentWin->win.skey, .groupId = groupId};
+      taosHashPut(pStUpdated, &pParentWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey));
+    }
     SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
     ASSERT(size > 0);
     setBufPageDirty(bufPage, true);
@@ -4198,7 +4280,46 @@ static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) {
   for (int32_t i = 0; i < size; i++) {
     SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i);
     SWinKey            res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId};
-    taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey));
+    taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
+  }
+}
+
+static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
+  int32_t size = taosArrayGetSize(pWins);
+  for (int32_t i = 0; i < size; i++) {
+    SResultWindowInfo* pWin = taosArrayGet(pWins, i);
+    taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
+  }
+}
+
+int32_t compareWinKey(void* pKey, void* data, int32_t index) {
+  SArray* res = (SArray*)data;
+  SResKeyPos* pos = taosArrayGetP(res, index);
+  SWinKey* pData = (SWinKey*)pKey;
+  if (pData->ts == *(int64_t*)pos->key) {
+    if (pData->groupId > pos->groupId) {
+      return 1;
+    } else if (pData->groupId < pos->groupId) {
+      return -1;
+    }
+    return 0;
+  } else if (pData->ts > *(int64_t*)pos->key) {
+    return 1;
+  }
+  return -1;
+}
+
+static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) {
+  int32_t size = taosHashGetSize(pStDeleted);
+  if (size == 0) {
+    return;
+  }
+
+  int32_t num = taosArrayGetSize(update);
+  for (int32_t i = 0; i < num; i++) {
+    SResKeyPos* pos = taosArrayGetP(update, i);
+    SWinKey winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId};
+    taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey));
   }
 }
 
@@ -4226,7 +4347,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
   _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   SHashObj*      pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   SOperatorInfo* downstream = pOperator->pDownstream[0];
-  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES);
+  SArray*        pUpdated = taosArrayInit(16, POINTER_BYTES); // SResKeyPos
   while (1) {
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
     if (pBlock == NULL) {
@@ -4242,9 +4363,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
         int32_t                        childIndex = getChildIndex(pBlock);
         SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);
         SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
-        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX,
+        doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX, pChildOp->exprSupp.numOfExprs,
-                              pChildOp->exprSupp.numOfExprs, 0, NULL);
+                              0, NULL);
-        rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
+        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, NULL, false);
       }
       taosArrayDestroy(pWins);
       continue;
@@ -4258,9 +4379,10 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
         SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;
         // gap must be 0
         doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL);
-        rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator);
+        rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated, true);
       }
       copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
+      removeSessionResults(pStUpdated, pWins);
       taosArrayDestroy(pWins);
       continue;
     } else if (pBlock->info.type == STREAM_GET_ALL) {
@@ -4303,6 +4425,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
                      pInfo->ignoreExpiredData, NULL);
   closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL);
   copyUpdateResult(pStUpdated, pUpdated);
+  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
   taosHashCleanup(pStUpdated);
 
   finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
@@ -4333,14 +4456,6 @@ static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
   pInfo->streamAggSup.currentPageId = -1;
 }
 
-static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) {
-  int32_t size = taosArrayGetSize(pWins);
-  for (int32_t i = 0; i < size; i++) {
-    SResultWindowInfo* pWin = taosArrayGet(pWins, i);
-    taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition));
-  }
-}
-
 static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
   SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
   SOptrBasicInfo*                pBInfo = &pInfo->binfo;
@@ -4349,30 +4464,34 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
 
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
-  } else if (pOperator->status == OP_RES_TO_RETURN) {
+  }
+  
+  {
     doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
     if (pBInfo->pRes->info.rows > 0) {
       printDataBlock(pBInfo->pRes, "semi session");
       return pBInfo->pRes;
     }
 
-    // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
+    doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
-    if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) {
+    if (pInfo->pDelRes->info.rows > 0) {
-      pInfo->returnDelete = true;
       printDataBlock(pInfo->pDelRes, "semi session");
       return pInfo->pDelRes;
     }
 
-    if (pInfo->pUpdateRes->info.rows > 0) {
+    if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
+      pInfo->returnUpdate = false;
       // process the rest of the data
-      pOperator->status = OP_OPENED;
       printDataBlock(pInfo->pUpdateRes, "semi session");
       return pInfo->pUpdateRes;
     }
-    // semi interval operator clear disk buffer
+
-    clearStreamSessionOperator(pInfo);
+    if (pOperator->status == OP_RES_TO_RETURN) {
-    pOperator->status = OP_EXEC_DONE;
+      // semi interval operator clear disk buffer
-    return NULL;
+      clearStreamSessionOperator(pInfo);
+      pOperator->status = OP_EXEC_DONE;
+      return NULL;
+    }
   }
 
   _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
@@ -4383,6 +4502,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
     if (pBlock == NULL) {
       clearSpecialDataBlock(pInfo->pUpdateRes);
+      pOperator->status = OP_RES_TO_RETURN;
       break;
     }
     printDataBlock(pBlock, "semi session recv");
@@ -4393,12 +4513,15 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
       removeSessionResults(pStUpdated, pWins);
       taosArrayDestroy(pWins);
       copyDataBlock(pInfo->pUpdateRes, pBlock);
+      pInfo->returnUpdate = true;
       break;
     } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
       // gap must be 0
-      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL, NULL);
+      SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
-      copyDataBlock(pInfo->pDelRes, pBlock);
+      doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL);
-      pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
+      copyDeleteWindowInfo(pWins, pInfo->pStDeleted);
+      removeSessionResults(pStUpdated, pWins);
+      taosArrayDestroy(pWins);
       break;
     } else if (pBlock->info.type == STREAM_GET_ALL) {
       getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession);
@@ -4411,18 +4534,15 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
     }
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
-    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, false);
+    doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false);
     maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
   }
 
   pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
   pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;
-  // restore the value
+
-  pOperator->status = OP_RES_TO_RETURN;
-  // semi operator
-  // closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
-  //                    getResWinForSession);
   copyUpdateResult(pStUpdated, pUpdated);
+  removeSessionDeleteResults(pUpdated, pInfo->pStDeleted);
   taosHashCleanup(pStUpdated);
 
   finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
@@ -4436,16 +4556,15 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
     return pBInfo->pRes;
   }
 
-  // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
+  doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
-  if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) {
+  if (pInfo->pDelRes->info.rows > 0) {
-    pInfo->returnDelete = true;
     printDataBlock(pInfo->pDelRes, "semi session");
     return pInfo->pDelRes;
   }
 
-  if (pInfo->pUpdateRes->info.rows > 0) {
+  if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) {
+    pInfo->returnUpdate = false;
     // process the rest of the data
-    pOperator->status = OP_OPENED;
     printDataBlock(pInfo->pUpdateRes, "semi session");
     return pInfo->pUpdateRes;
   }
@@ -4669,7 +4788,7 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, u
     if (pWinInfo->winInfo.win.skey > pTs[i]) {
       if (pSeDeleted && pWinInfo->winInfo.isOutput) {
         SWinKey res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId};
-        taosHashPut(pSeDeleted, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey));
+        taosHashPut(pSeDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey));
         pWinInfo->winInfo.isOutput = false;
       }
       pWinInfo->winInfo.win.skey = pTs[i];
@@ -4737,8 +4856,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
     winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, pSDataBlock->info.rows,
                                     i, &allEqual, pStDeleted);
     if (!allEqual) {
-      appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, GROUPID_COLUMN_INDEX,
+      uint64_t uid = 0;
-                   &groupId);
+      appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey,
+          &uid, &groupId);
       taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
       deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo);
       continue;
@@ -4767,6 +4887,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   SExprSupp*                   pSup = &pOperator->exprSupp;
   SStreamStateAggOperatorInfo* pInfo = pOperator->info;
   SOptrBasicInfo*              pBInfo = &pInfo->binfo;
+  int64_t maxTs = INT64_MIN;
   if (pOperator->status == OP_RES_TO_RETURN) {
     doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
     if (pInfo->pDelRes->info.rows > 0) {
@@ -4799,6 +4920,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
       SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
       doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo);
       copyDeleteWindowInfo(pWins, pInfo->pSeDeleted);
+      removeSessionResults(pSeUpdated, pWins);
       taosArrayDestroy(pWins);
       continue;
     } else if (pBlock->info.type == STREAM_GET_ALL) {
@@ -4813,8 +4935,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
     doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted);
-    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
+    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
   }
+  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
   // restore the value
   pOperator->status = OP_RES_TO_RETURN;
 
@@ -4913,7 +5036,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pOperator->info = pInfo;
   pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL,
                                          destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
-  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType);
+  initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex);
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;

source/libs/function/src/builtins.c

@@ -311,6 +311,22 @@ static int32_t translateInOutStr(SFunctionNode* pFunc, char* pErrBuf, int32_t le
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateMinMax(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  if (!IS_TIMESTAMP_TYPE(paraType) && !IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  } else if (IS_NULL_TYPE(paraType)) {
+    paraType = TSDB_DATA_TYPE_BIGINT;
+  }
+
+  pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType};
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateTrimStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isLtrim) {
   if (1 != LIST_LENGTH(pFunc->pParameterList)) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@@ -698,7 +714,7 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   }
 
   uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
-  if (!IS_NUMERIC_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+  if (!IS_NUMERIC_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
@@ -713,7 +729,7 @@ static int32_t translateSpreadImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t
 
   uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
   if (isPartial) {
-    if (!IS_NUMERIC_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+    if (!IS_NUMERIC_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
       return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
     }
     pFunc->node.resType = (SDataType){.bytes = getSpreadInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
@@ -788,7 +804,7 @@ static int32_t translateElapsedImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t
     }
 
     uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
-    if (TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+    if (!IS_TIMESTAMP_TYPE(paraType)) {
       return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
     }
 
@@ -1634,7 +1650,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
 
   uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
   if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType &&
-      TSDB_DATA_TYPE_TIMESTAMP != colType) {
+      !IS_TIMESTAMP_TYPE(colType)) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
@@ -1660,7 +1676,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   }
 
   uint8_t resType;
-  if (IS_SIGNED_NUMERIC_TYPE(colType) || TSDB_DATA_TYPE_BOOL == colType || TSDB_DATA_TYPE_TIMESTAMP == colType) {
+  if (IS_SIGNED_NUMERIC_TYPE(colType) || IS_TIMESTAMP_TYPE(colType) || TSDB_DATA_TYPE_BOOL == colType) {
     resType = TSDB_DATA_TYPE_BIGINT;
   } else {
     resType = TSDB_DATA_TYPE_DOUBLE;
@@ -1825,7 +1841,7 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
 
   // param0
   uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
-  if (!IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+  if (!IS_INTEGER_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
@@ -1878,7 +1894,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
 
   uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
   uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
-  if ((!IS_STR_DATA_TYPE(para1Type) && !IS_INTEGER_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) ||
+  if ((!IS_STR_DATA_TYPE(para1Type) && !IS_INTEGER_TYPE(para1Type) && !IS_TIMESTAMP_TYPE(para1Type)) ||
       !IS_INTEGER_TYPE(para2Type)) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
@@ -1911,7 +1927,7 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
 
   for (int32_t i = 0; i < 2; ++i) {
     uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type;
-    if (!IS_STR_DATA_TYPE(paraType) && !IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+    if (!IS_STR_DATA_TYPE(paraType) && !IS_INTEGER_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
       return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
     }
   }
@@ -2060,7 +2076,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "min",
     .type = FUNCTION_TYPE_MIN,
     .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
-    .translateFunc = translateInOutNum,
+    .translateFunc = translateMinMax,
     .dataRequiredFunc = statisDataRequired,
     .getEnvFunc   = getMinmaxFuncEnv,
     .initFunc     = minmaxFunctionSetup,
@@ -2075,7 +2091,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "max",
     .type = FUNCTION_TYPE_MAX,
     .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
-    .translateFunc = translateInOutNum,
+    .translateFunc = translateMinMax,
     .dataRequiredFunc = statisDataRequired,
     .getEnvFunc   = getMinmaxFuncEnv,
     .initFunc     = minmaxFunctionSetup,

source/libs/function/src/builtinsimpl.c

@@ -1204,7 +1204,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
         pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
       }
     } else {
-      if (IS_SIGNED_NUMERIC_TYPE(type)) {
+      if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type)) {
         int64_t prev = 0;
         GET_TYPED_DATA(prev, int64_t, type, &pBuf->v);
 
@@ -1216,7 +1216,6 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
             pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
           }
         }
-
       } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
         uint64_t prev = 0;
         GET_TYPED_DATA(prev, uint64_t, type, &pBuf->v);
@@ -1264,7 +1263,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
   int32_t start = pInput->startRowIndex;
   int32_t numOfRows = pInput->numOfRows;
 
-  if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
+  if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
     if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) {
       int8_t* pData = (int8_t*)pCol->pData;
       int8_t* val = (int8_t*)&pBuf->v;
@@ -1358,7 +1357,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
 
         numOfElems += 1;
       }
-    } else if (type == TSDB_DATA_TYPE_BIGINT) {
+    } else if (type == TSDB_DATA_TYPE_BIGINT ||
+               type == TSDB_DATA_TYPE_TIMESTAMP) {
       int64_t* pData = (int64_t*)pCol->pData;
       int64_t* val = (int64_t*)&pBuf->v;
 

source/libs/nodes/src/nodesCloneFuncs.c

@@ -777,6 +777,7 @@ SNode* nodesCloneNode(const SNode* pNode) {
       code = physiSessionCopy((const SSessionWinodwPhysiNode*)pNode, (SSessionWinodwPhysiNode*)pDst);
       break;
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
       code = physiPartitionCopy((const SPartitionPhysiNode*)pNode, (SPartitionPhysiNode*)pDst);
       break;
     default:

source/libs/nodes/src/nodesCodeFuncs.c

@@ -265,6 +265,8 @@ const char* nodesNodeName(ENodeType type) {
       return "PhysiStreamStateWindow";
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
       return "PhysiPartition";
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
+      return "PhysiStreamPartition";
     case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
       return "PhysiIndefRowsFunc";
     case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
@@ -4485,6 +4487,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
       return physiStateWindowNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
       return physiPartitionNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
       return physiIndefRowsFuncNodeToJson(pObj, pJson);
@@ -4632,6 +4635,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE:
       return jsonToPhysiStateWindowNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
       return jsonToPhysiPartitionNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
       return jsonToPhysiIndefRowsFuncNode(pJson, pObj);

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -537,7 +537,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
       }
       break;
     }
-    case QUERY_NODE_PHYSICAL_PLAN_PARTITION: {
+    case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: {
       SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)pNode;
       res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
       if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {

source/libs/nodes/src/nodesUtilFuncs.c

@@ -322,6 +322,8 @@ SNode* nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SStreamStateWinodwPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
       return makeNode(type, sizeof(SPartitionPhysiNode));
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION:
+      return makeNode(type, sizeof(SStreamPartitionPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
       return makeNode(type, sizeof(SIndefRowsFuncPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC:
@@ -951,7 +953,8 @@ void nodesDestroyNode(SNode* pNode) {
       nodesDestroyNode(pPhyNode->pStateKey);
       break;
     }
-    case QUERY_NODE_PHYSICAL_PLAN_PARTITION: {
+    case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: {
       SPartitionPhysiNode* pPhyNode = (SPartitionPhysiNode*)pNode;
       destroyPhysiNode((SPhysiNode*)pPhyNode);
       nodesDestroyList(pPhyNode->pExprs);

source/libs/parser/src/parAstCreater.c

@@ -795,6 +795,20 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode*
   return (SNode*)setOp;
 }
 
+static void updateWalOptionsDefault(SDatabaseOptions* pOptions) {
+  if (!pOptions->walRetentionPeriodIsSet) {
+    pOptions->walRetentionPeriod =
+        pOptions->replica > 1 ? TSDB_REPS_DEF_DB_WAL_RET_PERIOD : TSDB_REP_DEF_DB_WAL_RET_PERIOD;
+  }
+  if (!pOptions->walRetentionSizeIsSet) {
+    pOptions->walRetentionSize = pOptions->replica > 1 ? TSDB_REPS_DEF_DB_WAL_RET_SIZE : TSDB_REP_DEF_DB_WAL_RET_SIZE;
+  }
+  if (!pOptions->walRollPeriodIsSet) {
+    pOptions->walRollPeriod =
+        pOptions->replica > 1 ? TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD : TSDB_REP_DEF_DB_WAL_ROLL_PERIOD;
+  }
+}
+
 SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
   CHECK_PARSER_STATUS(pCxt);
   SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS);
@@ -819,9 +833,7 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) {
   pOptions->numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
   pOptions->singleStable = TSDB_DEFAULT_DB_SINGLE_STABLE;
   pOptions->schemaless = TSDB_DEFAULT_DB_SCHEMALESS;
-  pOptions->walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD;
+  updateWalOptionsDefault(pOptions);
-  pOptions->walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE;
-  pOptions->walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD;
   pOptions->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
   return (SNode*)pOptions;
 }
@@ -859,78 +871,83 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) {
 
 SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) {
   CHECK_PARSER_STATUS(pCxt);
+  SDatabaseOptions* pDbOptions = (SDatabaseOptions*)pOptions;
   switch (type) {
     case DB_OPTION_BUFFER:
-      ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_CACHEMODEL:
-      COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->cacheModelStr, (SToken*)pVal);
+      COPY_STRING_FORM_STR_TOKEN(pDbOptions->cacheModelStr, (SToken*)pVal);
       break;
     case DB_OPTION_CACHESIZE:
-      ((SDatabaseOptions*)pOptions)->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_COMP:
-      ((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_DAYS: {
       SToken* pToken = pVal;
       if (TK_NK_INTEGER == pToken->type) {
-        ((SDatabaseOptions*)pOptions)->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440;
+        pDbOptions->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440;
       } else {
-        ((SDatabaseOptions*)pOptions)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken);
+        pDbOptions->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken);
       }
       break;
     }
     case DB_OPTION_FSYNC:
-      ((SDatabaseOptions*)pOptions)->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_MAXROWS:
-      ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_MINROWS:
-      ((SDatabaseOptions*)pOptions)->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_KEEP:
-      ((SDatabaseOptions*)pOptions)->pKeep = pVal;
+      pDbOptions->pKeep = pVal;
       break;
     case DB_OPTION_PAGES:
-      ((SDatabaseOptions*)pOptions)->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_PAGESIZE:
-      ((SDatabaseOptions*)pOptions)->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_PRECISION:
-      COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->precisionStr, (SToken*)pVal);
+      COPY_STRING_FORM_STR_TOKEN(pDbOptions->precisionStr, (SToken*)pVal);
       break;
     case DB_OPTION_REPLICA:
-      ((SDatabaseOptions*)pOptions)->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+      updateWalOptionsDefault(pDbOptions);
       break;
     case DB_OPTION_STRICT:
-      COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->strictStr, (SToken*)pVal);
+      COPY_STRING_FORM_STR_TOKEN(pDbOptions->strictStr, (SToken*)pVal);
       break;
     case DB_OPTION_WAL:
-      ((SDatabaseOptions*)pOptions)->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_VGROUPS:
-      ((SDatabaseOptions*)pOptions)->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_SINGLE_STABLE:
-      ((SDatabaseOptions*)pOptions)->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_RETENTIONS:
-      ((SDatabaseOptions*)pOptions)->pRetentions = pVal;
+      pDbOptions->pRetentions = pVal;
       break;
     case DB_OPTION_WAL_RETENTION_PERIOD:
-      ((SDatabaseOptions*)pOptions)->walRetentionPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->walRetentionPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->walRetentionPeriodIsSet = true;
       break;
     case DB_OPTION_WAL_RETENTION_SIZE:
-      ((SDatabaseOptions*)pOptions)->walRetentionSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->walRetentionSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->walRetentionSizeIsSet = true;
       break;
     case DB_OPTION_WAL_ROLL_PERIOD:
-      ((SDatabaseOptions*)pOptions)->walRollPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->walRollPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->walRollPeriodIsSet = true;
       break;
     case DB_OPTION_WAL_SEGMENT_SIZE:
-      ((SDatabaseOptions*)pOptions)->walSegmentSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      pDbOptions->walSegmentSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     default:
       break;
@@ -1251,7 +1268,8 @@ SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
 
 static bool needDbShowStmt(ENodeType type) {
   return QUERY_NODE_SHOW_TABLES_STMT == type || QUERY_NODE_SHOW_STABLES_STMT == type ||
-         QUERY_NODE_SHOW_VGROUPS_STMT == type;
+         QUERY_NODE_SHOW_VGROUPS_STMT == type || QUERY_NODE_SHOW_INDEXES_STMT == type ||
+         QUERY_NODE_SHOW_TAGS_STMT == type;
 }
 
 SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type) {
@@ -1264,7 +1282,7 @@ SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type) {
 SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName,
                               EOperatorType tableCondType) {
   CHECK_PARSER_STATUS(pCxt);
-  if (needDbShowStmt(type) && NULL == pDbName && NULL == pCxt->pQueryCxt->db) {
+  if (needDbShowStmt(type) && NULL == pDbName) {
     snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "db not specified");
     pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
     return NULL;

source/libs/parser/src/parAstParser.c

@@ -374,8 +374,13 @@ static int32_t collectMetaKeyFromShowIndexes(SCollectMetaKeyCxt* pCxt, SShowStmt
 }
 
 static int32_t collectMetaKeyFromShowStables(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
-  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES,
+  int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES,
-                                 pCxt->pMetaCache);
+                                         pCxt->pMetaCache);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser,
+                                  ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ, pCxt->pMetaCache);
+  }
+  return code;
 }
 
 static int32_t collectMetaKeyFromShowStreams(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
@@ -387,11 +392,11 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt*
   int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES,
                                          pCxt->pMetaCache);
   if (TSDB_CODE_SUCCESS == code) {
-    if (NULL != pStmt->pDbName) {
+    code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
-      code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
+  }
-    } else {
+  if (TSDB_CODE_SUCCESS == code) {
-      code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, pCxt->pMetaCache);
+    code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser,
-    }
+                                  ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ, pCxt->pMetaCache);
   }
   return code;
 }
@@ -400,11 +405,7 @@ static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* p
   int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TAGS,
                                          pCxt->pMetaCache);
   if (TSDB_CODE_SUCCESS == code) {
-    if (NULL != pStmt->pDbName) {
+    code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
-      code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
-    } else {
-      code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, pCxt->pMetaCache);
-    }
   }
   return code;
 }

source/libs/parser/src/parAuthenticator.c

@@ -96,6 +96,10 @@ static int32_t authInsert(SAuthCxt* pCxt, SInsertStmt* pInsert) {
   return code;
 }
 
+static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) {
+  return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ);
+}
+
 static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) {
   return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_READ);
 }
@@ -127,6 +131,9 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
     case QUERY_NODE_SHOW_VNODES_STMT:
     case QUERY_NODE_SHOW_SCORES_STMT:
       return !pCxt->pParseCxt->enableSysInfo ? TSDB_CODE_PAR_PERMISSION_DENIED : TSDB_CODE_SUCCESS;
+    case QUERY_NODE_SHOW_TABLES_STMT:
+    case QUERY_NODE_SHOW_STABLES_STMT:
+      return authShowTables(pCxt, (SShowStmt*)pStmt);
     case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
     case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
       return authShowCreateTable(pCxt, (SShowCreateTableStmt*)pStmt);

source/libs/parser/src/parInsert.c

@@ -786,6 +786,7 @@ static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTa
   if (sname) pTbReq->ctb.name = strdup(sname);
   pTbReq->ctb.pTag = (uint8_t*)pTag;
   pTbReq->ctb.tagName = taosArrayDup(tagName);
+  pTbReq->ttl = TSDB_DEFAULT_TABLE_TTL;
   pTbReq->commentLen = -1;
 
   return;
@@ -1121,6 +1122,40 @@ static int32_t ignoreAutoCreateTableClause(SInsertParseContext* pCxt) {
   return code;
 }
 
+static int32_t parseTableOptions(SInsertParseContext* pCxt) {
+  do {
+    int32_t index = 0;
+    SToken  sToken;
+    NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index);
+    if (TK_TTL == sToken.type) {
+      pCxt->pSql += index;
+      NEXT_TOKEN(pCxt->pSql, sToken);
+      if (TK_NK_INTEGER != sToken.type) {
+        return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z);
+      }
+      pCxt->createTblReq.ttl = taosStr2Int32(sToken.z, NULL, 10);
+    } else if (TK_COMMENT == sToken.type) {
+      pCxt->pSql += index;
+      NEXT_TOKEN(pCxt->pSql, sToken);
+      if (TK_NK_STRING != sToken.type) {
+        return buildSyntaxErrMsg(&pCxt->msg, "Invalid option comment", sToken.z);
+      }
+      if (sToken.n >= TSDB_TB_COMMENT_LEN) {
+        return buildSyntaxErrMsg(&pCxt->msg, "comment too long", sToken.z);
+      }
+      int32_t len = trimString(sToken.z, sToken.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN);
+      pCxt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len);
+      if (NULL == pCxt->createTblReq.comment) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      pCxt->createTblReq.commentLen = len;
+    } else {
+      break;
+    }
+  } while (1);
+  return TSDB_CODE_SUCCESS;
+}
+
 // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)
 static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* tbFName) {
   int32_t      len = strlen(tbFName);
@@ -1172,7 +1207,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName*
     return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z);
   }
 
-  return TSDB_CODE_SUCCESS;
+  return parseTableOptions(pCxt);
 }
 
 static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, bool* gotRow,
@@ -1672,7 +1707,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache
   if (pContext->pStmtCb) {
     context.pVgroupsHashObj = NULL;
     context.pTableBlockHashObj = NULL;
-  }  
+  }
   destroyInsertParseContext(&context);
   return code;
 }
@@ -1700,6 +1735,21 @@ static int32_t skipValuesClause(SInsertParseSyntaxCxt* pCxt) {
 
 static int32_t skipTagsClause(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); }
 
+static int32_t skipTableOptions(SInsertParseSyntaxCxt* pCxt) {
+  do {
+    int32_t index = 0;
+    SToken  sToken;
+    NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index);
+    if (TK_TTL == sToken.type || TK_COMMENT == sToken.type) {
+      pCxt->pSql += index;
+      NEXT_TOKEN(pCxt->pSql, sToken);
+    } else {
+      break;
+    }
+  } while (1);
+  return TSDB_CODE_SUCCESS;
+}
+
 // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...)
 static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) {
   SToken sToken;
@@ -1718,6 +1768,7 @@ static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) {
     return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z);
   }
   CHECK_CODE(skipTagsClause(pCxt));
+  CHECK_CODE(skipTableOptions(pCxt));
 
   return TSDB_CODE_SUCCESS;
 }

source/libs/parser/test/parInitialCTest.cpp

@@ -111,9 +111,9 @@ TEST_F(ParserInitialCTest, createDatabase) {
     expect.numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
     expect.numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE;
     expect.schemaless = TSDB_DEFAULT_DB_SCHEMALESS;
-    expect.walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD;
+    expect.walRetentionPeriod = TSDB_REP_DEF_DB_WAL_RET_PERIOD;
-    expect.walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE;
+    expect.walRetentionSize = TSDB_REP_DEF_DB_WAL_RET_SIZE;
-    expect.walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD;
+    expect.walRollPeriod = TSDB_REP_DEF_DB_WAL_ROLL_PERIOD;
     expect.walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
   };
 
@@ -266,6 +266,14 @@ TEST_F(ParserInitialCTest, createDatabase) {
       "DURATION 100m "
       "KEEP 1440m,300h,400d ");
   clearCreateDbReq();
+
+  setCreateDbReqFunc("wxy_db", 1);
+  setDbReplicaFunc(3);
+  setDbWalRetentionPeriod(TSDB_REPS_DEF_DB_WAL_RET_PERIOD);
+  setDbWalRetentionSize(TSDB_REPS_DEF_DB_WAL_RET_SIZE);
+  setDbWalRollPeriod(TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD);
+  run("CREATE DATABASE IF NOT EXISTS wxy_db REPLICA 3");
+  clearCreateDbReq();
 }
 
 TEST_F(ParserInitialCTest, createDatabaseSemanticCheck) {

source/libs/planner/src/planOptimizer.c

@@ -16,6 +16,7 @@
 #include "filter.h"
 #include "functionMgt.h"
 #include "planInt.h"
+#include "tglobal.h"
 #include "ttime.h"
 
 #define OPTIMIZE_FLAG_MASK(n) (1 << n)
@@ -2410,7 +2411,7 @@ static const SOptimizeRule optimizeRuleSet[] = {
 static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule));
 
 static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
-  if (0 == (qDebugFlag & DEBUG_DEBUG)) {
+  if (!tsQueryPlannerTrace) {
     return;
   }
   char* pStr = NULL;

source/libs/planner/src/planPhysiCreater.c

@@ -1324,7 +1324,8 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
 static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
                                         SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) {
   SPartitionPhysiNode* pPart =
-      (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION);
+      (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode,
+          pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION : QUERY_NODE_PHYSICAL_PLAN_PARTITION);
   if (NULL == pPart) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }

source/libs/planner/src/planSpliter.c

@@ -1427,7 +1427,7 @@ static const SSplitRule splitRuleSet[] = {
 static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule));
 
 static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) {
-  if (0 == (qDebugFlag & DEBUG_DEBUG)) {
+  if (!tsQueryPlannerTrace) {
     return;
   }
   char* pStr = NULL;

source/libs/planner/src/planner.c

@@ -17,9 +17,10 @@
 
 #include "planInt.h"
 #include "scalar.h"
+#include "tglobal.h"
 
 static void dumpQueryPlan(SQueryPlan* pPlan) {
-  if (0 == (qDebugFlag & DEBUG_DEBUG)) {
+  if (!tsQueryPlannerTrace) {
     return;
   }
   char* pStr = NULL;

source/libs/stream/src/streamUpdate.c

@@ -170,8 +170,17 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) {
   if (ts < maxTs - pInfo->watermark) {
     // this window has been closed.
     if (pInfo->pCloseWinSBF) {
-      return tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY));
+      res = tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY));
+      if (res == TSDB_CODE_SUCCESS) {
+        return false;
+      } else {
+         qDebug("===stream===Update close window sbf. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
+                maxTs, *pMapMaxTs, ts);
+        return true;
+      }
     }
+    qDebug("===stream===Update close window. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
+           maxTs, *pMapMaxTs, ts);
     return true;
   }
 
@@ -193,7 +202,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) {
   }
 
   if (ts < pInfo->minTS) {
-    qDebug("===stream===Update. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
+    qDebug("===stream===Update min ts. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId,
            maxTs, *pMapMaxTs, ts);
     return true;
   } else if (res == TSDB_CODE_SUCCESS) {

source/libs/sync/src/syncSnapshot.c

@@ -583,7 +583,7 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
                                                            &(pReceiver->snapshot));
     if (code != 0) {
       syncNodeErrorLog(pReceiver->pSyncNode, "snapshot stop writer true error");
-      ASSERT(0);
+      // ASSERT(0);
       return -1;
     }
     pReceiver->pWriter = NULL;

source/libs/sync/src/syncTimeout.c

@@ -91,16 +91,16 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) {
   } else if (pMsg->timeoutType == SYNC_TIMEOUT_ELECTION) {
     if (atomic_load_64(&ths->electTimerLogicClockUser) <= pMsg->logicClock) {
       ++(ths->electTimerCounter);
-      sInfo("vgId:%d, sync timeout, type:election count:%d, electTimerLogicClockUser:%ld", ths->vgId,
+      sTrace("vgId:%d, sync timer, type:election count:%d, electTimerLogicClockUser:%ld", ths->vgId,
-            ths->electTimerCounter, ths->electTimerLogicClockUser);
+             ths->electTimerCounter, ths->electTimerLogicClockUser);
       syncNodeElect(ths);
     }
 
   } else if (pMsg->timeoutType == SYNC_TIMEOUT_HEARTBEAT) {
     if (atomic_load_64(&ths->heartbeatTimerLogicClockUser) <= pMsg->logicClock) {
       ++(ths->heartbeatTimerCounter);
-      sInfo("vgId:%d, sync timeout, type:replicate count:%d, heartbeatTimerLogicClockUser:%ld", ths->vgId,
+      sTrace("vgId:%d, sync timer, type:replicate count:%d, heartbeatTimerLogicClockUser:%ld", ths->vgId,
-            ths->heartbeatTimerCounter, ths->heartbeatTimerLogicClockUser);
+             ths->heartbeatTimerCounter, ths->heartbeatTimerLogicClockUser);
       syncNodeReplicate(ths, true);
     }
   } else {

source/os/src/osDir.c

@@ -160,6 +160,66 @@ int32_t taosMulMkDir(const char *dirname) {
   return code;
 }
 
+int32_t taosMulModeMkDir(const char *dirname, int mode) {
+  if (dirname == NULL) return -1;
+  char    temp[1024];
+  char   *pos = temp;
+  int32_t code = 0;
+#ifdef WINDOWS
+  taosRealPath(dirname, temp, sizeof(temp));
+  if (temp[1] == ':') pos += 3;
+#else
+  strcpy(temp, dirname);
+#endif
+
+  if (taosDirExist(temp)) {
+    chmod(temp, mode);
+    return code;
+  }
+
+  if (strncmp(temp, TD_DIRSEP, 1) == 0) {
+    pos += 1;
+  } else if (strncmp(temp, "." TD_DIRSEP, 2) == 0) {
+    pos += 2;
+  }
+
+  for (; *pos != '\0'; pos++) {
+    if (*pos == TD_DIRSEP[0]) {
+      *pos = '\0';
+#ifdef WINDOWS
+      code = _mkdir(temp, mode);
+#else
+      code = mkdir(temp, mode);
+#endif
+      if (code < 0 && errno != EEXIST) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return code;
+      }
+      *pos = TD_DIRSEP[0];
+    }
+  }
+
+  if (*(pos - 1) != TD_DIRSEP[0]) {
+#ifdef WINDOWS
+    code = _mkdir(temp, mode);
+#else
+    code = mkdir(temp, mode);
+#endif
+    if (code < 0 && errno != EEXIST) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return code;
+    }
+  }
+
+  if (code < 0 && errno == EEXIST) {
+    chmod(temp, mode);
+    return 0;
+  }
+
+  chmod(temp, mode);
+  return code;
+}
+
 void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
   TdDirPtr pDir = taosOpenDir(dirname);
   if (pDir == NULL) return;

source/util/src/terror.c

@@ -622,7 +622,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REGEX_MATCH,               "Rsma regex match")
 
 //index
 TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING,               "Index is rebuilding")
-TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING,               "Invalid index file")
+TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_INVALID_FILE,             "Index file is invalid")
 
 //tmq
 TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG,                "Invalid message")

tests/pytest/tools/taosdumpTest2.py

@@ -11,15 +11,19 @@
 
 # -*- coding: utf-8 -*-
 
+from logging.config import dictConfig
 import sys
 import os
 from util.log import *
 from util.cases import *
 from util.sql import *
 from util.dnodes import *
+import string
+import random
 
 
 class TDTestCase:
+
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
@@ -47,12 +51,19 @@ class TDTestCase:
             return ""
         return paths[0]
 
+    def generateString(self, length):
+        chars = string.ascii_uppercase + string.ascii_lowercase
+        v = ""
+        for i in range(length):
+            v += random.choice(chars)
+        return v
+
     def run(self):
         if not os.path.exists("./taosdumptest/tmp"):
             os.makedirs("./taosdumptest/tmp")
         else:
-            os.system("rm -rf ./taosdumptest/tmp")
+            print("directory exists")
-            os.makedirs("./taosdumptest/tmp")
+            os.system("rm -rf ./taosdumptest/tmp/*")
 
         tdSql.prepare()
 
@@ -76,17 +87,19 @@ class TDTestCase:
             tdLog.info("taosdump found in %s" % binPath)
 
         os.system("rm ./taosdumptest/tmp/*.sql")
+        os.system("rm ./taosdumptest/tmp/*.avro*")
+        os.system("rm -rf ./taosdumptest/taosdump.*")
         os.system(
-            "%s --databases db -o ./taosdumptest/tmp -B 32766 -L 1048576" %
+            "%s --databases db -o ./taosdumptest/tmp " %
             binPath)
 
         tdSql.execute("drop database db")
-        tdSql.query("select * from information_schema.ins_databases")
+        tdSql.query("show databases")
         tdSql.checkRows(2)
 
-        os.system("%s -i ./taosdumptest/tmp" % binPath)
+        os.system("%s -i ./taosdumptest/tmp -y" % binPath)
 
-        tdSql.query("select * from information_schema.ins_databases")
+        tdSql.query("show databases")
         tdSql.checkRows(3)
         tdSql.checkData(2, 0, 'db')
 
@@ -105,17 +118,17 @@ class TDTestCase:
             "create table stb(ts timestamp, c1 binary(16374), c2 binary(16374), c3 binary(16374)) tags(t1 nchar(256))")
         tdSql.execute(
             "insert into t1 using stb tags('t1') values(now, '%s', '%s', '%s')" %
-            ("16374",
+            (self.generateString(16374),
-             "16374",
+             self.generateString(16374),
-             "16374"))
+             self.generateString(16374)))
 
-#        sys.exit(0)
         os.system("rm ./taosdumptest/tmp/*.sql")
         os.system("rm ./taosdumptest/tmp/*.avro*")
+        os.system("rm -rf ./taosdumptest/tmp/taosdump.*")
         os.system("%s -D test -o ./taosdumptest/tmp -y" % binPath)
 
         tdSql.execute("drop database test")
-        tdSql.query("select * from information_schema.ins_databases")
+        tdSql.query("show databases")
         tdSql.checkRows(3)
 
         os.system("%s -i ./taosdumptest/tmp -y" % binPath)

tests/script/tsim/stable/alter_comment.sim

@@ -95,7 +95,7 @@ sql_error alter table db.stb add tag t1 int
 sql_error alter table db.stb add tag t2 int
 sql_error alter table db.stb add tag t3 int
 sql alter table db.stb add tag t4 bigint
-sql alter table db.stb add tag c1 int  
+sql alter table db.stb add tag c1 int
 sql alter table db.stb add tag t5 binary(12)
 
 sql select * from information_schema.ins_stables where db_name = 'db'

tests/script/tsim/stable/alter_count.sim

@@ -5,8 +5,8 @@ print ========= start dnode1 as master
 system sh/exec.sh -n dnode1 -s start
 sql connect
 
-print ======== step1 
+print ======== step1
-sql create database d1 replica 1 duration 7 keep 50 
+sql create database d1 replica 1 duration 7 keep 50
 sql use d1
 sql create table tb (ts timestamp, a int)
 sql insert into tb values(now-28d, -28)
@@ -83,7 +83,7 @@ if $data00 != 3 then
 endi
 
 print ======== step8
-# sql alter table tb(ts timestamp, a int, b smallint, c tinyint, d int, e bigint, f float, g double, h binary(10) ) 
+# sql alter table tb(ts timestamp, a int, b smallint, c tinyint, d int, e bigint, f float, g double, h binary(10) )
 sql alter table tb add column h binary(10)
 sql insert into tb values(now-7d, -7, 18, 0, 0, 0, 0, 0, '0')
 sql insert into tb values(now-6d, -6, 19, 1, 1, 1, 1, 1, '1')
@@ -260,4 +260,4 @@ if $data00 != 31 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/alter_import.sim

@@ -5,7 +5,7 @@ print ========= start dnode1 as master
 system sh/exec.sh -n dnode1 -s start
 sql connect
 
-print ======== step1 
+print ======== step1
 sql create database d1 replica 1 duration 7 keep 50
 sql use d1
 sql create table tb (ts timestamp, a int)
@@ -42,4 +42,4 @@ if $data00 != 6 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/alter_insert1.sim

@@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/exec.sh -n dnode1 -s start
 sql connect
 
-print ======== step1 
+print ======== step1
 sql create database d3
 sql use d3
 sql create table tb (ts timestamp, a int)
@@ -1137,4 +1137,4 @@ if $data79 != null then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/alter_insert2.sim

@@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/exec.sh -n dnode1 -s start
 sql connect
 
-print ======== step1 
+print ======== step1
 sql create database d4
 sql use d4
 sql create table tb (ts timestamp, a int, b smallint, c tinyint, d int, e bigint, f float, g double, h binary(10))
@@ -662,4 +662,4 @@ if $data62 != null then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/alter_metrics.sim

@@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/exec.sh -n dnode1 -s start
 sql connect
 
-print ======== step1 
+print ======== step1
 sql create database d2
 sql use d2
 sql create table mt (ts timestamp, a int) TAGS (t int)
@@ -757,8 +757,8 @@ endi
 print ======= over
 sql drop database d2
 sql select * from information_schema.ins_databases
-if $rows != 2 then 
+if $rows != 2 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/column_add.sim

@@ -116,7 +116,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 2 then
   return -1
-endi  
+endi
 if $data[0][3] != NULL then
   return -1
 endi
@@ -153,7 +153,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 2 then
   return -1
-endi  
+endi
 if $data[0][3] != NULL then
   return -1
 endi
@@ -299,4 +299,4 @@ if $rows != 10 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/column_modify.sim

@@ -31,7 +31,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 1234 then
   return -1
-endi  
+endi
 if $data[0][3] != 101 then
   return -1
 endi
@@ -92,7 +92,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 1234 then
   return -1
-endi  
+endi
 if $data[0][3] != 101 then
   return -1
 endi
@@ -106,4 +106,4 @@ if $data[1][3] != 101 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/disk.sim

@@ -188,4 +188,4 @@ if $rows != 2 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/refcount.sim

@@ -123,4 +123,4 @@ if $rows != 2 then
 endi
 
 print =============== step6
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/tag_add.sim

@@ -139,7 +139,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 2 then
   return -1
-endi  
+endi
 if $data[0][3] != 101 then
   return -1
 endi
@@ -170,7 +170,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 2 then
   return -1
-endi  
+endi
 if $data[0][3] != 101 then
   return -1
 endi
@@ -190,4 +190,4 @@ if $rows != 7 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/tag_drop.sim

@@ -165,7 +165,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 2 then
   return -1
-endi  
+endi
 if $data[0][3] != 101 then
   return -1
 endi
@@ -196,7 +196,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 2 then
   return -1
-endi  
+endi
 if $data[0][3] != 201 then
   return -1
 endi
@@ -229,7 +229,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 2 then
   return -1
-endi  
+endi
 if $data[0][3] != 201 then
   return -1
 endi
@@ -261,7 +261,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 2 then
   return -1
-endi  
+endi
 if $data[0][3] != 301 then
   return -1
 endi
@@ -323,7 +323,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 2 then
   return -1
-endi  
+endi
 if $data[0][3] != 302 then
   return -1
 endi
@@ -334,4 +334,4 @@ if $data[0][5] != 304 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/tag_filter.sim

@@ -27,47 +27,47 @@ sql create table db.ctb6 using db.stb tags(6, "102")
 sql insert into db.ctb6 values(now, 6, "2")
 
 sql select * from db.stb where t1 = 1
-if $rows != 1 then 
+if $rows != 1 then
   return -1
 endi
 
-sql select * from db.stb where t1 < 1 
+sql select * from db.stb where t1 < 1
-if $rows != 0 then 
+if $rows != 0 then
   return -=1
 endi
 
-sql select * from db.stb where t1 < 2 
+sql select * from db.stb where t1 < 2
-if $rows != 1 then 
+if $rows != 1 then
   return -1
 endi
 
-sql select * from db.stb where t1 <= 2 
+sql select * from db.stb where t1 <= 2
-if $rows != 2 then 
+if $rows != 2 then
   return -1
 endi
 
-sql select * from db.stb where t1 >= 1 
+sql select * from db.stb where t1 >= 1
-if $rows != 6 then 
+if $rows != 6 then
   return -1
 endi
 
-sql select * from db.stb where t1 > 1 
+sql select * from db.stb where t1 > 1
-if $rows != 5 then 
+if $rows != 5 then
   return -1
 endi
 
-sql select * from db.stb where t1 between 1 and 1 
+sql select * from db.stb where t1 between 1 and 1
-if $rows != 1 then 
+if $rows != 1 then
   return -1
 endi
 
-sql select * from db.stb where t1 between 1 and 6 
+sql select * from db.stb where t1 between 1 and 6
-if $rows != 6 then 
+if $rows != 6 then
   return -1
 endi
 
-sql select * from db.stb where t1 between 1 and 7 
+sql select * from db.stb where t1 between 1 and 7
-if $rows != 6 then 
+if $rows != 6 then
   return -1
 endi
 
@@ -88,25 +88,25 @@ sql insert into db.ctbBin2 values(now, 3, "2")
 sql create table db.ctbBin3 using db.stbBin tags("d")
 sql insert into db.ctbBin3 values(now, 4, "2")
 
-sql select * from db.stbBin where t1 = "a" 
+sql select * from db.stbBin where t1 = "a"
-if $rows != 1 then 
+if $rows != 1 then
   return -1
 endi
 
 
-sql select * from db.stbBin where t1 < "a" 
+sql select * from db.stbBin where t1 < "a"
-if $rows != 0 then 
+if $rows != 0 then
   return -=1
 endi
 
-sql select * from db.stbBin where t1 < "b" 
+sql select * from db.stbBin where t1 < "b"
-if $rows != 1 then 
+if $rows != 1 then
   return -1
 endi
 
 
-sql select * from db.stbBin where t1 between "a" and "e" 
+sql select * from db.stbBin where t1 between "a" and "e"
-if $rows != 4 then 
+if $rows != 4 then
   return -1
 endi
 
@@ -127,25 +127,25 @@ sql insert into db.ctbNc2 values(now, 3, "2")
 sql create table db.ctbNc3 using db.stbNc tags("d")
 sql insert into db.ctbNc3 values(now, 4, "2")
 
-sql select * from db.stbNc where t1 = "a" 
+sql select * from db.stbNc where t1 = "a"
-if $rows != 1 then 
+if $rows != 1 then
   return -1
 endi
 
 
-sql select * from db.stbNc where t1 < "a" 
+sql select * from db.stbNc where t1 < "a"
-if $rows != 0 then 
+if $rows != 0 then
   return -=1
 endi
 
-sql select * from db.stbNc where t1 < "b" 
+sql select * from db.stbNc where t1 < "b"
-if $rows != 1 then 
+if $rows != 1 then
   return -1
 endi
 
 
-sql select * from db.stbNc where t1 between "a" and "e" 
+sql select * from db.stbNc where t1 between "a" and "e"
-if $rows != 4 then 
+if $rows != 4 then
   return -1
 endi
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/tag_modify.sim

@@ -28,7 +28,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 1234 then
   return -1
-endi  
+endi
 if $data[0][3] != 101 then
   return -1
 endi
@@ -55,7 +55,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 1234 then
   return -1
-endi  
+endi
 if $data[0][3] != 101 then
   return -1
 endi
@@ -120,4 +120,4 @@ if $data[4][2] != 5 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stable/tag_rename.sim

@@ -28,7 +28,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 1234 then
   return -1
-endi  
+endi
 if $data[0][3] != 101 then
   return -1
 endi
@@ -52,7 +52,7 @@ if $data[0][1] != 1 then
 endi
 if $data[0][2] != 1234 then
   return -1
-endi  
+endi
 if $data[0][3] != 101 then
   return -1
 endi
@@ -117,4 +117,4 @@ if $data[4][2] != 4 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stream/basic1.sim

@@ -462,10 +462,10 @@ if $data25 != 3 then
   return -1
 endi
 
-sql create database test2 vgroups 1
+sql create database test2 vgroups 1;
-sql select * from information_schema.ins_databases
+sql select * from information_schema.ins_databases;
 
-sql use test2
+sql use test2;
 sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
 sql create table t1 using st tags(1,1,1);
 sql create table t2 using st tags(2,2,2);

tests/script/tsim/stream/partitionbyColumn0.sim

@@ -0,0 +1,570 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test  vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select  _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop0:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop0
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop0
+endi
+
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+
+loop1:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop1
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop1
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+
+$loop_count = 0
+
+loop2:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop2
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop2
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791213001,2,2,3,1.0);
+sql insert into t1 values(1648791213002,2,2,3,1.0);
+sql insert into t1 values(1648791213002,1,2,3,1.0);
+
+$loop_count = 0
+
+loop3:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop3
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop3
+endi
+
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 100
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop4
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop4
+endi
+
+if $data11 != 1 then
+  print =====data11=$data11
+  goto loop4
+endi
+
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop4
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop4
+endi
+
+if $data22 != 1 then
+  print =====data22=$data22
+  goto loop4
+endi
+
+if $data31 != 1 then
+  print =====data31=$data31
+  goto loop4
+endi
+
+if $data32 != 2 then
+  print =====data32=$data32
+  goto loop4
+endi
+
+if $data41 != 1 then
+  print =====data41=$data41
+  goto loop4
+endi
+
+if $data42 != 3 then
+  print =====data42=$data42
+  goto loop4
+endi
+
+sql drop stream if exists streams1;
+sql drop database if exists test1;
+sql create database test1  vgroups 1;
+sql use test1;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams1 trigger at_once into streamt1 as select  _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,1,2,1,1.0);
+sql insert into t1 values(1648791213001,2,1,2,2.0);
+sql insert into t1 values(1648791213001,1,2,3,2.0);
+
+$loop_count = 0
+
+loop5:
+sleep 100
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop5
+endi
+
+sql insert into t1 values(1648791223000,1,2,4,2.0);
+sql insert into t1 values(1648791223001,1,2,5,2.0);
+sql insert into t1 values(1648791223002,1,2,5,2.0);
+sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
+
+$loop_count = 0
+
+loop6:
+sleep 100
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop6
+endi
+
+if $data02 != 6 then
+  print =====data02=$data02
+  goto loop6
+endi
+
+if $data11 != 1 then
+  print =====data11=$data11
+  goto loop6
+endi
+
+if $data12 != 1 then
+  print =====data12=$data12
+  goto loop6
+endi
+
+if $data21 != 1 then
+  print =====data21=$data21
+  goto loop6
+endi
+
+if $data22 != 7 then
+  print =====data22=$data22
+  goto loop6
+endi
+
+if $data31 != 2 then
+  print =====data31=$data31
+  goto loop6
+endi
+
+if $data32 != 5 then
+  print =====data32=$data32
+  goto loop6
+endi
+
+sql drop stream if exists streams2;
+sql drop database if exists test2;
+sql create database test2  vgroups 4;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select  _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop7
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop7
+endi
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+sql insert into t2 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+
+loop8:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop8
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop8
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+
+$loop_count = 0
+
+loop9:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop9
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop9
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791213001,2,2,3,1.0);
+sql insert into t1 values(1648791213002,2,2,3,1.0);
+sql insert into t1 values(1648791213002,1,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213001,2,2,3,1.0);
+sql insert into t2 values(1648791213002,2,2,3,1.0);
+sql insert into t2 values(1648791213002,1,2,3,1.0);
+
+$loop_count = 0
+
+loop10:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop10
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop10
+endi
+
+if $data11 != 4 thenloop4
+  print =====data11=$data11
+  goto loop10
+endi
+
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop10
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+$loop_count = 0
+
+loop11:
+sleep 100
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop11
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop11
+endi
+
+if $data11 != 4 then
+  print =====data11=$data11
+  goto loop11
+endi
+
+if $data12 != 1 then
+  print =====data12=$data12
+  goto loop11
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop11
+endi
+
+if $data22 != 2 then
+  print =====data22=$data22
+  goto loop11
+endi
+
+if $data31 != 2 then
+  print =====data31=$data31
+  goto loop11
+endi
+
+if $data32 != 3 then
+  print =====data32=$data32
+  goto loop11
+endi
+
+if $data41 != 4 then
+  print =====data41=$data41
+  goto loop11
+endi
+
+if $data42 != 1 then
+  print =====data42=$data42
+  goto loop11
+endi
+
+sql drop stream if exists streams4;
+sql drop database if exists test4;
+sql create database test4  vgroups 4;
+sql use test4;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create table t3 using st tags(2,2,2);
+sql create table t4 using st tags(2,2,2);
+sql create stream streams4 trigger at_once into test.streamt4 as select  _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s);
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+sql insert into t3 values(1648791213000,2,2,3,1.0);
+sql insert into t4 values(1648791213000,2,2,3,1.0);
+sql insert into t4 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+
+loop13:
+sleep 100
+sql select * from test.streamt4 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop13
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop13
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop13
+endi
+
+if $data11 != 3 then
+  print =====data11=$data11
+  goto loop13
+endi
+
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop13
+endi
+
+sql insert into t4 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791233000,2,2,3,1.0);
+
+
+sql insert into t1 values(1648791213000,1,2,3,1.0); 
+
+$loop_count = 0
+
+loop14:
+sleep 100
+sql select * from test.streamt4 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 20 then
+  return -1
+endi
+
+if $rows != 3 then
+  print =====rows=$rows
+  goto loop14
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop14
+endi
+
+if $data11 != 3 then
+  print =====data11=$data11
+  goto loop14
+endi
+
+if $data21 != 1 then
+  print =====data21=$data21
+  goto loop14
+endi
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest

tests/script/tsim/stream/partitionbyColumn1.sim

@@ -0,0 +1,546 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test  vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select  _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a session(ts, 5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop0:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop0
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop0
+endi
+
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+
+loop1:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop1
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop1
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+
+loop2:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop2
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop2
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791213001,2,2,3,1.0);
+sql insert into t1 values(1648791213002,2,2,3,1.0);
+sql insert into t1 values(1648791213002,1,2,3,1.0);
+
+loop3:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop3
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+if $data11 != 1 then
+  print =====data11=$data11
+  goto loop3
+endi
+
+if $data12 != 1 then
+  print =====data12=$data12
+  goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+loop4:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop4
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop4
+endi
+
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop4
+endi
+
+if $data12 != 1 then
+  print =====data12=$data12
+  goto loop4
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop4
+endi
+
+if $data22 != 1 then
+  print =====data22=$data22
+  goto loop4
+endi
+
+if $data31 != 1 then
+  print =====data31=$data31
+  goto loop4
+endi
+
+if $data32 != 2 then
+  print =====data32=$data32
+  goto loop4
+endi
+
+if $data41 != 1 then
+  print =====data41=$data41
+  goto loop4
+endi
+
+if $data42 != 3 then
+  print =====data42=$data42
+  goto loop4
+endi
+
+sql drop database if exists test1;
+sql create database test1  vgroups 1;
+sql use test1;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams1 trigger at_once into streamt1 as select  _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b session(ts, 5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,1,2,1,1.0);
+sql insert into t1 values(1648791213001,2,1,2,2.0);
+sql insert into t1 values(1648791213001,1,2,3,2.0);
+
+$loop_count = 0
+
+loop5:
+sleep 300
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop5
+endi
+
+sql insert into t1 values(1648791223000,1,2,4,2.0);
+sql insert into t1 values(1648791223001,1,2,5,2.0);
+sql insert into t1 values(1648791223002,1,2,5,2.0);
+sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
+
+loop6:
+sleep 300
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop6
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop6
+endi
+
+if $data11 != 1 then
+  print =====data11=$data11
+  goto loop6
+endi
+
+if $data12 != 6 then
+  print =====data12=$data12
+  goto loop6
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop6
+endi
+
+if $data22 != 5 then
+  print =====data22=$data22
+  goto loop6
+endi
+
+if $data31 != 1 then
+  print =====data31=$data31
+  goto loop6
+endi
+
+if $data32 != 7 then
+  print =====data32=$data32
+  goto loop6
+endi
+
+sql drop database if exists test2;
+sql create database test2  vgroups 4;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select  _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop7
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop7
+endi
+
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+sql insert into t2 values(1648791213000,1,2,3,1.0);
+
+loop8:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop8
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop8
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+loop9:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop9
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop9
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791213001,2,2,3,1.0);
+sql insert into t1 values(1648791213002,2,2,3,1.0);
+sql insert into t1 values(1648791213002,1,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213001,2,2,3,1.0);
+sql insert into t2 values(1648791213002,2,2,3,1.0);
+sql insert into t2 values(1648791213002,1,2,3,1.0);
+
+loop10:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 4 then
+  print =====data01=$data01
+  goto loop10
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop10
+endi
+
+if $data11 != 2 thenloop4
+  print =====data11=$data11
+  goto loop3
+endi
+
+if $data12 != 1 then
+  print =====data12=$data12
+  goto loop10
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+loop11:
+sleep 300
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop11
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop11
+endi
+
+if $data11 != 4 then
+  print =====data11=$data11
+  goto loop11
+endi
+
+if $data12 != 1 then
+  print =====data12=$data12
+  goto loop11
+endi
+
+if $data21 != 4 then
+  print =====data21=$data21
+  goto loop11
+endi
+
+if $data22 != 1 then
+  print =====data22=$data22
+  goto loop11
+endi
+
+if $data31 != 2 then
+  print =====data31=$data31
+  goto loop11
+endi
+
+if $data32 != 2 then
+  print =====data32=$data32
+  goto loop11
+endi
+
+if $data41 != 2 then
+  print =====data41=$data41
+  goto loop11
+endi
+
+if $data42 != 3 then
+  print =====data42=$data42
+  goto loop11
+endi
+
+sql drop database if exists test4;
+sql create database test4  vgroups 4;
+sql use test4;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create table t3 using st tags(2,2,2);
+sql create table t4 using st tags(2,2,2);
+sql create stream streams4 trigger at_once into test.streamt4 as select  _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s);
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+sql insert into t2 values(1648791213000,2,2,3,1.0);
+sql insert into t3 values(1648791213000,2,2,3,1.0);
+sql insert into t4 values(1648791213000,2,2,3,1.0);
+sql insert into t4 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+
+loop13:
+sleep 300
+sql select * from test.streamt4 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop14
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop13
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop13
+endi
+
+if $data11 != 3 then
+  print =====data11=$data11
+  goto loop11
+endi
+
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop11
+endi
+
+sql insert into t4 values(1648791213000,2,2,3,1.0);
+sql insert into t1 values(1648791233000,2,2,3,1.0);
+
+
+sql insert into t1 values(1648791213000,1,2,3,1.0); 
+
+loop14:
+sleep 300
+sql select * from test.streamt4 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 3 then
+  print =====rows=$rows
+  goto loop14
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop14
+endi
+
+if $data11 != 3 then
+  print =====data11=$data11
+  goto loop14
+endi
+
+if $data21 != 1 then
+  print =====data21=$data21
+  goto loop14
+endi
+
+system sh/stop_dnodes.sh
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+#goto looptest

tests/script/tsim/stream/partitionbyColumn2.sim

@@ -0,0 +1,269 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 50
+sql connect
+
+sql drop database if exists test;
+sql create database test  vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select  _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a state_window(b);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop0:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop0
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop0
+endi
+
+
+sql insert into t1 values(1648791213000,1,1,3,1.0);
+
+loop1:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop1
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop1
+endi
+
+sql insert into t1 values(1648791213000,2,1,3,1.0);
+
+loop2:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop2
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop2
+endi
+
+sql insert into t1 values(1648791213000,2,1,3,1.0);
+sql insert into t1 values(1648791213001,2,1,3,1.0);
+sql insert into t1 values(1648791213002,2,1,3,1.0);
+sql insert into t1 values(1648791213002,1,1,3,1.0);
+
+loop3:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop3
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+if $data11 != 1 then
+  print =====data11=$data11
+  goto loop3
+endi
+
+if $data12 != 1 then
+  print =====data12=$data12
+  goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t1 values(1648791213001,1,1,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
+
+loop4:
+sleep 300
+sql select * from streamt order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop4
+endi
+
+if $data02 != 2 then
+  print =====data02=$data02
+  goto loop4
+endi
+
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop4
+endi
+
+if $data12 != 1 then
+  print =====data12=$data12
+  goto loop4
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop4
+endi
+
+if $data22 != 1 then
+  print =====data22=$data22
+  goto loop4
+endi
+
+if $data31 != 1 then
+  print =====data31=$data31
+  goto loop4
+endi
+
+if $data32 != 2 then
+  print =====data32=$data32
+  goto loop4
+endi
+
+if $data41 != 1 then
+  print =====data41=$data41
+  goto loop4
+endi
+
+if $data42 != 3 then
+  print =====data42=$data42
+  goto loop4
+endi
+
+sql drop database if exists test1;
+sql create database test1  vgroups 1;
+sql use test1;
+sql create table t1(ts timestamp, a int, b int , c int, d int);
+sql create stream streams1 trigger at_once into streamt1 as select  _wstart c1, count(*) c2, max(d) c3, _group_key(a+b) c4 from t1 partition by a+b state_window(c);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t1 values(1648791213000,1,2,1,1);
+sql insert into t1 values(1648791213001,2,1,1,2);
+sql insert into t1 values(1648791213001,1,2,1,3);
+
+$loop_count = 0
+
+loop5:
+sleep 300
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop5
+endi
+
+sql insert into t1 values(1648791223000,1,2,2,4);
+sql insert into t1 values(1648791223001,1,2,2,5);
+sql insert into t1 values(1648791223002,1,2,2,6);
+sql insert into t1 values(1648791213001,1,1,1,7) (1648791223002,1,1,2,8);
+
+loop6:
+sleep 300
+sql select * from streamt1 order by c1, c4, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop6
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop6
+endi
+
+if $data11 != 1 then
+  print =====data11=$data11
+  goto loop6
+endi
+
+if $data12 != 7 then
+  print =====data12=$data12
+  goto loop6
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop6
+endi
+
+if $data22 != 5 then
+  print =====data22=$data22
+  goto loop6
+endi
+
+if $data31 != 1 then
+  print =====data31=$data31
+  goto loop6
+endi
+
+if $data32 != 8 then
+  print =====data32=$data32
+  goto loop6
+endi
+
+system sh/stop_dnodes.sh
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+#goto looptest

tests/script/tsim/sync/create-mnode.sim

@@ -0,0 +1,20 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+
+sql connect
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+
+sql create mnode on dnode 2
+sql create mnode on dnode 3

tests/system-test/0-others/user_control.py

@@ -282,12 +282,12 @@ class TDTestCase:
                 use.error(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())")
             elif check_priv == PRIVILEGES_WRITE:
                 use.query(f"use {DBNAME}")
-                use.query(f"show {DBNAME}.tables")
+                use.error(f"show {DBNAME}.tables")
                 use.error(f"select * from {DBNAME}.{CTBNAME}")
                 use.query(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())")
             elif check_priv is None:
                 use.error(f"use {DBNAME}")
-                # use.error(f"show {DBNAME}.tables")
+                use.error(f"show {DBNAME}.tables")
                 use.error(f"show tables")
                 use.error(f"select * from {DBNAME}.{CTBNAME}")
                 use.error(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())")

tests/system-test/2-query/max.py

@@ -26,7 +26,7 @@ class TDTestCase:
                           % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
             intData.append(i + 1)
             floatData.append(i + 0.1)
-        for i in ['ts','col11','col12','col13']:
+        for i in ['col11','col12','col13']:
             for j in ['stb','stb_1']:
                 tdSql.error(f'select max({i} from {dbname}.{j} )')
 
@@ -37,6 +37,20 @@ class TDTestCase:
                     tdSql.checkData(0, 0, np.max(intData))
                 elif i>=9:
                     tdSql.checkData(0, 0, np.max(floatData))
+
+        tdSql.query(f"select max(now()) from {dbname}.stb_1")
+        tdSql.checkRows(1)
+
+        tdSql.query(f"select last(ts) from {dbname}.stb_1")
+        lastTs = tdSql.getData(0, 0)
+        tdSql.query(f"select max(ts) from {dbname}.stb_1")
+        tdSql.checkData(0, 0, lastTs)
+
+        tdSql.query(f"select last(ts) from {dbname}.stb")
+        lastTs = tdSql.getData(0, 0)
+        tdSql.query(f"select max(ts) from {dbname}.stb")
+        tdSql.checkData(0, 0, lastTs)
+
         tdSql.query(f"select max(col1) from {dbname}.stb_1 where col2<=5")
         tdSql.checkData(0,0,5)
         tdSql.query(f"select max(col1) from {dbname}.stb where col2<=5")
@@ -53,7 +67,7 @@ class TDTestCase:
                           % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
             intData.append(i + 1)
             floatData.append(i + 0.1)
-        for i in ['ts','col11','col12','col13']:
+        for i in ['col11','col12','col13']:
             for j in ['ntb']:
                 tdSql.error(f'select max({i} from {dbname}.{j} )')
         for i in range(1,11):
@@ -63,6 +77,15 @@ class TDTestCase:
                     tdSql.checkData(0, 0, np.max(intData))
                 elif i>=9:
                     tdSql.checkData(0, 0, np.max(floatData))
+
+        tdSql.query(f"select max(now()) from {dbname}.ntb")
+        tdSql.checkRows(1)
+
+        tdSql.query(f"select last(ts) from {dbname}.ntb")
+        lastTs = tdSql.getData(0, 0)
+        tdSql.query(f"select max(ts) from {dbname}.ntb")
+        tdSql.checkData(0, 0, lastTs)
+
         tdSql.query(f"select max(col1) from {dbname}.ntb where col2<=5")
         tdSql.checkData(0,0,5)
 

tests/system-test/2-query/min.py

@@ -37,13 +37,11 @@ class TDTestCase:
             floatData.append(i + 0.1)
 
         # max verifacation
-        tdSql.error(f"select min(ts) from {dbname}.stb_1")
         tdSql.error(f"select min(col7) from {dbname}.stb_1")
         tdSql.error(f"select min(col8) from {dbname}.stb_1")
         tdSql.error(f"select min(col9) from {dbname}.stb_1")
         tdSql.error(f"select min(a) from {dbname}.stb_1")
         tdSql.query(f"select min(1) from {dbname}.stb_1")
-        tdSql.error(f"select min(now()) from {dbname}.stb_1")
         tdSql.error(f"select min(count(c1),count(c2)) from {dbname}.stb_1")
 
         tdSql.query(f"select min(col1) from {dbname}.stb_1")
@@ -69,14 +67,25 @@ class TDTestCase:
         tdSql.query(f"select min(col1) from {dbname}.stb_1 where col2>=5")
         tdSql.checkData(0,0,5)
 
+        tdSql.query(f"select min(now()) from {dbname}.stb_1")
+        tdSql.checkRows(1)
+
+        tdSql.query(f"select first(ts) from {dbname}.stb_1")
+        firstTs = tdSql.getData(0, 0)
+        tdSql.query(f"select min(ts) from {dbname}.stb_1")
+        tdSql.checkData(0, 0, firstTs)
+
+        tdSql.query(f"select first(ts) from {dbname}.stb_1")
+        firstTs = tdSql.getData(0, 0)
+        tdSql.query(f"select min(ts) from {dbname}.stb_1")
+        tdSql.checkData(0, 0, firstTs)
+
 
-        tdSql.error(f"select min(ts) from {dbname}.stb_1")
         tdSql.error(f"select min(col7) from {dbname}.stb_1")
         tdSql.error(f"select min(col8) from {dbname}.stb_1")
         tdSql.error(f"select min(col9) from {dbname}.stb_1")
         tdSql.error(f"select min(a) from {dbname}.stb_1")
         tdSql.query(f"select min(1) from {dbname}.stb_1")
-        tdSql.error(f"select min(now()) from {dbname}.stb_1")
         tdSql.error(f"select min(count(c1),count(c2)) from {dbname}.stb_1")
 
         tdSql.query(f"select min(col1) from {dbname}.stb")
@@ -102,13 +111,24 @@ class TDTestCase:
         tdSql.query(f"select min(col1) from {dbname}.stb where col2>=5")
         tdSql.checkData(0,0,5)
 
-        tdSql.error(f"select min(ts) from {dbname}.ntb")
+        tdSql.query(f"select min(now()) from {dbname}.stb_1")
+        tdSql.checkRows(1)
+
+        tdSql.query(f"select first(ts) from {dbname}.stb_1")
+        firstTs = tdSql.getData(0, 0)
+        tdSql.query(f"select min(ts) from {dbname}.stb_1")
+        tdSql.checkData(0, 0, firstTs)
+
+        tdSql.query(f"select first(ts) from {dbname}.stb_1")
+        firstTs = tdSql.getData(0, 0)
+        tdSql.query(f"select min(ts) from {dbname}.stb_1")
+        tdSql.checkData(0, 0, firstTs)
+
         tdSql.error(f"select min(col7) from {dbname}.ntb")
         tdSql.error(f"select min(col8) from {dbname}.ntb")
         tdSql.error(f"select min(col9) from {dbname}.ntb")
         tdSql.error(f"select min(a) from {dbname}.ntb")
         tdSql.query(f"select min(1) from {dbname}.ntb")
-        tdSql.error(f"select min(now()) from {dbname}.ntb")
         tdSql.error(f"select min(count(c1),count(c2)) from {dbname}.ntb")
 
         tdSql.query(f"select min(col1) from {dbname}.ntb")
@@ -134,6 +154,19 @@ class TDTestCase:
         tdSql.query(f"select min(col1) from {dbname}.ntb where col2>=5")
         tdSql.checkData(0,0,5)
 
+        tdSql.query(f"select min(now()) from {dbname}.stb_1")
+        tdSql.checkRows(1)
+
+        tdSql.query(f"select first(ts) from {dbname}.stb_1")
+        firstTs = tdSql.getData(0, 0)
+        tdSql.query(f"select min(ts) from {dbname}.stb_1")
+        tdSql.checkData(0, 0, firstTs)
+
+        tdSql.query(f"select first(ts) from {dbname}.stb_1")
+        firstTs = tdSql.getData(0, 0)
+        tdSql.query(f"select min(ts) from {dbname}.stb_1")
+        tdSql.checkData(0, 0, firstTs)
+
 
     def stop(self):
         tdSql.close()

tests/system-test/7-tmq/tmqShow.py

@@ -19,6 +19,11 @@ class TDTestCase:
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
         #tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+    def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,offset=1,cdbName='cdb'):
+        sql = "insert into %s.consumeinfo values "%cdbName
+        sql += "(now+%ds, %d, '%s', '%s', %d, %d, %d)"%(offset,consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit)
+        tdLog.info("consume info sql: %s"%sql)
+        tdSql.query(sql)
 
     def tmqCase1(self):
         tdLog.printNoPrefix("======== test case 1: ")
@@ -95,19 +100,23 @@ class TDTestCase:
         ifcheckdata  = 0
         ifManualCommit = 0
         keyList      = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[0]
-        tmqCom.insertConsumerInfo(consumerIdList[0], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        tsOffset=1
+        self.insertConsumerInfo(consumerIdList[0], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit,tsOffset)
 
         topicList    = topicNameList[1]
         keyList      = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[1]
-        tmqCom.insertConsumerInfo(consumerIdList[1], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        tsOffset=2
+        self.insertConsumerInfo(consumerIdList[1], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit,tsOffset)
 
         topicList    = topicNameList[2]
         keyList      = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[2]
-        tmqCom.insertConsumerInfo(consumerIdList[2], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        tsOffset=3
+        self.insertConsumerInfo(consumerIdList[2], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit,tsOffset)
 
         topicList    = topicNameList[3]
         keyList      = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[3]
-        tmqCom.insertConsumerInfo(consumerIdList[3], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+        tsOffset=4
+        self.insertConsumerInfo(consumerIdList[3], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit,tsOffset)
 
         tdLog.info("start consume processor")
         tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])

tests/system-test/fulltest.sh

@@ -314,8 +314,8 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py
 python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py
 python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
 python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
-#python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
+python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
-#python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
+python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
 python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
 python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py
 python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py
@@ -514,3 +514,4 @@ python3 ./test.py -f 2-query/last_row.py -Q 3
 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3
 python3 ./test.py -f 2-query/sml.py -Q 3
 python3 ./test.py -f 2-query/interp.py -Q 3
+