add fulltest.sh

Jenkinsfile2

@@ -269,7 +269,7 @@ pipeline {
                     }
                 }
                 stage('linux test') {
-                    agent{label " slave3_0 || slave15 || slave16 || slave17 "}
+                    agent{label " worker03 || slave215 || slave217 || slave219 "}
                     options { skipDefaultCheckout() } 
                     when {
                         changeRequest()
@@ -287,9 +287,9 @@ pipeline {
                                 '''
                                 sh '''
                                     cd ${WKC}/tests/parallel_test
-                                    export DEFAULT_RETRY_TIME=1
+                                    export DEFAULT_RETRY_TIME=2
                                     date
-                                    timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME} -l ${WKDIR}/log -o 480
+                                    timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480
                                 '''
                             }
                         }

cmake/cmake.options

@@ -49,7 +49,7 @@ IF(${TD_WINDOWS})
     option(
         BUILD_TEST 
         "If build unit tests using googletest" 
-        OFF
+        ON
     )
 ELSE ()
 

contrib/test/craft/raftMain.c

@@ -243,7 +243,7 @@ void console(SRaftServer *pRaftServer) {
 
 
 		} else if (strcmp(cmd, "dropnode") == 0) {
-			char host[HOST_LEN];
+			char host[HOST_LEN] = {0};
 			uint32_t port;
 			parseAddr(param1, host, HOST_LEN, &port);
 			uint64_t rid = raftId(host, port);
@@ -258,7 +258,7 @@ void console(SRaftServer *pRaftServer) {
 
 
 		} else if (strcmp(cmd, "put") == 0) {
-			char buf[256];
+			char buf[256] = {0};
 			snprintf(buf, sizeof(buf), "%s--%s", param1, param2);
 			putValue(&pRaftServer->raft, buf);
 

docs-cn/07-develop/06-subscribe.mdx

@@ -145,7 +145,7 @@ void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
 taos_unsubscribe(tsub, keep);
 ```
 
-其第二个参数，用于决定是否在客户端保留订阅的进度信息。如果这个参数是**false**（**0**），那无论下次调用 `taos_subscribe` 时的 `restart` 参数是什么，订阅都只能重新开始。另外，进度信息的保存位置是 _{DataDir}/subscribe/_ 这个目录下，每个订阅有一个与其 `topic` 同名的文件，删掉某个文件，同样会导致下次创建其对应的订阅时只能重新开始。
+其第二个参数，用于决定是否在客户端保留订阅的进度信息。如果这个参数是**false**（**0**），那无论下次调用 `taos_subscribe` 时的 `restart` 参数是什么，订阅都只能重新开始。另外，进度信息的保存位置是 _{DataDir}/subscribe/_ 这个目录下（注：`taos.cfg` 配置文件中 `DataDir` 参数值默认为 **/var/lib/taos/**,但是 Windows 服务器上本身不存在该目录，所以需要在 Windows 的配置文件中修改 `DataDir` 参数值为相应的已存在目录"），每个订阅有一个与其 `topic` 同名的文件，删掉某个文件，同样会导致下次创建其对应的订阅时只能重新开始。
 
 代码介绍完毕，我们来看一下实际的运行效果。假设：
 

docs-cn/12-taos-sql/07-function.md

@@ -1766,6 +1766,8 @@ SELECT TIMEDIFF(ts_val1 | datetime_string1 | ts_col1, ts_val2 | datetime_string2
           1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)。
 - 如果时间单位 time_unit 未指定， 返回的时间差值精度与当前 DATABASE 设置的时间精度一致。
 
+**支持的版本**：2.6.0.0 及以后的版本。
+
 **示例**：
 
 ```sql

docs-cn/27-train-faq/01-faq.md

@@ -128,19 +128,30 @@ properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8");
 Connection = DriverManager.getConnection(url, properties);
 ```
 
-### 13.JDBC 报错： the executed SQL is not a DML or a DDL？
+### 13.  Windows 系统下客户端无法正常显示中文字符？
+
+Windows 系统中一般是采用 GBK/GB18030 存储中文字符，而 TDengine 的默认字符集为 UTF-8 ，在 Windows 系统中使用 TDengine 客户端时，客户端驱动会将字符统一转换为 UTF-8 编码后发送到服务端存储，因此在应用开发过程中，调用接口时正确配置当前的中文字符集即可。
+
+【 v2.2.1.5以后版本 】在 Windows 10 环境下运行 TDengine 客户端命令行工具 taos 时，若无法正常输入、显示中文，可以对客户端 taos.cfg 做如下配置：
+
+```
+locale C 
+charset UTF-8
+```
+
+### 14. JDBC 报错： the executed SQL is not a DML or a DDL？
 
 请更新至最新的 JDBC 驱动，参考 [Java 连接器](/reference/connector/java)
 
-### 14. taos connect failed, reason: invalid timestamp
+### 15. taos connect failed, reason: invalid timestamp
 
 常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。
 
-### 15. 表名显示不全
+### 16. 表名显示不全
 
 由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。
 
-### 16. 如何进行数据迁移？
+### 17. 如何进行数据迁移？
 
 TDengine 是根据 hostname 唯一标志一台机器的，在数据文件从机器 A 移动机器 B 时，注意如下两件事：
 
@@ -148,7 +159,7 @@ TDengine 是根据 hostname 唯一标志一台机器的，在数据文件从机
  - 2.0.7.0 及以后的版本，到/var/lib/taos/dnode 下，修复 dnodeEps.json 的 dnodeId 对应的 FQDN，重启。确保机器内所有机器的此文件是完全相同的。
  - 1.x 和 2.x 版本的存储结构不兼容，需要使用迁移工具或者自己开发应用导出导入数据。
 
-### 17. 如何在命令行程序 taos 中临时调整日志级别
+### 18. 如何在命令行程序 taos 中临时调整日志级别
 
 为了调试方便，从 2.0.16 版本开始，命令行程序 taos 新增了与日志记录相关的两条指令：
 
@@ -169,7 +180,7 @@ ALTER LOCAL RESETLOG;
 
 <a class="anchor" id="timezone"></a>
 
-### 18. go 语言编写组件编译失败怎样解决？
+### 19. go 语言编写组件编译失败怎样解决？
 
 TDengine 2.3.0.0 及之后的版本包含一个使用 go 语言开发的 taosAdapter 独立组件，需要单独运行，取代之前 taosd 内置的 httpd ，提供包含原 httpd 功能以及支持多种其他软件（Prometheus、Telegraf、collectd、StatsD 等）的数据接入功能。
 使用最新 develop 分支代码编译需要先 `git submodule update --init --recursive` 下载 taosAdapter 仓库代码后再编译。
@@ -184,7 +195,7 @@ go env -w GOPROXY=https://goproxy.cn,direct
 如果希望继续使用之前的内置 httpd，可以关闭 taosAdapter 编译，使用
 `cmake .. -DBUILD_HTTP=true` 使用原来内置的 httpd。
 
-### 19. 如何查询数据占用的存储空间大小？
+### 20. 如何查询数据占用的存储空间大小？
 
 默认情况下，TDengine 的数据文件存储在 /var/lib/taos ，日志文件存储在 /var/log/taos 。
 
@@ -193,3 +204,50 @@ go env -w GOPROXY=https://goproxy.cn,direct
 若想查看单个数据库占用的大小，可在命令行程序 taos 内指定要查看的数据库后执行 `show vgroups;` ，通过得到的 VGroup id 去 /var/lib/taos/vnode 下查看包含的文件夹大小。
 
 若仅仅想查看指定（超级）表的数据块分布及大小，可查看[_block_dist 函数](https://docs.taosdata.com/taos-sql/select/#_block_dist-%E5%87%BD%E6%95%B0)
+
+### 21. 客户端连接串如何保证高可用？
+
+请看为此问题撰写的 [技术博客](https://www.taosdata.com/blog/2021/04/16/2287.html)
+
+### 22. 时间戳的时区信息是怎样处理的？
+
+TDengine 中时间戳的时区总是由客户端进行处理，而与服务端无关。具体来说，客户端会对 SQL 语句中的时间戳进行时区转换，转为 UTC 时区（即 Unix 时间戳——Unix Timestamp）再交由服务端进行写入和查询；在读取数据时，服务端也是采用 UTC 时区提供原始数据，客户端收到后再根据本地设置，把时间戳转换为本地系统所要求的时区进行显示。
+
+客户端在处理时间戳字符串时，会采取如下逻辑：
+
+1. 在未做特殊设置的情况下，客户端默认使用所在操作系统的时区设置。
+2. 如果在 taos.cfg 中设置了 timezone 参数，则客户端会以这个配置文件中的设置为准。
+3. 如果在 C/C++/Java/Python 等各种编程语言的 Connector Driver 中，在建立数据库连接时显式指定了 timezone，那么会以这个指定的时区设置为准。例如 Java Connector 的 JDBC URL 中就有 timezone 参数。
+4. 在书写 SQL 语句时，也可以直接使用 Unix 时间戳（例如 `1554984068000`）或带有时区的时间戳字符串，也即以 RFC 3339 格式（例如 `2013-04-12T15:52:01.123+08:00`）或 ISO-8601 格式（例如 `2013-04-12T15:52:01.123+0800`）来书写时间戳，此时这些时间戳的取值将不再受其他时区设置的影响。
+
+### 23. TDengine 2.0 都会用到哪些网络端口？
+
+在 TDengine 2.0 版本中，会用到以下这些网络端口（以默认端口 6030 为前提进行说明，如果修改了配置文件中的设置，那么这里列举的端口都会随之出现变化），管理员可以参考这里的信息调整防火墙设置：
+
+| 协议 | 默认端口   | 用途说明                         | 修改方法                        |
+| :--- | :-------- | :---------------------------------- | :------------------------------- |
+| TCP | 6030      | 客户端与服务端之间通讯。            | 由配置文件设置 serverPort 决定。 |
+| TCP | 6035      | 多节点集群的节点间通讯。            | 随 serverPort 端口变化。        |
+| TCP | 6040      | 多节点集群的节点间数据同步。        | 随 serverPort 端口变化。         |
+| TCP | 6041      | 客户端与服务端之间的 RESTful 通讯。 | 随 serverPort 端口变化。2.4.0.0 及以上版本由 taosAdapter 配置。  |
+| TCP | 6042      | Arbitrator 的服务端口。           | 随 Arbitrator 启动参数设置变化。 |
+| TCP | 6043      | TaosKeeper 监控服务端口。         | 随 TaosKeeper 启动参数设置变化。 |
+| TCP | 6044      | 支持 StatsD 的数据接入端口。       | 随 taosAdapter 启动参数设置变化（ 2.4.0.0 及以上版本）。 |
+| UDP | 6045      | 支持 collectd 数据接入端口。       | 随 taosAdapter 启动参数设置变化（ 2.4.0.0 及以上版本）。 |
+| TCP | 6060      | 企业版内 Monitor 服务的网络端口。   |                               |
+| UDP | 6030-6034 | 客户端与服务端之间通讯。            | 随 serverPort 端口变化。        |
+| UDP | 6035-6039 | 多节点集群的节点间通讯。            | 随 serverPort 端口变化。        |
+
+### 24. 为什么 RESTful 接口无响应、Grafana 无法添加 TDengine 为数据源、TDengineGUI 选了 6041 端口还是无法连接成功？？
+
+taosAdapter 从 TDengine 2.4.0.0 版本开始成为 TDengine 服务端软件的组成部分，是 TDengine 集群和应用程序之间的桥梁和适配器。在此之前 RESTful 接口等功能是由 taosd 内置的 HTTP 服务提供的，而如今要实现上述功能需要执行：```systemctl start taosadapter``` 命令来启动 taosAdapter 服务。
+
+需要说明的是，taosAdapter 的日志路径 path 需要单独配置，默认路径是 /var/log/taos ；日志等级 logLevel 有 8 个等级，默认等级是 info ，配置成 panic 可关闭日志输出。请注意操作系统 / 目录的空间大小，可通过命令行参数、环境变量或配置文件来修改配置，默认配置文件是 /etc/taos/taosadapter.toml 。
+
+有关 taosAdapter 组件的详细介绍请看文档：[taosAdapter](https://docs.taosdata.com/reference/taosadapter/)
+
+### 25. 发生了 OOM 怎么办？
+
+OOM 是操作系统的保护机制，当操作系统内存(包括 SWAP )不足时，会杀掉某些进程，从而保证操作系统的稳定运行。通常内存不足主要是如下两个原因导致，一是剩余内存小于 vm.min_free_kbytes ；二是程序请求的内存大于剩余内存。还有一种情况是内存充足但程序占用了特殊的内存地址，也会触发 OOM 。
+
+TDengine 会预先为每个 VNode 分配好内存，每个 Database 的 VNode 个数受 maxVgroupsPerDb 影响，每个 VNode 占用的内存大小受 Blocks 和 Cache 影响。要防止 OOM，需要在项目建设之初合理规划内存，并合理设置 SWAP ，除此之外查询过量的数据也有可能导致内存暴涨，这取决于具体的查询语句。TDengine 企业版对内存管理做了优化，采用了新的内存分配器，对稳定性有更高要求的用户可以考虑选择企业版。

docs-en/07-develop/01-connect/index.md

@@ -1,6 +1,6 @@
 ---
-sidebar_label: Connection
+sidebar_label: Connect
-title: Connect to TDengine
+title: Connect
 description: "This document explains how to establish connections to TDengine, and briefly introduces how to install and use TDengine connectors."
 ---
 

docs-en/07-develop/03-insert-data/01-sql-writing.mdx

@@ -1,5 +1,5 @@
 ---
-sidebar_label: SQL
+sidebar_label: Insert Using SQL
 title: Insert Using SQL
 ---
 

docs-en/07-develop/03-insert-data/index.md

@@ -1,5 +1,5 @@
 ---
-title: Insert
+title: Insert Data
 ---
 
 TDengine supports multiple protocols of inserting data, including SQL, InfluxDB Line protocol, OpenTSDB Telnet protocol, and OpenTSDB JSON protocol. Data can be inserted row by row, or in batches. Data from one or more collection points can be inserted simultaneously. Data can be inserted with multiple threads, and out of order data and historical data can be inserted as well. InfluxDB Line protocol, OpenTSDB Telnet protocol and OpenTSDB JSON protocol are the 3 kinds of schemaless insert protocols supported by TDengine. It's not necessary to create STables and tables in advance if using schemaless protocols, and the schemas can be adjusted automatically based on the data being inserted.

docs-en/07-develop/04-query-data/_category_.yml

@@ -1 +1 @@
-label: Select Data
+label: Query Data

docs-en/07-develop/04-query-data/index.mdx

@@ -1,6 +1,6 @@
 ---
-Sidebar_label: Select
+Sidebar_label: Query data
-title: Select
+title: Query data
 description: "This chapter introduces major query functionalities and how to perform sync and async query using connectors."
 ---
 

docs-en/07-develop/06-subscribe.mdx

@@ -1,5 +1,5 @@
 ---
-sidebar_label: Subscription
+sidebar_label: Data Subscription
 description: "Lightweight service for data subscription and publishing. Time series data inserted into TDengine continuously can be pushed automatically to subscribing clients."
 title: Data Subscription
 ---
@@ -151,7 +151,7 @@ void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
 taos_unsubscribe(tsub, keep);
 ```
 
-The second parameter `keep` is used to specify whether to keep the subscription progress on the client sde. If it is **false**, i.e. **0**, then subscription will be restarted from beginning regardless of the `restart` parameter's value when `taos_subscribe` is invoked again. The subscription progress information is stored in _{DataDir}/subscribe/_ , under which there is a file with the same name as `topic` for each subscription, the subscription will be restarted from the beginning if the corresponding progress file is removed.
+The second parameter `keep` is used to specify whether to keep the subscription progress on the client sde. If it is **false**, i.e. **0**, then subscription will be restarted from beginning regardless of the `restart` parameter's value when `taos_subscribe` is invoked again. The subscription progress information is stored in _{DataDir}/subscribe/_ , under which there is a file with the same name as `topic` for each subscription(Note: The default value of `DataDir` in the `taos.cfg` file is **/var/lib/taos/**. However, **/var/lib/taos/** does not exist on the Windows server. So you need to change the `DataDir` value to the corresponding existing directory."), the subscription will be restarted from the beginning if the corresponding progress file is removed.
 
 Now let's see the effect of the above sample code, assuming below prerequisites have been done.
 

docs-en/07-develop/08-udf.md

@@ -1,6 +1,6 @@
 ---
 sidebar_label: UDF
-title: User Defined Functions
+title: User Defined Functions(UDF)
 description: "Scalar functions and aggregate functions developed by users can be utilized by the query framework to expand query capability"
 ---
 

docs-en/12-taos-sql/07-function.md

@@ -22,8 +22,8 @@ SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause];
 
 **More explanation**:
 
-- Wildcard (\*) can be used to represent all columns, it's used to get the number of all rows
+- Wildcard (\*) is used to represent all columns. The `COUNT` function is used to get the total number of all rows.
-- The number of non-NULL values will be returned if this function is used on a specific column
+- The number of non-NULL values will be returned if this function is used on a specific column.
 
 **Examples**:
 
@@ -87,7 +87,7 @@ SELECT TWA(field_name) FROM tb_name WHERE clause;
 
 **More explanations**:
 
-- From version 2.1.3.0, function TWA can be used on stable with `GROUP BY`, i.e. timelines generated by `GROUP BY tbname` on a STable.
+- Since version 2.1.3.0, function TWA can be used on stable with `GROUP BY`, i.e. timelines generated by `GROUP BY tbname` on a STable.
 
 ### IRATE
 
@@ -105,7 +105,7 @@ SELECT IRATE(field_name) FROM tb_name WHERE clause;
 
 **More explanations**:
 
-- From version 2.1.3.0, function IRATE can be used on stble with `GROUP BY`, i.e. timelines generated by `GROUP BY tbname` on a STable.
+- Since version 2.1.3.0, function IRATE can be used on stble with `GROUP BY`, i.e. timelines generated by `GROUP BY tbname` on a STable.
 
 ### SUM
 
@@ -149,7 +149,7 @@ SELECT STDDEV(field_name) FROM tb_name [WHERE clause];
 
 **Applicable column types**: Data types except for timestamp, binary, nchar and bool
 
-**Applicable table types**: table, STable (starting from version 2.0.15.1)
+**Applicable table types**: table, STable (since version 2.0.15.1)
 
 **Examples**:
 
@@ -193,13 +193,13 @@ SELECT MODE(field_name) FROM tb_name [WHERE clause];
 
 **Description**:The value which has the highest frequency of occurrence. NULL is returned if there are multiple values which have highest frequency of occurrence. It can't be used on timestamp column or tags.
 
-**Return value type**:Same as the data type of the column being operated
+**Return value type**:Same as the data type of the column being operated upon
 
 **Applicable column types**:Data types except for timestamp
 
 **More explanations**:Considering the number of returned result set is unpredictable, it's suggested to limit the number of unique values to 100,000, otherwise error will be returned.
 
-**Applicable version**:From version 2.6.0.0
+**Applicable version**:Since version 2.6.0.0
 
 **Examples**:
 
@@ -234,7 +234,7 @@ SELECT HYPERLOGLOG(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **More explanations**: The benefit of using hyperloglog algorithm is that the memory usage is under control when the data volume is huge. However, when the data volume is very small, the result may be not accurate, it's recommented to use `select count(data) from (select unique(col) as data from table)` in this case.
 
-**Applicable versions**:From version 2.6.0.0
+**Applicable versions**:Since version 2.6.0.0
 
 **Examples**:
 
@@ -271,7 +271,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause];
 
 **Description**: The minimum value of a specific column in a table or STable
 
-**Return value type**: Same as the data type of the column being operated
+**Return value type**: Same as the data type of the column being operated upon
 
 **Applicable column types**: Data types except for timestamp, binary, nchar and bool
 
@@ -301,7 +301,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **Description**: The maximum value of a specific column of a table or STable
 
-**Return value type**: Same as the data type of the column being operated
+**Return value type**: Same as the data type of the column being operated upon
 
 **Applicable column types**: Data types except for timestamp, binary, nchar and bool
 
@@ -331,7 +331,7 @@ SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **Description**: The first non-null value of a specific column in a table or STable
 
-**Return value type**: Same as the column being operated
+**Return value type**: Same as the column being operated upon
 
 **Applicable column types**: Any data type
 
@@ -341,7 +341,7 @@ SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 - FIRST(\*) can be used to get the first non-null value of all columns
 - NULL will be returned if all the values of the specified column are all NULL
-- No result will NOT be returned if all the columns in the result set are all NULL
+- A result will NOT be returned if all the columns in the result set are all NULL
 
 **Examples**:
 
@@ -367,7 +367,7 @@ SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **Description**: The last non-NULL value of a specific column in a table or STable
 
-**Return value type**: Same as the column being operated
+**Return value type**: Same as the column being operated upon
 
 **Applicable column types**: Any data type
 
@@ -403,7 +403,7 @@ SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause];
 
 **Description**: The greatest _k_ values of a specific column in a table or STable. If a value has multiple occurrences in the column but counting all of them in will exceed the upper limit _k_, then a part of them will be returned randomly.
 
-**Return value type**: Same as the column being operated
+**Return value type**: Same as the column being operated upon
 
 **Applicable column types**: Data types except for timestamp, binary, nchar and bool
 
@@ -442,7 +442,7 @@ SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause];
 
 **Description**: The least _k_ values of a specific column in a table or STable. If a value has multiple occurrences in the column but counting all of them in will exceed the upper limit _k_, then a part of them will be returned randomly.
 
-**Return value type**: Same as the column being operated
+**Return value type**: Same as the column being operated upon
 
 **Applicable column types**: Data types except for timestamp, binary, nchar and bool
 
@@ -549,7 +549,7 @@ SELECT LAST_ROW(field_name) FROM { tb_name | stb_name };
 
 **Description**: The last row of a table or STable
 
-**Return value type**: Same as the column being operated
+**Return value type**: Same as the column being operated upon
 
 **Applicable column types**: Any data type
 
@@ -576,7 +576,7 @@ SELECT LAST_ROW(field_name) FROM { tb_name | stb_name };
  Query OK, 1 row(s) in set (0.001042s)
 ```
 
-### INTERP [From version 2.3.1]
+### INTERP [Since version 2.3.1]
 
 ```
 SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [ RANGE(timestamp1,timestamp2) ] [EVERY(interval)] [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})];
@@ -584,7 +584,7 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [
 
 **Description**: The value that matches the specified timestamp range is returned, if existing; or an interpolation value is returned.
 
-**Return value type**: Same as the column being operated
+**Return value type**: Same as the column being operated upon
 
 **Applicable column types**: Numeric data types
 
@@ -593,7 +593,7 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [
 **More explanations**
 
 - `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter.
-- The input data of `INTERP` is the value of the specified column, `where` can be used to filter the original data. If no `where` condition is specified then all original data is the input.
+- The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input.
 - The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1<=timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified. If `RANGE` is not specified, then the timestamp of the first row that matches the filter condition is treated as timestamp1, the timestamp of the last row that matches the filter condition is treated as timestamp2.
 - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. If `EVERY` parameter is not used, the time windows will be considered as no ending timestamp, i.e. there is only one time window from timestamp1.
 - Interpolation is performed based on `FILL` parameter. No interpolation is performed if `FILL` is not used, that means either the original data that matches is returned or nothing is returned.
@@ -632,7 +632,7 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] [
   taos> SELECT INTERP(current) FROM t1 where ts >= '2017-07-14 17:00:00' and ts <= '2017-07-14 20:00:00' RANGE('2017-7-14 18:00:00','2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR);
 ```
 
-### INTERP [Prior to version 2.3.1]
+### INTERP [Since version 2.0.15.0]
 
 ```
 SELECT INTERP(field_name) FROM { tb_name | stb_name } WHERE ts='timestamp' [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})];
@@ -640,7 +640,7 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } WHERE ts='timestamp' [FILL
 
 **Description**: The value of a specific column that matches the specified time slice
 
-**Return value type**: Same as the column being operated
+**Return value type**: Same as the column being operated upon
 
 **Applicable column types**: Numeric data type
 
@@ -648,7 +648,6 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } WHERE ts='timestamp' [FILL
 
 **More explanations**:
 
-- It can be used from version 2.0.15.0
 - Time slice must be specified. If there is no data matching the specified time slice, interpolation is performed based on `FILL` parameter. Conditions such as tags or `tbname` can be used `Where` clause can be used to filter data.
 - The timestamp specified must be within the time range of the data rows of the table or STable. If it is beyond the valid time range, nothing is returned even with `FILL` parameter.
 - `INTERP` can be used to query only single time point once. `INTERP` can be used with `EVERY` to get the interpolation value every time interval.
@@ -696,11 +695,11 @@ SELECT TAIL(field_name, k, offset_val) FROM {tb_name | stb_name} [WHERE clause];
 
 **Parameter value range**: k: [1,100] offset_val: [0,100]
 
-**Return value type**: Same as the column being operated
+**Return value type**: Same as the column being operated upon
 
 **Applicable column types**: Any data type except form timestamp, i.e. the primary key
 
-**Applicable versions**: From version 2.6.0.0
+**Applicable versions**: Since version 2.6.0.0
 
 **Examples**:
 
@@ -732,11 +731,11 @@ SELECT UNIQUE(field_name) FROM {tb_name | stb_name} [WHERE clause];
 
 **Description**: The values that occur the first time in the specified column. The effect is similar to `distinct` keyword, but it can also be used to match tags or timestamp.
 
-**Return value type**: Same as the column or tag being operated
+**Return value type**: Same as the column or tag being operated upon
 
 **Applicable column types**: Any data types except for timestamp
 
-**Applicable versions**: From version 2.6.0.0
+**Applicable versions**: Since version 2.6.0.0
 
 **More explanations**:
 
@@ -780,7 +779,7 @@ SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHER
 
 **Description**: The different of each row with its previous row for a specific column. `ignore_negative` can be specified as 0 or 1, the default value is 1 if it's not specified. `1` means negative values are ignored.
 
-**Return value type**: Same as the column being operated
+**Return value type**: Same as the column being operated upon
 
 **Applicable column types**: Data types except for timestamp, binary, nchar and bool
 
@@ -789,8 +788,8 @@ SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHER
 **More explanations**:
 
 - The number of result rows is the number of rows subtracted by one, no output for the first row
-- From version 2.1.30, `DIFF` can be used on STable with `GROUP by tbname`
+- Since version 2.1.30, `DIFF` can be used on STable with `GROUP by tbname`
-- From version 2.6.0, `ignore_negative` parameter is supported
+- Since version 2.6.0, `ignore_negative` parameter is supported
 
 **Examples**:
 
@@ -874,7 +873,7 @@ Query OK, 1 row(s) in set (0.000836s)
 SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause];
 ```
 
-**Description**: The round up value of a specific column
+**Description**: The rounded up value of a specific column
 
 **Return value type**: Same as the column being used
 
@@ -896,9 +895,9 @@ SELECT CEIL(field_name) FROM { tb_name | stb_name } [WHERE clause];
 SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause];
 ```
 
-**Description**: The round down value of a specific column
+**Description**: The rounded down value of a specific column
 
-**More explanations**: The restrictions are same as `CEIL` function.
+**More explanations**: The restrictions are same as those of the `CEIL` function.
 
 ### ROUND
 
@@ -906,7 +905,7 @@ SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause];
 SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 ```
 
-**Description**: The round value of a specific column.
+**Description**: The rounded value of a specific column.
 
 **More explanations**: The restrictions are same as `CEIL` function.
 
@@ -933,7 +932,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 - Can only be used with aggregate functions
 - `Group by tbname` must be used together on a STable to force the result on a single timeline
 
-**Applicable versions**: From 2.3.0.x
+**Applicable versions**: Since 2.3.0.x
 
 ### MAVG
 
@@ -958,7 +957,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 - Can't be used with aggregate functions.
 - Must be used with `GROUP BY tbname` when it's used on a STable to force the result on each single timeline.
 
-**Applicable versions**: From 2.3.0.x
+**Applicable versions**: Since 2.3.0.x
 
 ### SAMPLE
 
@@ -981,7 +980,7 @@ SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
 - Arithmetic operation can't be operated on the result of `SAMPLE` function
 - Must be used with `Group by tbname` when it's used on a STable to force the result on each single timeline
 
-**Applicable versions**: From 2.3.0.x
+**Applicable versions**: Since 2.3.0.x
 
 ### ASIN
 
@@ -1460,8 +1459,8 @@ SELECT field_name [+|-|*|/|%][Value|field_name] FROM { tb_name | stb_name }  [WH
 
 **More explanations**:
 
-- Arithmetic operations can be performed on two or more columns, `()` can be used to control the precedence
+- Arithmetic operations can be performed on two or more columns, Parentheses `()` can be used to control the order of precedence.
-- NULL doesn't participate the operation, if one of the operands is NULL then result is NULL
+- NULL doesn't participate in the operation i.e. if one of the operands is NULL then result is NULL.
 
 **Examples**:
 
@@ -1586,7 +1585,7 @@ Query OK, 6 row(s) in set (0.002613s)
 
 ## Time Functions
 
-From version 2.6.0.0, below time related functions can be used in TDengine.
+Since version 2.6.0.0, below time related functions can be used in TDengine.
 
 ### NOW
 
@@ -1840,6 +1839,8 @@ SELECT TIMEDIFF(ts_val1 | datetime_string1 | ts_col1, ts_val2 | datetime_string2
   1u(microsecond),1a(millisecond),1s(second),1m(minute),1h(hour),1d(day).
 - The precision of the returned timestamp is same as the precision set for the current data base in use
 
+**Applicable versions**:Since version 2.6.0.0
+
 **Examples**:
 
 ```sql

docs-en/12-taos-sql/08-interval.md

@@ -3,36 +3,36 @@ sidebar_label: Interval
 title: Aggregate by Time Window
 ---
 
-Aggregate by time window is supported in TDengine. For example, each temperature sensor reports the temperature every second, the average temperature every 10 minutes can be retrieved by query with time window.
+Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window.
-Window related clauses are used to divide the data set to be queried into subsets and then aggregate. There are three kinds of windows, time window, status window, and session window. There are two kinds of time windows, sliding window and flip time window.
+Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are three kinds of windows: time window, status window, and session window. There are two kinds of time windows: sliding window and flip time/tumbling window.
 
 ## Time Window
 
-`INTERVAL` clause is used to generate time windows of the same time interval, `SLIDING` is used to specify the time step for which the time window moves forward. The query is performed on one time window each time, and the time window moves forward with time. When defining continuous query both the size of time window and the step of forward sliding time need to be specified. As shown in the figure blow, [t0s, t0e] ，[t1s , t1e]， [t2s, t2e] are respectively the time ranges of three time windows on which continuous queries are executed. The time step for which time window moves forward is marked by `sliding time`. Query, filter and aggregate operations are executed on each time window respectively. When the time step specified by `SLIDING` is same as the time interval specified by `INTERVAL`, the sliding time window is actually a flip time window.
+The `INTERVAL` clause is used to generate time windows of the same time interval. The `SLIDING` parameter is used to specify the time step for which the time window moves forward. The query is performed on one time window each time, and the time window moves forward with time. When defining a continuous query, both the size of the time window and the step of forward sliding time need to be specified. As shown in the figure blow, [t0s, t0e] ，[t1s , t1e]， [t2s, t2e] are respectively the time ranges of three time windows on which continuous queries are executed. The time step for which time window moves forward is marked by `sliding time`. Query, filter and aggregate operations are executed on each time window respectively. When the time step specified by `SLIDING` is same as the time interval specified by `INTERVAL`, the sliding time window is actually a flip time/tumbling window.
 
 ![TDengine Database Time Window](./timewindow-1.webp)
 
-`INTERVAL` and `SLIDING` should be used with aggregate functions and select functions. Below SQL statement is illegal because no aggregate or selection function is used with `INTERVAL`.
+`INTERVAL` and `SLIDING` should be used with aggregate functions and select functions. The SQL statement below is illegal because no aggregate or selection function is used with `INTERVAL`.
 
 ```
 SELECT * FROM temp_tb_1 INTERVAL(1m);
 ```
 
-The time step specified by `SLIDING` can't exceed the time interval specified by `INTERVAL`. Below SQL statement is illegal because the time length specified by `SLIDING` exceeds that specified by `INTERVAL`.
+The time step specified by `SLIDING` cannot exceed the time interval specified by `INTERVAL`. The SQL statement below is illegal because the time length specified by `SLIDING` exceeds that specified by `INTERVAL`.
 
 ```
 SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m);
 ```
 
-When the time length specified by `SLIDING` is the same as that specified by `INTERVAL`, the sliding window is actually a flip window. The minimum time range specified by `INTERVAL` is 10 milliseconds (10a) prior to version 2.1.5.0. From version 2.1.5.0, the minimum time range by `INTERVAL` can be 1 microsecond (1u). However, if the DB precision is millisecond, the minimum time range is 1 millisecond (1a). Please note that the `timezone` parameter should be configured to be the same value in the `taos.cfg` configuration file on client side and server side.
+When the time length specified by `SLIDING` is the same as that specified by `INTERVAL`, the sliding window is actually a flip/tumbling window. The minimum time range specified by `INTERVAL` is 10 milliseconds (10a) prior to version 2.1.5.0. Since version 2.1.5.0, the minimum time range by `INTERVAL` can be 1 microsecond (1u). However, if the DB precision is millisecond, the minimum time range is 1 millisecond (1a). Please note that the `timezone` parameter should be configured to be the same value in the `taos.cfg` configuration file on client side and server side.
 
 ## Status Window
 
-In case of using integer, bool, or string to represent the device status at a moment, the continuous rows with same status belong to same status window. Once the status changes, the status window closes. As shown in the following figure, there are two status windows according to status, [2019-04-28 14:22:07，2019-04-28 14:22:10] and [2019-04-28 14:22:11，2019-04-28 14:22:12]. Status window is not applicable to STable for now.
+In case of using integer, bool, or string to represent the status of a device at any given moment, continuous rows with the same status belong to a status window. Once the status changes, the status window closes. As shown in the following figure, there are two status windows according to status, [2019-04-28 14:22:07，2019-04-28 14:22:10] and [2019-04-28 14:22:11，2019-04-28 14:22:12]. Status window is not applicable to STable for now.
 
 ![TDengine Database Status Window](./timewindow-3.webp)
 
-`STATE_WINDOW` is used to specify the column based on which to define status window, for example：
+`STATE_WINDOW` is used to specify the column on which the status window will be based. For example：
 
 ```
 SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status);
@@ -44,7 +44,7 @@ SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status);
 SELECT COUNT(*), FIRST(ts) FROM temp_tb_1 SESSION(ts, tol_val);
 ```
 
-The primary key, i.e. timestamp, is used to determine which session window the row belongs to. If the time interval between two adjacent rows is within the time range specified by `tol_val`, they belong to the same session window; otherwise they belong to two different time windows. As shown in the figure below, if the limit of time interval for the session window is specified as 12 seconds, then the 6 rows in the figure constitutes 2 time windows, [2019-04-28 14:22:10，2019-04-28 14:22:30] and [2019-04-28 14:23:10，2019-04-28 14:23:30], because the time difference between 2019-04-28 14:22:30 and 2019-04-28 14:23:10 is 40 seconds, which exceeds the time interval limit of 12 seconds.
+The primary key, i.e. timestamp, is used to determine which session window a row belongs to. If the time interval between two adjacent rows is within the time range specified by `tol_val`, they belong to the same session window; otherwise they belong to two different session windows. As shown in the figure below, if the limit of time interval for the session window is specified as 12 seconds, then the 6 rows in the figure constitutes 2 time windows, [2019-04-28 14:22:10，2019-04-28 14:22:30] and [2019-04-28 14:23:10，2019-04-28 14:23:30], because the time difference between 2019-04-28 14:22:30 and 2019-04-28 14:23:10 is 40 seconds, which exceeds the time interval limit of 12 seconds.
 
 ![TDengine Database Session Window](./timewindow-2.webp)
 
@@ -73,7 +73,7 @@ SELECT function_list FROM stb_name
 
 ### Restrictions
 
-- Aggregate functions and select functions can be used in `function_list`, with each function having only one output, for example COUNT, AVG, SUM, STDDEV, LEASTSQUARES, PERCENTILE, MIN, MAX, FIRST, LAST. Functions having multiple output can't be used, for example DIFF or arithmetic operations.
+- Aggregate functions and select functions can be used in `function_list`, with each function having only one output. For example COUNT, AVG, SUM, STDDEV, LEASTSQUARES, PERCENTILE, MIN, MAX, FIRST, LAST. Functions having multiple outputs, such as DIFF or arithmetic operations can't be used.
 - `LAST_ROW` can't be used together with window aggregate.
 - Scalar functions, like CEIL/FLOOR, can't be used with window aggregate.
 - `WHERE` clause can be used to specify the starting and ending time and other filter conditions
@@ -87,8 +87,8 @@ SELECT function_list FROM stb_name
 
 :::info
 
-1. Huge volume of interpolation output may be returned using `FILL`, so it's recommended to specify the time range when using `FILL`. The maximum interpolation values that can be returned in single query is 10,000,000.
+1. A huge volume of interpolation output may be returned using `FILL`, so it's recommended to specify the time range when using `FILL`. The maximum number of interpolation values that can be returned in a single query is 10,000,000.
-2. The result set is in ascending order of timestamp in aggregate by time window aggregate.
+2. The result set is in ascending order of timestamp when you aggregate by time window.
 3. If aggregate by window is used on STable, the aggregate function is performed on all the rows matching the filter conditions. If `GROUP BY` is not used in the query, the result set will be returned in ascending order of timestamp; otherwise the result set is not exactly in the order of ascending timestamp in each group.
 
 :::
@@ -97,13 +97,13 @@ Aggregate by time window is also used in continuous query, please refer to [Cont
 
 ## Examples
 
-The table of intelligent meters can be created by the SQL statement below：
+A table of intelligent meters can be created by the SQL statement below：
 
 ```sql
 CREATE TABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT);
 ```
 
-The average current, maximum current and median of current in every 10 minutes for the past 24 hours can be calculated using the below SQL statement, with missing values filled with the previous non-NULL values.
+The average current, maximum current and median of current in every 10 minutes for the past 24 hours can be calculated using the SQL statement below, with missing values filled with the previous non-NULL values.
 
 ```
 SELECT AVG(current), MAX(current), APERCENTILE(current, 50) FROM meters

docs-en/12-taos-sql/09-limit.md

@@ -4,8 +4,8 @@ title: Limits & Restrictions
 
 ## Naming Rules
 
-1. Only English characters, digits and underscore are allowed
+1. Only characters from the English alphabet, digits and underscore are allowed
-2. Can't start with a digit
+2. Names cannot start with a digit
 3. Case insensitive without escape character "\`"
 4. Identifier with escape character "\`"
    To support more flexible table or column names, a new escape character "\`" is introduced. For more details please refer to [escape](/taos-sql/escape).
@@ -16,38 +16,38 @@ The legal character set is `[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]`.
 
 ## General Limits
 
-- Maximum length of database name is 32 bytes
+- Maximum length of database name is 32 bytes.
-- Maximum length of table name is 192 bytes, excluding the database name prefix and the separator
+- Maximum length of table name is 192 bytes, excluding the database name prefix and the separator.
-- Maximum length of each data row is 48K bytes from version 2.1.7.0 , before which the limit is 16K bytes. Please note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type.
+- Maximum length of each data row is 48K bytes since version 2.1.7.0 , before which the limit was 16K bytes. Please note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type.
-- Maximum of column name is 64.
+- Maximum length of column name is 64.
 - Maximum number of columns is 4096. There must be at least 2 columns, and the first column must be timestamp.
 - Maximum length of tag name is 64.
 - Maximum number of tags is 128. There must be at least 1 tag. The total length of tag values should not exceed 16K bytes.
-- Maximum length of singe SQL statement is 1048576, i.e. 1 MB bytes. It can be configured in the parameter `maxSQLLength` in the client side, the applicable range is [65480, 1048576].
+- Maximum length of singe SQL statement is 1048576, i.e. 1 MB. It can be configured in the parameter `maxSQLLength` in the client side, the applicable range is [65480, 1048576].
-- At most 4096 columns (or 1024 prior to 2.1.7.0) can be returned by `SELECT`, functions in the query statement may constitute columns. Error will be returned if the limit is exceeded.
+- At most 4096 columns (or 1024 prior to 2.1.7.0) can be returned by `SELECT`. Functions in the query statement constitute columns. An error is returned if the limit is exceeded.
-- Maximum numbers of databases, STables, tables are only depending on the system resources.
+- Maximum numbers of databases, STables, tables are dependent only on the system resources.
 - Maximum of database name is 32 bytes, and it can't include "." or special characters.
-- Maximum replica number of database is 3
+- Maximum number of replicas for a database is 3.
-- Maximum length of user name is 23 bytes
+- Maximum length of user name is 23 bytes.
-- Maximum length of password is 15 bytes
+- Maximum length of password is 15 bytes.
-- Maximum number of rows depends on the storage space only.
+- Maximum number of rows depends only on the storage space.
-- Maximum number of tables depends on the number of nodes only.
+- Maximum number of tables depends only on the number of nodes.
-- Maximum number of databases depends on the number of nodes only.
+- Maximum number of databases depends only on the number of nodes.
-- Maximum number of vnodes for single database is 64.
+- Maximum number of vnodes for a single database is 64.
 
 ## Restrictions of `GROUP BY`
 
-`GROUP BY` can be performed on tags and `TBNAME`. It can be performed on data columns too, with one restriction that only one column and the number of unique values on that column is lower than 100,000. Please note that `GROUP BY` can't be performed on float or double types.
+`GROUP BY` can be performed on tags and `TBNAME`. It can be performed on data columns too, with the only restriction being it can only be performed on one data column and the number of unique values in that column is lower than 100,000. Please note that `GROUP BY` cannot be performed on float or double types.
 
 ## Restrictions of `IS NOT NULL`
 
-`IS NOT NULL` can be used on any data type of columns. The non-empty string evaluation expression, i.e. `<\>""` can only be used on non-numeric data types.
+`IS NOT NULL` can be used on any data type of columns. The non-empty string evaluation expression, i.e. `< > ""` can only be used on non-numeric data types.
 
 ## Restrictions of `ORDER BY`
 
 - Only one `order by` is allowed for normal table and subtable.
 - At most two `order by` are allowed for STable, and the second one must be `ts`.
-- `order by tag` must be used with `group by tag` on same tag, this rule is also applicable to `tbname`.
+- `order by tag` must be used with `group by tag` on same tag. This rule is also applicable to `tbname`.
 - `order by column` must be used with `group by column` or `top/bottom` on same column. This rule is applicable to table and STable.
 - `order by ts` is applicable to table and STable.
 - If `order by ts` is used with `group by`, the result set is sorted using `ts` in each group.
@@ -56,7 +56,7 @@ The legal character set is `[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]`.
 
 ### Name Restrictions of Table/Column
 
-The name of a table or column can only be composed of ASCII characters, digits and underscore, while it can't start with a digit. The maximum length is 192 bytes. Names are case insensitive. The name mentioned in this rule doesn't include the database name prefix and the separator.
+The name of a table or column can only be composed of ASCII characters, digits and underscore and it cannot start with a digit. The maximum length is 192 bytes. Names are case insensitive. The name mentioned in this rule doesn't include the database name prefix and the separator.
 
 ### Name Restrictions After Escaping
 

docs-en/12-taos-sql/10-json.md

@@ -4,7 +4,7 @@ title: JSON Type
 
 ## Syntax
 
-1. Tag of JSON type
+1. Tag of type JSON
 
    ```sql
    create STable s1 (ts timestamp, v1 int) tags (info json);
@@ -12,7 +12,7 @@ title: JSON Type
    create table s1_1 using s1 tags ('{"k1": "v1"}');
    ```
 
-2. -> Operator of JSON
+2. "->" Operator of JSON
 
    ```sql
    select * from s1 where info->'k1' = 'v1';
@@ -20,7 +20,7 @@ title: JSON Type
    select info->'k1' from s1;
    ```
 
-3. contains Operator of JSON
+3. "contains" Operator of JSON
 
    ```sql
    select * from s1 where info contains 'k2';
@@ -30,7 +30,7 @@ title: JSON Type
 
 ## Applicable Operations
 
-1. When JSON data type is used in `where`, `match/nmatch/between and/like/and/or/is null/is no null` can be used but `in` can't be used.
+1. When a JSON data type is used in `where`, `match/nmatch/between and/like/and/or/is null/is no null` can be used but `in` can't be used.
 
    ```sql
    select * from s1 where info->'k1' match 'v*';
@@ -42,9 +42,9 @@ title: JSON Type
    select * from s1 where info->'k1' is not null;
    ```
 
-2. Tag of JSON type can be used in `group by`, `order by`, `join`, `union all` and sub query, for example `group by json->'key'`
+2. A tag of JSON type can be used in `group by`, `order by`, `join`, `union all` and sub query; for example `group by json->'key'`
 
-3. `Distinct` can be used with tag of JSON type
+3. `Distinct` can be used with a tag of type JSON
 
    ```sql
    select distinct info->'k1' from s1;
@@ -52,9 +52,9 @@ title: JSON Type
 
 4. Tag Operations
 
-   The value of JSON tag can be altered. Please note that the full JSON will be overriden when doing this.
+   The value of a JSON tag can be altered. Please note that the full JSON will be overriden when doing this.
 
-   The name of JSON tag can be altered. A tag of JSON type can't be added or removed. The column length of a JSON tag can't be changed.
+   The name of a JSON tag can be altered. A tag of JSON type can't be added or removed. The column length of a JSON tag can't be changed.
 
 ## Other Restrictions
 
@@ -64,17 +64,17 @@ title: JSON Type
 
 - JSON format：
 
-  - The input string for JSON can be empty, i.e. "", "\t", or NULL, but can't be non-NULL string, bool or array.
+  - The input string for JSON can be empty, i.e. "", "\t", or NULL, but it can't be non-NULL string, bool or array.
-  - object can be {}, and the whole JSON is empty if so. Key can be "", and it's ignored if so.
+  - object can be {}, and the entire JSON is empty if so. Key can be "", and it's ignored if so.
-  - value can be int, double, string, boll or NULL, can't be array. Nesting is not allowed, that means value can't be another JSON.
+  - value can be int, double, string, bool or NULL, and it can't be an array. Nesting is not allowed which means that the value of a key can't be JSON.
   - If one key occurs twice in JSON, only the first one is valid.
   - Escape characters are not allowed in JSON.
 
-- NULL is returned if querying a key that doesn't exist in JSON.
+- NULL is returned when querying a key that doesn't exist in JSON.
 
 - If a tag of JSON is the result of inner query, it can't be parsed and queried in the outer query.
 
-For example, the below SQL statements are not supported.
+For example, the SQL statements below are not supported.
 
 ```sql;
 select jtag->'key' from (select jtag from STable);

docs-en/13-operation/01-pkg-install.md

@@ -6,7 +6,7 @@ description: Install, Uninstall, Start, Stop and Upgrade
 import Tabs from "@theme/Tabs";
 import TabItem from "@theme/TabItem";
 
-TDengine community version provides dev and rpm packages for users to choose based on the system environment. deb supports Debian, Ubuntu and systems derived from them. rpm supports CentOS, RHEL, SUSE and systems derived from them. Furthermore, tar.gz package is provided for enterprise customers.
+TDengine community version provides deb and rpm packages for users to choose from, based on their system environment. The deb package supports Debian, Ubuntu and derivative systems. The rpm package supports CentOS, RHEL, SUSE and derivative systems. Furthermore, a tar.gz package is provided for TDengine Enterprise customers.
 
 ## Install
 
@@ -124,7 +124,7 @@ taoskeeper is installed, enable it by `systemctl enable taoskeeper`
 ```
 
 :::info
-Some configuration will be prompted for users to provide when install.sh is executing, the interactive mode can be disabled by executing `./install.sh -e no`. `./install -h` can show all parameters and detailed explanation.
+Users will be prompted to enter some configuration information when install.sh is executing. The interactive mode can be disabled by executing `./install.sh -e no`. `./install.sh -h` can show all parameters with detailed explanation.
 
 :::
 
@@ -132,7 +132,7 @@ Some configuration will be prompted for users to provide when install.sh is exec
 </Tabs>
 
 :::note
-When installing on the first node in the cluster, when "Enter FQDN:" is prompted, nothing needs to be provided. When installing on following nodes, when "Enter FQDN:" is prompted, the end point of the first dnode in the cluster can be input if it is already up; or just ignore it and configure later after installation is done.
+When installing on the first node in the cluster, at the "Enter FQDN:" prompt, nothing needs to be provided. When installing on subsequent nodes, at the "Enter FQDN:" prompt, you must enter the end point of the first dnode in the cluster if it is already up. You can also just ignore it and configure it later after installation is finished.
 
 :::
 
@@ -181,14 +181,14 @@ taosKeeper is removed successfully!
 
 :::note
 
-- It's strongly suggested not to use multiple kinds of installation packages on a single host TDengine
+- We strongly recommend not to use multiple kinds of installation packages on a single host TDengine. 
-- After deb package is installed, if the installation directory is removed manually so that uninstall or reinstall can't succeed, it can be resolved by cleaning up TDengine package information as in the command below and then reinstalling.
+- After deb package is installed, if the installation directory is removed manually, uninstall or reinstall will not work. This issue can be resolved by using the command below which cleans up TDengine package information. You can then reinstall if needed.
 
 ```bash
    $ sudo rm -f /var/lib/dpkg/info/tdengine*
 ```
 
-- After rpm package is installed, if the installation directory is removed manually so that uninstall or reinstall can't succeed, it can be resolved by cleaning up TDengine package information as in the command below and then reinstalling.
+- After rpm package is installed, if the installation directory is removed manually, uninstall or reinstall will not work. This issue can be resolved by using the command below which cleans up TDengine package information. You can then reinstall if needed.
 
 ```bash
    $ sudo rpm -e --noscripts tdengine
@@ -219,7 +219,7 @@ lrwxrwxrwx  1 root root   13 Feb 22 09:34 log -> /var/log/taos/
 During the installation process:
 
 - Configuration directory, data directory, and log directory are created automatically if they don't exist
-- The default configuration file is located at /etc/taos/taos.cfg, which is a copy of /usr/local/taos/cfg/taos.cfg if not existing
+- The default configuration file is located at /etc/taos/taos.cfg, which is a copy of /usr/local/taos/cfg/taos.cfg
 - The default data directory is /var/lib/taos, which is a soft link to /usr/local/taos/data
 - The default log directory is /var/log/taos, which is a soft link to /usr/local/taos/log
 - The executables at /usr/local/taos/bin are linked to /usr/bin
@@ -228,7 +228,7 @@ During the installation process:
 
 :::note
 
-- When TDengine is uninstalled, the configuration /etc/taos/taos.cfg, data directory /var/lib/taos, log directory /var/log/taos are kept. They can be deleted manually with caution because data can't be recovered
+- When TDengine is uninstalled, the configuration /etc/taos/taos.cfg, data directory /var/lib/taos, log directory /var/log/taos are kept. They can be deleted manually with caution, because data can't be recovered. Please follow data integrity, security, backup or relevant SOPs before deleting any data.
 - When reinstalling TDengine, if the default configuration file /etc/taos/taos.cfg exists, it will be kept and the configuration file in the installation package will be renamed to taos.cfg.orig and stored at /usr/local/taos/cfg to be used as configuration sample. Otherwise the configuration file in the installation package will be installed to /etc/taos/taos.cfg and used.
 
 ## Start and Stop
@@ -263,18 +263,19 @@ Active: inactive (dead)
 
 There are two aspects in upgrade operation: upgrade installation package and upgrade a running server.
 
-Upgrading package should follow the steps mentioned previously to first uninstall the old version then install the new version.
+To upgrade a package, follow the steps mentioned previously to first uninstall the old version then install the new version.
 
-Upgrading a running server is much more complex. First please check the version number of the old version and the new version. The version number of TDengine consists of 4 sections, only if the first 3 section match can the old version be upgraded to the new version. The steps of upgrading a running server are as below:
+Upgrading a running server is much more complex. First please check the version number of the old version and the new version. The version number of TDengine consists of 4 sections, only if the first 3 sections match can the old version be upgraded to the new version. The steps of upgrading a running server are as below:
 
 - Stop inserting data
-- Make sure all data are persisted into disk
+- Make sure all data is persisted to disk
+- Make some simple queries (Such as total rows in stables, tables and so on. Note down the values. Follow best practices and relevant SOPs.)
 - Stop the cluster of TDengine
 - Uninstall old version and install new version
 - Start the cluster of TDengine
-- Make some simple queries to make sure no data loss
+- Execute simple queries, such as the ones executed prior to installing the new package, to make sure there is no data loss
-- Make some simple data insertion to make sure the cluster works well
+- Run some simple data insertion statements to make sure the cluster works well
-- Restore business data
+- Restore business services
 
 :::warning
 

docs-en/13-operation/02-planning.mdx

@@ -2,17 +2,17 @@
 title: Resource Planning
 ---
 
-The computing and storage resources need to be planned if using TDengine to build an IoT platform. How to plan the CPU, memory and disk required will be described in this chapter.
+It is important to plan computing and storage resources if using TDengine to build an IoT, time-series or Big Data platform. How to plan the CPU, memory and disk resources required, will be described in this chapter.
 
 ## Memory Requirement of Server Side
 
-The number of vgroups created for each database is the same as the number of CPU cores by default and can be configured by parameter `maxVgroupsPerDb`, each vnode in a vgroup stores one replica. Each vnode consumes a fixed size of memory, i.e. `blocks` \* `cache`. Besides, some memory is required for tag values associated with each table. A fixed amount of memory is required for each cluster. So, the memory required for each DB can be calculated using the formula below:
+By default, the number of vgroups created for each database is the same as the number of CPU cores. This can be configured by the parameter `maxVgroupsPerDb`. Each vnode in a vgroup stores one replica. Each vnode consumes a fixed amount of memory, i.e. `blocks` \* `cache`. In addition, some memory is required for tag values associated with each table. A fixed amount of memory is required for each cluster. So, the memory required for each DB can be calculated using the formula below:
 
 ```
 Database Memory Size = maxVgroupsPerDb * replica * (blocks * cache + 10MB) + numOfTables * (tagSizePerTable + 0.5KB)
 ```
 
-For example, assuming the default value of `maxVgroupPerDB` is 64, the default value of `cache` 16M, the default value of `blocks` is 6, there are 100,000 tables in a DB, the replica number is 1, total length of tag values is 256 bytes, the total memory required for this DB is: 64 \* 1 \* (16 \* 6 + 10) + 100000 \* (0.25 + 0.5) / 1000 = 6792M.
+For example, assuming the default value of `maxVgroupPerDB` is 64, the default value of `cache` is 16M, the default value of `blocks` is 6, there are 100,000 tables in a DB, the replica number is 1, total length of tag values is 256 bytes, the total memory required for this DB is: 64 \* 1 \* (16 \* 6 + 10) + 100000 \* (0.25 + 0.5) / 1000 = 6792M.
 
 In the real operation of TDengine, we are more concerned about the memory used by each TDengine server process `taosd`.
 
@@ -22,10 +22,10 @@ In the real operation of TDengine, we are more concerned about the memory used b
 
 In the above formula:
 
-1. "vnode_memory" of a `taosd` process is the memory used by all vnodes hosted by this `taosd` process. It can be roughly calculated by firstly adding up the total memory of all DBs whose memory usage can be derived according to the formula mentioned previously then dividing by number of dnodes and multiplying the number of replicas.
+1. "vnode_memory" of a `taosd` process is the memory used by all vnodes hosted by this `taosd` process. It can be roughly calculated by firstly adding up the total memory of all DBs whose memory usage can be derived according to the formula for Database Memory Size, mentioned above, then dividing by number of dnodes and multiplying the number of replicas.
 
 ```
-    vnode_memory = sum(Database memory) / number_of_dnodes * replica
+    vnode_memory = (sum(Database Memory Size) / number_of_dnodes) * replica
 ```
 
 2. "mnode_memory" of a `taosd` process is the memory consumed by a mnode. If there is one (and only one) mnode hosted in a `taosd` process, the memory consumed by "mnode" is "0.2KB \* the total number of tables in the cluster".
@@ -56,8 +56,8 @@ So, at least 3GB needs to be reserved for such a client.
 
 The CPU resources required depend on two aspects:
 
-- **Data Insertion** Each dnode of TDengine can process at least 10,000 insertion requests in one second, while each insertion request can have multiple rows. The computing resource consumed between inserting 1 row one time and inserting 10 rows one time is very small. So, the more the rows to insert one time, the higher the efficiency. Inserting in bach also exposes requirements for the client side which needs to cache rows and insert in batch once the cached rows reaches a threshold.
+- **Data Insertion** Each dnode of TDengine can process at least 10,000 insertion requests in one second, while each insertion request can have multiple rows. The difference in computing resource consumed, between inserting 1 row at a time, and inserting 10 rows at a time is very small. So, the more the number of rows that can be inserted one time, the higher the efficiency. Inserting in batch also imposes requirements on the client side which needs to cache rows to insert in batch once the number of cached rows reaches a threshold.
-- **Data Query** High efficiency query is provided in TDengine, but it's hard to estimate the CPU resource required because the queries used in different use cases and the frequency of queries vary significantly. It can only be verified with the query statements, query frequency, data size to be queried, etc provided by user.
+- **Data Query** High efficiency query is provided in TDengine, but it's hard to estimate the CPU resource required because the queries used in different use cases and the frequency of queries vary significantly. It can only be verified with the query statements, query frequency, data size to be queried, and other requirements provided by users.
 
 In short, the CPU resource required for data insertion can be estimated but it's hard to do so for query use cases. In real operation, it's suggested to control CPU usage below 50%. If this threshold is exceeded, it's a reminder for system operator to add more nodes in the cluster to expand resources.
 
@@ -71,12 +71,12 @@ Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable
 
 For example, there are 10,000,000 meters, while each meter collects data every 15 minutes and the data size of each collection is 128 bytes, so the raw data size of one year is: 10000000 \* 128 \* 24 \* 60 / 15 \* 365 = 44.8512(TB). Assuming compression ratio is 5, the actual disk size is: 44.851 / 5 = 8.97024(TB).
 
-Parameter `keep` can be used to set how long the data will be kept on disk. To further reduce storage cost, multiple storage levels can be enabled in TDengine, with the coldest data stored on the cheapest storage device, and this is transparent to application programs.
+Parameter `keep` can be used to set how long the data will be kept on disk. To further reduce storage cost, multiple storage levels can be enabled in TDengine, with the coldest data stored on the cheapest storage device. This is completely transparent to application programs.
 
-To increase the performance, multiple disks can be setup for parallel data reading or data inserting. Please note that an expensive disk array is not necessary because replications are used in TDengine to provide high availability.
+To increase performance, multiple disks can be setup for parallel data reading or data inserting. Please note that an expensive disk array is not necessary because replications are used in TDengine to provide high availability.
 
 ## Number of Hosts
 
-A host can be either physical or virtual. The total memory, total CPU, total disk required can be estimated according to the formulas mentioned previously. Then, according to the system resources that a single host can provide, assuming all hosts have the same resources, the number of hosts can be derived easily.
+A host can be either physical or virtual. The total memory, total CPU, total disk required can be estimated according to the formulae mentioned previously. Then, according to the system resources that a single host can provide, assuming all hosts have the same resources, the number of hosts can be derived easily.
 
 **Quick Estimation for CPU, Memory and Disk** Please refer to [Resource Estimate](https://www.taosdata.com/config/config.html).

docs-en/13-operation/03-tolerance.md

@@ -7,26 +7,26 @@ title: Fault Tolerance & Disaster Recovery
 
 TDengine uses **WAL**, i.e. Write Ahead Log, to achieve fault tolerance and high reliability.
 
-When a data block is received by TDengine, the original data block is first written into WAL. The log in WAL will be deleted only after the data has been written into data files in the database. Data can be recovered from WAL in case the server is stopped abnormally due to any reason and then restarted.
+When a data block is received by TDengine, the original data block is first written into WAL. The log in WAL will be deleted only after the data has been written into data files in the database. Data can be recovered from WAL in case the server is stopped abnormally for any reason and then restarted.
 
 There are 2 configuration parameters related to WAL:
 
 - walLevel：
-  - 0：wal is disabled; 
+  - 0：wal is disabled 
-  - 1：wal is enabled without fsync; 
+  - 1：wal is enabled without fsync 
-  - 2：wal is enabled with fsync.
+  - 2：wal is enabled with fsync
-- fsync：only valid when walLevel is set to 2, it specifies the interval of invoking fsync. If set to 0, it means fsync is invoked immediately once WAL is written.
+- fsync：This parameter is only valid when walLevel is set to 2. It specifies the interval, in milliseconds, of invoking fsync. If set to 0, it means fsync is invoked immediately once WAL is written.
 
-To achieve absolutely no data loss, walLevel needs to be set to 2 and fsync needs to be set to 1. The penalty is the performance of data ingestion downgrades. However, if the concurrent threads of data insertion on the client side can reach a big enough number, for example 50, the data ingestion performance would be still good enough, our verification shows that the drop is only 30% compared to fsync is set to 3,000 milliseconds.
+To achieve absolutely no data loss, walLevel should be set to 2 and fsync should be set to 1. There is a performance penalty to the data ingestion rate. However, if the concurrent data insertion threads on the client side can reach a big enough number, for example 50, the data ingestion performance will be still good enough. Our verification shows that the drop is only 30% when fsync is set to 3,000 milliseconds.
 
 ## Disaster Recovery
 
-TDengine uses replications to provide high availability and disaster recovery capability.
+TDengine uses replication to provide high availability and disaster recovery capability.
 
-TDengine cluster is managed by mnode. To make sure the high availability of mnode, multiple replicas can be configured by the system parameter `numOfMnodes`. The data replication between mnode replicas is performed in a synchronous way to guarantee the metadata consistency.
+A TDengine cluster is managed by mnode. To ensure the high availability of mnode, multiple replicas can be configured by the system parameter `numOfMnodes`. The data replication between mnode replicas is performed in a synchronous way to guarantee metadata consistency.
 
-The number of replicas for the time series data in TDengine is associated with each database, there can be a lot of databases in a cluster while each database can be configured with a different number of replicas. When creating a database, parameter `replica` is used to configure the number of replications. To achieve high availability, `replica` needs to be higher than 1.
+The number of replicas for time series data in TDengine is associated with each database. There can be many databases in a cluster and each database can be configured with a different number of replicas. When creating a database, parameter `replica` is used to configure the number of replications. To achieve high availability, `replica` needs to be higher than 1.
 
 The number of dnodes in a TDengine cluster must NOT be lower than the number of replicas for any database, otherwise it would fail when trying to create a table.
 
-As long as the dnodes of a TDengine cluster are deployed on different physical machines and the replica number is set to bigger than 1, high availability can be achieved without any other assistance. If dnodes of TDengine cluster are deployed in geographically different data centers, disaster recovery can be achieved too.
+As long as the dnodes of a TDengine cluster are deployed on different physical machines and the replica number is higher than 1, high availability can be achieved without any other assistance. For disaster recovery, dnodes of a TDengine cluster should be deployed in geographically different data centers.

docs-en/13-operation/08-export.md

@@ -2,11 +2,13 @@
 title: Data Export
 ---
 
-There are two ways of exporting data from a TDengine cluster, one is SQL statement in TDengine CLI, the other one is `taosdump`.
+There are two ways of exporting data from a TDengine cluster:
+- Using a SQL statement in TDengine CLI
+- Using the `taosdump` tool
 
 ## Export Using SQL
 
-If you want to export the data of a table or a STable, please execute below SQL statement in TDengine CLI.
+If you want to export the data of a table or a STable, please execute the SQL statement below, in the TDengine CLI.
 
 ```sql
 select * from <tb_name> >> data.csv;
@@ -16,4 +18,4 @@ The data of table or STable specified by `tb_name` will be exported into a file
 
 ## Export Using taosdump
 
-With `taosdump`, you can choose to export the data of all databases, a database, a table or a STable, you can also choose export the data within a time range, or even only export the schema definition of a table. For the details of using `taosdump` please refer to [Tool for exporting and importing data: taosdump](/reference/taosdump).
+With `taosdump`, you can choose to export the data of all databases, a database, a table or a STable, you can also choose to export the data within a time range, or even only export the schema definition of a table. For the details of using `taosdump` please refer to [Tool for exporting and importing data: taosdump](/reference/taosdump).

docs-en/13-operation/09-status.md

@@ -3,7 +3,7 @@ sidebar_label: Connections & Tasks
 title: Manage Connections and Query Tasks
 ---
 
-A system operator can use TDengine CLI to show the connections, ongoing queries, stream computing, and can close connection or stop ongoing query task or stream computing.
+A system operator can use the TDengine CLI to show connections, ongoing queries, stream computing, and can close connections or stop ongoing query tasks or stream computing.
 
 ## Show Connections
 
@@ -13,7 +13,7 @@ SHOW CONNECTIONS;
 
 One column of the output of the above SQL command is "ip:port", which is the end point of the client.
 
-## Close Connections Forcedly
+## Force Close Connections
 
 ```sql
 KILL CONNECTION <connection-id>;
@@ -27,9 +27,9 @@ In the above SQL command, `connection-id` is from the first column of the output
 SHOW QUERIES;
 ```
 
-The first column of the output is query ID, which is composed of the corresponding connection ID and the sequence number of the current query task started on this connection, in format of "connection-id:query-no".
+The first column of the output is query ID, which is composed of the corresponding connection ID and the sequence number of the current query task started on this connection. The format is "connection-id:query-no".
 
-## Close Queries Forcedly
+## Force Close Queries
 
 ```sql
 KILL QUERY <query-id>;
@@ -43,9 +43,9 @@ In the above SQL command, `query-id` is from the first column of the output of `
 SHOW STREAMS;
 ```
 
-The first column of the output is stream ID, which is composed of the connection ID and the sequence number of the current stream started on this connection, in the format of "connection-id:stream-no".
+The first column of the output is stream ID, which is composed of the connection ID and the sequence number of the current stream started on this connection. The format is "connection-id:stream-no".
 
-## Close Continuous Query Forcedly
+## Force Close Continuous Query
 
 ```sql
 KILL STREAM <stream-id>;

docs-en/13-operation/10-monitor.md

@@ -2,13 +2,13 @@
 title: TDengine Monitoring
 ---
 
-After TDengine is started, a database named `log` for monitoring is created automatically. The information about CPU, memory, disk, bandwidth, number of requests, disk I/O speed, slow query is written into `log` database on the basis of a predefined interval. Additionally, some important system operations, like logon, create user, drop database, and alerts and warnings generated in TDengine are written into the `log` database too. A system operator can view the data in `log` database from TDengine CLI or from a web console.
+After TDengine is started, a database named `log` is created automatically to help with monitoring. Information that includes CPU, memory and disk usage, bandwidth, number of requests, disk I/O speed, slow queries, is written into the `log` database at a predefined interval. Additionally, some important system operations, like logon, create user, drop database, and alerts and warnings generated in TDengine are written into the `log` database too. A system operator can view the data in `log` database from TDengine CLI or from a web console.
 
 The collection of the monitoring information is enabled by default, but can be disabled by parameter `monitor` in the configuration file. 
 
 ## TDinsight 
 
-TDinsight is a complete solution which uses the monitor database `log` mentioned previously and Grafana to monitor a TDengine cluster.
+TDinsight is a complete solution which uses the monitoring database `log` mentioned previously, and Grafana, to monitor a TDengine cluster.
 
 From version 2.3.3.0, more monitoring data has been added in the `log` database. Please refer to [TDinsight Grafana Dashboard](https://grafana.com/grafana/dashboards/15167) to learn more details about using TDinsight to monitor TDengine.
 

docs-en/13-operation/17-diagnose.md

@@ -4,13 +4,13 @@ title: Problem Diagnostics
 
 ## Network Connection Diagnostics
 
-When the client is unable to access the server, the network connection between the client side and the server side needs to be checked to find out the root cause and resolve problems.
+When a TDengine client is unable to access a TDengine server, the network connection between the client side and the server side must be checked to find the root cause and resolve problems.
 
-The diagnostic for network connection can be executed between Linux and Linux or between Linux and Windows.
+Diagnostics for network connections can be executed between Linux and Linux or between Linux and Windows.
 
 Diagnostic steps：
 
-1. If the port range to be diagnosed are being occupied by a `taosd` server process, please first stop `taosd.
+1. If the port range to be diagnosed is being occupied by a `taosd` server process, please first stop `taosd.
 2. On the server side, execute command `taos -n server -P <port> -l <pktlen>` to monitor the port range starting from the port specified by `-P` parameter with the role of "server".
 3. On the client side, execute command `taos -n client -h <fqdn of server> -P <port> -l <pktlen>` to send a testing package to the specified server and port.
  
@@ -65,13 +65,13 @@ Output of the client side for the example is below:
 12/21 14:50:22.721274 0x7fc95d859200 UTL successed to test UDP port:6011
 ```
 
-The output needs to be checked carefully for the system operator to find out the root cause and solve the problem.
+The output needs to be checked carefully for the system operator to find the root cause and resolve the problem.
 
 ## Startup Status and RPC Diagnostic
 
-`taos -n startup -h <fqdn of server>` can be used to check the startup status of a `taosd` process. This is a comman task for a system operator to do to determine whether `taosd` has been started successfully, especially in case of cluster.
+`taos -n startup -h <fqdn of server>` can be used to check the startup status of a `taosd` process. This is a common task which should be performed by a system operator, especially in the case of a cluster, to determine whether `taosd` has been started successfully.
 
-`taos -n rpc -h <fqdn of server>` can be used to check whether the port of a started `taosd` can be accessed or not. If `taosd` process doesn't respond or is working abnormally, this command can be used to initiate a rpc communication with the specified fqdn to determine whether it's a network problem or `taosd` is abnormal.
+`taos -n rpc -h <fqdn of server>` can be used to check whether the port of a started `taosd` can be accessed or not. If `taosd` process doesn't respond or is working abnormally, this command can be used to initiate a rpc communication with the specified fqdn to determine whether it's a network problem or whether `taosd` is abnormal.
 
 ## Sync and Arbitrator Diagnostic
 
@@ -80,13 +80,13 @@ taos -n sync -P 6040 -h <fqdn of server>
 taos -n sync -P 6042 -h <fqdn of server>
 ```
 
-The above commands can be executed on Linux Shell to check whether the port for sync is working well and whether the sync module on the server side is working well. Additionally, `-P 6042` is used to check whether the arbitrator is configured properly and is working well.
+The above commands can be executed in a Linux shell to check whether the port for sync is working well and whether the sync module on the server side is working well. Additionally, `-P 6042` is used to check whether the arbitrator is configured properly and is working well.
 
 ## Network Speed Diagnostic
 
 `taos -n speed -h <fqdn of server> -P 6030 -N 10 -l 10000000 -S TCP`
 
-From version 2.2.0.0, the above command can be executed on Linux Shell to test the network speed, it sends uncompressed package to a running `taosd` server process or a simulated server process started by `taos -n server` to test the network speed. Parameters can be used when testing network speed are as below:
+From version 2.2.0.0 onwards, the above command can be executed in a Linux shell to test network speed. The command sends uncompressed packages to a running `taosd` server process or a simulated server process started by `taos -n server` to test the network speed. Parameters can be used when testing network speed are as below:
 
 -n：When set to "speed", it means testing network speed. 
 -h：The FQDN or IP of the server process to be connected to; if not set, the FQDN configured in `taos.cfg` is used. 
@@ -99,23 +99,23 @@ From version 2.2.0.0, the above command can be executed on Linux Shell to test t
 
 `taos -n fqdn -h <fqdn of server>`
 
-From version 2.2.0.0, the above command can be executed on Linux Shell to test the resolution speed of FQDN. It can be used to try to resolve a FQDN to an IP address and record the time spent in this process. The parameters that can be used for this purpose are as below:
+From version 2.2.0.0 onward, the above command can be executed in a Linux shell to test the resolution speed of FQDN. It can be used to try to resolve a FQDN to an IP address and record the time spent in this process. The parameters that can be used for this purpose are as below:
 
 -n：When set to "fqdn", it means testing the speed of resolving FQDN. 
 -h：The FQDN to be resolved. If not set, the `FQDN` parameter in `taos.cfg` is used by default.  
 
 ## Server Log
 
-The parameter `debugFlag` is used to control the log level of the `taosd` server process. The default value is 131, for debug purpose it needs to be escalated to 135 or 143. 
+The parameter `debugFlag` is used to control the log level of the `taosd` server process. The default value is 131. For debugging and tracing, it needs to be set to either 135 or 143 respectively. 
 
-Once this parameter is set to 135 or 143, the log file grows very quickly especially when there is a huge volume of data insertion and data query requests. If all the logs are stored together, some important information may be missed very easily, so on server side important information is stored at different place from other logs.
+Once this parameter is set to 135 or 143, the log file grows very quickly especially when there is a huge volume of data insertion and data query requests. If all the logs are stored together, some important information may be missed very easily and so on the server side, important information is stored in a different place from other logs.
 
 - The log at level of INFO, WARNING and ERROR is stored in `taosinfo` so that it is easy to find important information 
 - The log at level of DEBUG (135) and TRACE (143) and other information not handled by `taosinfo` are stored in `taosdlog`
 
 ## Client Log
 
-An independent log file, named as "taoslog+<seq num\>" is generated for each client program, i.e. a client process. The default value of `debugFlag` is also 131 and only logs at level of INFO/ERROR/WARNING are recorded, for debugging purposes it needs to be changed to 135 or 143 so that logs at DEBUG or TRACE level can be recorded.
+An independent log file, named as "taoslog+<seq num\>" is generated for each client program, i.e. a client process. The default value of `debugFlag` is also 131 and only logs at level of INFO/ERROR/WARNING are recorded. As stated above, for debugging and tracing, it needs to be changed to 135 or 143 respectively, so that logs at DEBUG or TRACE level can be recorded.
 
 The maximum length of a single log file is controlled by parameter `numOfLogLines` and only 2 log files are kept for each `taosd` server process.
 

docs-en/13-operation/index.md

@@ -2,7 +2,7 @@
 title: Administration
 ---
 
-This chapter is mainly written for system administrators, covering download, install/uninstall, data import/export, system monitoring, user management, connection management, etc. Capacity planning and system optimization are also covered.
+This chapter is mainly written for system administrators. It covers download, install/uninstall, data import/export, system monitoring, user management, connection management, capacity planning and system optimization.
 
 ```mdx-code-block
 import DocCardList from '@theme/DocCardList';

docs-en/14-reference/02-rest-api/02-rest-api.mdx

@@ -2,10 +2,10 @@
 title: REST API
 ---
 
-To support the development of various types of platforms, TDengine provides an API that conforms to the REST principle, namely REST API. To minimize the learning cost, different from the other database REST APIs, TDengine directly requests the SQL command contained in the request BODY through HTTP POST to operate the database and only requires a URL.
+To support the development of various types of applications and platforms, TDengine provides an API that conforms to REST principles; namely REST API. To minimize the learning cost, unlike REST APIs for other database engines, TDengine allows insertion of SQL commands in the BODY of an HTTP POST request, to operate the database.
 
 :::note 
-One difference from the native connector is that the REST interface is stateless, so the `USE db_name` command has no effect. All references to table names and super table names need to specify the database name prefix. (Since version 2.2.0.0, it is supported to specify db_name in RESTful URL. If the database name prefix is not specified in the SQL command, the `db_name` specified in the URL will be used. Since version 2.4.0.0, REST service is provided by taosAdapter by default. And it requires that the `db_name` must be specified in the URL.)
+One difference from the native connector is that the REST interface is stateless and so the `USE db_name` command has no effect. All references to table names and super table names need to specify the database name in the prefix. (Since version 2.2.0.0, TDengine supports specification of the db_name in RESTful URL. If the database name prefix is not specified in the SQL command, the `db_name` specified in the URL will be used. Since version 2.4.0.0, REST service is provided by taosAdapter by default and it requires that the `db_name` must be specified in the URL.)
 :::
 
 ## Installation
@@ -16,9 +16,9 @@ The REST interface does not rely on any TDengine native library, so the client a
 
 If the TDengine server is already installed, it can be verified as follows:
 
-The following is an Ubuntu environment using the `curl` tool (to confirm that it is installed) to verify that the REST interface is working.
+The following example is in an Ubuntu environment and uses the `curl` tool to verify that the REST interface is working. Note that the `curl` tool may need to be installed in your environment.
 
-The following example lists all databases, replacing `h1.taosdata.com` and `6041` (the default port) with the actual running TDengine service FQDN and port number.
+The following example lists all databases on the host h1.taosdata.com. To use it in your environment, replace `h1.taosdata.com` and `6041` (the default port) with the actual running TDengine service FQDN and port number.
 
 ```html
 curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' h1.taosdata.com:6041/rest/sql
@@ -89,7 +89,7 @@ For example, `http://h1.taos.com:6041/rest/sql/test` is a URL to `h1.taos.com:60
 
 TDengine supports both Basic authentication and custom authentication mechanisms, and subsequent versions will provide a standard secure digital signature mechanism for authentication.
 
-- The custom authentication information is as follows (Let's introduce token later)
+- The custom authentication information is as follows. More details about "token" later.
 
   ```
   Authorization: Taosd <TOKEN>
@@ -136,7 +136,7 @@ The return result is in JSON format, as follows:
 
 Description:
 
-- status: tell if the operation result is success or failure.
+- status: tells you whethre the operation result is success or failure.
 - head: the definition of the table, or just one column "affected_rows" if no result set is returned. (As of version 2.0.17.0, it is recommended not to rely on the head return value to determine the data column type but rather use column_meta. In later versions, the head item may be removed from the return value.)
 - column_meta: this item is added to the return value to indicate the data type of each column in the data with version 2.0.17.0 and later versions. Each column is described by three values: column name, column type, and type length. For example, `["current",6,4]` means that the column name is "current", the column type is 6, which is the float type, and the type length is 4, which is the float type with 4 bytes. If the column type is binary or nchar, the type length indicates the maximum length of content stored in the column, not the length of the specific data in this return value. When the column type is nchar, the type length indicates the number of Unicode characters that can be saved, not bytes.
 - data: The exact data returned, presented row by row, or just [[affected_rows]] if no result set is returned. The order of the data columns in each row of data is the same as that of the data columns described in column_meta.

docs-en/14-reference/08-taos-shell.md

@@ -1,10 +1,10 @@
 ---
-title: TDengine Command Line (CLI)
+title: TDengine Command Line Interface (CLI)
-sidebar_label: TDengine CLI
+sidebar_label: Command Line Interface
 description: Instructions and tips for using the TDengine CLI
 ---
 
-The TDengine command-line application (hereafter referred to as `TDengine CLI`) is the simplest way for users to manipulate and interact with TDengine instances.
+The TDengine command-line interface (hereafter referred to as `TDengine CLI`) is the simplest way for users to manipulate and interact with TDengine instances.
 
 ## Installation
 

docs-en/14-reference/index.md

@@ -2,7 +2,7 @@
 title: Reference
 ---
 
-The reference guide is the detailed introduction to TDengine, various TDengine's connectors in different languages, and the tools that come with it.
+The reference guide is a detailed introduction to TDengine including various TDengine connectors in different languages, and the tools that come with TDengine.
 
 ```mdx-code-block
 import DocCardList from '@theme/DocCardList';

docs-examples/go/connect/cgoexample/main.go

@@ -20,4 +20,4 @@ func main() {
 
 // use
 // var taosDSN = "root:taosdata@tcp(localhost:6030)/dbName"
-// if you want to connect to a default database.
+// if you want to connect a specified database named "dbName".

docs-examples/go/connect/restexample/main.go

@@ -20,4 +20,4 @@ func main() {
 
 // use
 // var taosDSN = "root:taosdata@http(localhost:6041)/dbName"
-// if you want to connect to a default database.
+// if you want to connect a specified database named "dbName".

docs-examples/java/src/main/java/com/taos/example/JNIConnectExample.java

@@ -22,4 +22,4 @@ public class JNIConnectExample {
 
 // use
 // String jdbcUrl = "jdbc:TAOS://localhost:6030/dbName?user=root&password=taosdata";
-// if you want to connect to a default database.
+// if you want to connect a specified database named "dbName".

include/dnode/qnode/qnode.h

@@ -26,14 +26,17 @@ extern "C" {
 typedef struct SQnode SQnode;
 
 typedef struct {
-  int64_t numOfStartTask;
+  int64_t numOfProcessedQuery;
-  int64_t numOfStopTask;
+  int64_t numOfProcessedCQuery;
-  int64_t numOfRecvedFetch;
+  int64_t numOfProcessedFetch;
-  int64_t numOfSentHb;
+  int64_t numOfProcessedDrop;
-  int64_t numOfSentFetch;
+  int64_t memSizeInCache;
-  int64_t numOfTaskInQueue;
+  int64_t dataSizeSend;
+  int64_t dataSizeRecv;
+  int64_t numOfQueryInQueue;
   int64_t numOfFetchInQueue;
-  int64_t numOfErrors;
+  int64_t waitTimeInQueryQUeue;
+  int64_t waitTimeInFetchQUeue;
 } SQnodeLoad;
 
 typedef struct {
@@ -71,7 +74,7 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad);
  * @param pQnode The qnode object.
  * @param pMsg The request message
  */
-int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg);
+int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }

include/libs/function/functionMgt.h

@@ -23,6 +23,9 @@ extern "C" {
 #include "function.h"
 #include "querynodes.h"
 
+#define FUNC_AGGREGATE_UDF_ID 5001
+#define FUNC_SCALAR_UDF_ID    5002
+
 typedef enum EFunctionType {
   // aggregate function
   FUNCTION_TYPE_APERCENTILE = 1,
@@ -126,21 +129,12 @@ typedef enum EFunctionType {
 struct SqlFunctionCtx;
 struct SResultRowEntryInfo;
 struct STimeWindow;
-struct SCatalog;
-
-typedef struct SFmGetFuncInfoParam {
-  struct SCatalog* pCtg;
-  void*            pRpc;
-  const SEpSet*    pMgmtEps;
-  char*            pErrBuf;
-  int32_t          errBufLen;
-} SFmGetFuncInfoParam;
 
 int32_t fmFuncMgtInit();
 
 void fmFuncMgtDestroy();
 
-int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc);
+int32_t fmGetFuncInfo(SFunctionNode* pFunc, char* pMsg, int32_t msgLen);
 
 bool fmIsBuiltinFunc(const char* pFunc);
 

include/libs/index/index.h

@@ -192,11 +192,16 @@ void        indexTermDestroy(SIndexTerm* p);
 void indexInit();
 
 /* index filter */
+typedef struct SIndexMetaArg {
+  void*    metaHandle;
+  uint64_t suid;
+} SIndexMetaArg;
+
 typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus;
 
 SIdxFltStatus idxGetFltStatus(SNode* pFilterNode);
 
-int32_t doFilterTag(const SNode* pFilterNode, SArray* result);
+int32_t doFilterTag(const SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result);
 /*
  * destory index env
  *

include/libs/nodes/plannodes.h

@@ -56,6 +56,9 @@ typedef struct SScanLogicNode {
   int8_t             intervalUnit;
   int8_t             slidingUnit;
   SNode*             pTagCond;
+  int8_t             triggerType;
+  int64_t            watermark;
+  int16_t            tsColId;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -216,6 +219,9 @@ typedef struct STableScanPhysiNode {
   int64_t        sliding;
   int8_t         intervalUnit;
   int8_t         slidingUnit;
+  int8_t         triggerType;
+  int64_t        watermark;
+  int16_t        tsColId;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;

include/libs/qworker/qworker.h

@@ -52,22 +52,24 @@ typedef struct {
 
 int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb);
 
-int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
 
-int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
 
-int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
 
-int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
 
-int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
 
-int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
 
-int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
 
 void qWorkerDestroy(void **qWorkerMgmt);
 
+int64_t qWorkerGetWaitTimeInQueue(void *qWorkerMgmt, EQueueType type);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/sync/sync.h

@@ -66,12 +66,6 @@ typedef struct SSyncCfg {
   SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
 } SSyncCfg;
 
-typedef struct SSnapshot {
-  void*     data;
-  SyncIndex lastApplyIndex;
-  SyncTerm  lastApplyTerm;
-} SSnapshot;
-
 typedef struct SFsmCbMeta {
   SyncIndex  index;
   bool       isWeak;
@@ -93,6 +87,12 @@ typedef struct SReConfigCbMeta {
   uint64_t  flag;
 } SReConfigCbMeta;
 
+typedef struct SSnapshot {
+  void *data;
+  SyncIndex lastApplyIndex;
+  SyncTerm  lastApplyTerm;
+} SSnapshot;
+
 typedef struct SSyncFSM {
   void* data;
 
@@ -101,23 +101,17 @@ typedef struct SSyncFSM {
   void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
 
   void (*FpRestoreFinishCb)(struct SSyncFSM* pFsm);
-  int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
-
-  // if (*ppIter == NULL)
-  //   *ppIter = new iter;
-  // else
-  //   *ppIter.next();
-  //
-  // if success, return 0. else return error code
-  int32_t (*FpSnapshotRead)(struct SSyncFSM* pFsm, const SSnapshot* pSnapshot, void** ppIter, char** ppBuf,
-                            int32_t* len);
-
-  // apply data into fsm
-  int32_t (*FpSnapshotApply)(struct SSyncFSM* pFsm, const SSnapshot* pSnapshot, char* pBuf, int32_t len);
-
   void (*FpReConfigCb)(struct SSyncFSM* pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta);
 
-  // int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
+  int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
+
+  int32_t (*FpSnapshotStartRead)(struct SSyncFSM* pFsm, void** ppReader);
+  int32_t (*FpSnapshotStopRead)(struct SSyncFSM* pFsm, void* pReader);
+  int32_t (*FpSnapshotDoRead)(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len);
+
+  int32_t (*FpSnapshotStartWrite)(struct SSyncFSM* pFsm, void** ppWriter);
+  int32_t (*FpSnapshotStopWrite)(struct SSyncFSM* pFsm, void* pWriter, bool isApply);
+  int32_t (*FpSnapshotDoWrite)(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len);
 
 } SSyncFSM;
 

include/util/taoserror.h

@@ -69,6 +69,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_DUP_KEY                       TAOS_DEF_ERROR_CODE(0, 0x0027)
 #define TSDB_CODE_NEED_RETRY                    TAOS_DEF_ERROR_CODE(0, 0x0028)
 #define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE       TAOS_DEF_ERROR_CODE(0, 0x0029)
+#define TSDB_CODE_INVALID_TIMESTAMP             TAOS_DEF_ERROR_CODE(0, 0x0030)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0040)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0041)
@@ -655,7 +656,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FUNC_FUNTION_PARA_NUM         TAOS_DEF_ERROR_CODE(0, 0x2801)
 #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE        TAOS_DEF_ERROR_CODE(0, 0x2802)
 #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE       TAOS_DEF_ERROR_CODE(0, 0x2803)
-#define TSDB_CODE_FUNC_INVALID_FUNTION          TAOS_DEF_ERROR_CODE(0, 0x2804)
+#define TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION      TAOS_DEF_ERROR_CODE(0, 0x2804)
 
 //udf
 #define TSDB_CODE_UDF_STOPPING                  TAOS_DEF_ERROR_CODE(0, 0x2901)

include/util/tqueue.h

@@ -46,6 +46,7 @@ typedef struct {
   void   *ahandle;
   int32_t workerId;
   int32_t threadNum;
+  int64_t timestamp;
 } SQueueInfo;
 
 typedef enum {
@@ -80,7 +81,7 @@ int32_t    taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle);
 void       taosRemoveFromQset(STaosQset *qset, STaosQueue *queue);
 int32_t    taosGetQueueNumber(STaosQset *qset);
 
-int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp);
+int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, int64_t *ts, void **ahandle, FItem *itemFp);
 int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp);
 void    taosResetQsetThread(STaosQset *qset, void *pItem);
 

source/client/src/clientImpl.c

@@ -1249,6 +1249,8 @@ void resetConnectDB(STscObj* pTscObj) {
 int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4) {
   assert(pResultInfo != NULL && pRsp != NULL);
 
+  taosMemoryFreeClear(pResultInfo->pRspMsg);
+  
   pResultInfo->pRspMsg = (const char*)pRsp;
   pResultInfo->pData = (void*)pRsp->data;
   pResultInfo->numOfRows = htonl(pRsp->numOfRows);

source/common/src/tdatablock.c

@@ -611,6 +611,7 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity)
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
+    pCol->hasNull = true;
 
     if (IS_VAR_DATA_TYPE(pCol->info.type)) {
       size_t metaSize = capacity * sizeof(int32_t);
@@ -1153,9 +1154,11 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) {
   if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
     pColumn->varmeta.length = 0;
   } else {
+    if (pColumn->nullbitmap != NULL) {
       memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows));
     }
   }
+}
 
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
   int32_t code = 0;
@@ -1290,8 +1293,8 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
 
 static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-    memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n));
+    memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n) * sizeof(int32_t));
-    memset(&pColInfoData->varmeta.offset[total - n - 1], 0, n);
+    memset(&pColInfoData->varmeta.offset[total - n], 0, n);
   } else {
     int32_t bytes = pColInfoData->info.bytes;
     memmove(pColInfoData->pData, ((char*)pColInfoData->pData + n * bytes), (total - n) * bytes);
@@ -1460,7 +1463,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
 }
 
 void blockDebugShowData(const SArray* dataBlocks) {
-  char    pBuf[128];
+  char    pBuf[128] = {0};
   int32_t sz = taosArrayGetSize(dataBlocks);
   for (int32_t i = 0; i < sz; i++) {
     SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i);

source/common/src/tglobal.c

@@ -293,7 +293,7 @@ int32_t taosAddClientLogCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "idxDebugFlag", 0, 0, 255, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, 1) != 0) return -1;
   return 0;
 }
 

source/common/src/ttime.c

@@ -521,10 +521,10 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
   if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
     newColData = taosMemoryCalloc(1,  charLen + 1);
     memcpy(newColData, varDataVal(inputData), charLen);
-    bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);
+    int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);
     if (ret != TSDB_CODE_SUCCESS) {
       taosMemoryFree(newColData);
-      return ret;
+      return TSDB_CODE_INVALID_TIMESTAMP;
     }
     taosMemoryFree(newColData);
   } else if (type == TSDB_DATA_TYPE_NCHAR) {
@@ -783,7 +783,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio
 //     2020-07-03 17:48:42
 // and the parameter can also be a variable.
 const char* fmtts(int64_t ts) {
-  static char buf[96];
+  static char buf[96] = {0};
   size_t      pos = 0;
   struct tm   tm;
 

source/dnode/mgmt/mgmt_qnode/src/qmHandle.c

@@ -16,7 +16,11 @@
 #define _DEFAULT_SOURCE
 #include "qmInt.h"
 
-void qmGetMonitorInfo(SQnodeMgmt *pMgmt, SMonQmInfo *qmInfo) {}
+void qmGetMonitorInfo(SQnodeMgmt *pMgmt, SMonQmInfo *qmInfo) {
+  SQnodeLoad qload = {0};
+  qndGetLoad(pMgmt->pQnode, &qload);
+
+}
 
 int32_t qmProcessGetMonitorInfoReq(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   SMonQmInfo qmInfo = {0};

source/dnode/mgmt/mgmt_qnode/src/qmWorker.c

@@ -36,7 +36,7 @@ static void qmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
       code = qmProcessGetMonitorInfoReq(pMgmt, pMsg);
       break;
     default:
-      code = qndProcessQueryMsg(pMgmt->pQnode, pMsg);
+      code = qndProcessQueryMsg(pMgmt->pQnode, pInfo->timestamp, pMsg);
       break;
   }
 

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -62,7 +62,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
     dmProcessNetTestReq(pDnode, pRpc);
     return;
   } else if (pRpc->msgType == TDMT_MND_SYSTABLE_RETRIEVE_RSP || pRpc->msgType == TDMT_VND_FETCH_RSP) {
-    qWorkerProcessFetchRsp(NULL, NULL, pRpc);
+    qWorkerProcessFetchRsp(NULL, NULL, pRpc, 0);
     return;
   } else if (pRpc->msgType == TDMT_MND_STATUS_RSP && pEpSet != NULL) {
     dmSetMnodeEpSet(&pDnode->data, pEpSet);

source/dnode/mnode/impl/src/mndMain.c

@@ -370,7 +370,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
     return TAOS_SYNC_PROPOSE_OTHER_ERROR;
   }
  
-  char  logBuf[512];
+  char  logBuf[512] = {0};
   char *syncNodeStr = sync2SimpleStr(pMgmt->sync);
   snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr);
   syncRpcMsgLog2(logBuf, pMsg);

source/dnode/mnode/impl/src/mndQuery.c

@@ -26,19 +26,19 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg) {
   mTrace("msg:%p, in query queue is processing", pMsg);
   switch (pMsg->msgType) {
     case TDMT_VND_QUERY:
-      code = qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pMsg);
+      code = qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pMsg, 0);
       break;
     case TDMT_VND_QUERY_CONTINUE:
-      code = qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pMsg);
+      code = qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pMsg, 0);
       break;
     case TDMT_VND_FETCH:
-      code = qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg);
+      code = qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg, 0);
       break;
     case TDMT_VND_DROP_TASK:
-      code = qWorkerProcessDropMsg(pMnode, pMnode->pQuery, pMsg);
+      code = qWorkerProcessDropMsg(pMnode, pMnode->pQuery, pMsg, 0);
       break;
     case TDMT_VND_QUERY_HEARTBEAT:
-      code = qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg);
+      code = qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg, 0);
       break;
     default:
       terrno = TSDB_CODE_VND_APP_ERROR;

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -940,7 +940,7 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
     }
 
     // do not show for cleared subscription
-#if 0
+#if 1
     int32_t sz = taosArrayGetSize(pSub->unassignedVgs);
     for (int32_t i = 0; i < sz; i++) {
       SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i);

source/dnode/mnode/impl/src/mndSync.c

@@ -48,6 +48,10 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM
       mError("trans:%d, failed to propose since %s", transId, tstrerror(pMgmt->errCode));
     }
     tsem_post(&pMgmt->syncSem);
+  } else {
+    if (cbMeta.index - sdbGetApplyIndex(pMnode->pSdb) > 100) {
+      sdbWriteFile(pMnode->pSdb);
+    }
   }
 }
 
@@ -64,42 +68,11 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) {
     mInfo("mnode sync restore finished");
     mndTransPullup(pMnode);
     mndSetRestore(pMnode, true);
-  }
-}
-
-int32_t mndSnapshotRead(struct SSyncFSM *pFsm, const SSnapshot *pSnapshot, void **ppIter, char **ppBuf, int32_t *len) {
-  SMnode *pMnode = pFsm->data;
-  mInfo("start to read snapshot from sdb");
-
-  int32_t code = sdbReadSnapshot(pMnode->pSdb, (SSdbIter **)ppIter, ppBuf, len);
-  if (code != 0) {
-    mError("failed to read snapshot from sdb since %s", terrstr());
   } else {
-    if (*ppIter == NULL) {
+    mInfo("mnode sync restore finished, and will set ready after first deploy");
-      mInfo("successfully to read snapshot from sdb");
   }
 }
 
-  return code;
-}
-
-int32_t mndSnapshotApply(struct SSyncFSM *pFsm, const SSnapshot *pSnapshot, char *pBuf, int32_t len) {
-  SMnode *pMnode = pFsm->data;
-  mndSetRestore(pMnode, false);
-  mInfo("start to apply snapshot to sdb, len:%d", len);
-
-  int32_t code = sdbApplySnapshot(pMnode->pSdb, pBuf, len);
-  if (code != 0) {
-    mError("failed to apply snapshot to sdb, len:%d", len);
-  } else {
-    mInfo("successfully to apply snapshot to sdb, len:%d", len);
-    mndSetRestore(pMnode, true);
-  }
-
-  // taosMemoryFree(pBuf);
-  return code;
-}
-
 void mndReConfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
   SMnode    *pMnode = pFsm->data;
   SSyncMgmt *pMgmt = &pMnode->syncMgmt;
@@ -116,20 +89,55 @@ void mndReConfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta)
   }
 }
 
+int32_t mndSnapshotStartRead(struct SSyncFSM *pFsm, void **ppReader) {
+  mInfo("start to read snapshot from sdb");
+  SMnode *pMnode = pFsm->data;
+  return sdbStartRead(pMnode->pSdb, (SSdbIter **)ppReader);
+}
+
+int32_t mndSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) {
+  mInfo("stop to read snapshot from sdb");
+  SMnode *pMnode = pFsm->data;
+  return sdbStopRead(pMnode->pSdb, pReader);
+}
+
+int32_t mndSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
+  SMnode *pMnode = pFsm->data;
+  return sdbDoRead(pMnode->pSdb, pReader, ppBuf, len);
+}
+
+int32_t mndSnapshotStartWrite(struct SSyncFSM *pFsm, void **ppWriter) {
+  mInfo("start to apply snapshot to sdb");
+  SMnode *pMnode = pFsm->data;
+  return sdbStartWrite(pMnode->pSdb, (SSdbIter **)ppWriter);
+}
+
+int32_t mndSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) {
+  mInfo("stop to apply snapshot to sdb, apply:%d", isApply);
+  SMnode *pMnode = pFsm->data;
+  return sdbStopWrite(pMnode->pSdb, pWriter, isApply);
+}
+
+int32_t mndSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) {
+  SMnode *pMnode = pFsm->data;
+  return sdbDoWrite(pMnode->pSdb, pWriter, pBuf, len);
+}
+
 SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) {
   SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
   pFsm->data = pMnode;
-
   pFsm->FpCommitCb = mndSyncCommitMsg;
   pFsm->FpPreCommitCb = NULL;
   pFsm->FpRollBackCb = NULL;
-
-  pFsm->FpGetSnapshot = mndSyncGetSnapshot;
   pFsm->FpRestoreFinishCb = mndRestoreFinish;
-  pFsm->FpSnapshotRead = mndSnapshotRead;
-  pFsm->FpSnapshotApply = mndSnapshotApply;
   pFsm->FpReConfigCb = mndReConfig;
-
+  pFsm->FpGetSnapshot = mndSyncGetSnapshot;
+  pFsm->FpSnapshotStartRead = mndSnapshotStartRead;
+  pFsm->FpSnapshotStopRead = mndSnapshotStopRead;
+  pFsm->FpSnapshotDoRead = mndSnapshotDoRead;
+  pFsm->FpSnapshotStartWrite = mndSnapshotStartWrite;
+  pFsm->FpSnapshotStopWrite = mndSnapshotStopWrite;
+  pFsm->FpSnapshotDoWrite = mndSnapshotDoWrite;
   return pFsm;
 }
 

source/dnode/mnode/impl/test/acct/CMakeLists.txt

@@ -5,7 +5,9 @@ target_link_libraries(
     PUBLIC sut
 )
 
+if(NOT TD_WINDOWS)
     add_test(
         NAME acctTest
         COMMAND acctTest
     )
+endif(NOT TD_WINDOWS)

source/dnode/mnode/impl/test/func/CMakeLists.txt

@@ -5,7 +5,9 @@ target_link_libraries(
     PUBLIC sut
 )
 
+if(NOT TD_WINDOWS)
     add_test(
         NAME funcTest
         COMMAND funcTest
     )
+endif(NOT TD_WINDOWS)

source/dnode/mnode/impl/test/profile/CMakeLists.txt

@@ -5,7 +5,9 @@ target_link_libraries(
     PUBLIC sut
 )
 
+if(NOT TD_WINDOWS)
     add_test(
         NAME profileTest
         COMMAND profileTest
     )
+endif(NOT TD_WINDOWS)

source/dnode/mnode/impl/test/sdb/sdbTest.cpp

@@ -895,7 +895,35 @@ TEST_F(MndTestSdb, 01_Read_Str) {
     ASSERT_EQ(code, TSDB_CODE_SDB_OBJ_CREATING);
   }
 
-  sdbCleanup(pSdb);
+  {
-  ASSERT_EQ(mnode.insertTimes, 5);
+    SSdbIter *pReader = NULL;
-  ASSERT_EQ(mnode.deleteTimes, 5);
+    SSdbIter *pWritter = NULL;
+    void     *pBuf = NULL;
+    int32_t   len = 0;
+    int32_t   code = 0;
+
+    code = sdbStartRead(pSdb, &pReader);
+    ASSERT_EQ(code, 0);
+    code = sdbStartWrite(pSdb, &pWritter);
+    ASSERT_EQ(code, 0);
+
+    while (sdbDoRead(pSdb, pReader, &pBuf, &len) == 0) {
+      if (pBuf != NULL && len != 0) {
+        sdbDoWrite(pSdb, pWritter, pBuf, len);
+        taosMemoryFree(pBuf);
+      } else {
+        break;
+      }
+    }
+
+    sdbStopRead(pSdb, pReader);
+    sdbStopWrite(pSdb, pWritter, true);
+  }
+
+  ASSERT_EQ(sdbGetSize(pSdb, SDB_CONSUMER), 1);
+  ASSERT_EQ(sdbGetTableVer(pSdb, SDB_CONSUMER), 4);
+
+  sdbCleanup(pSdb);
+  ASSERT_EQ(mnode.insertTimes, 9);
+  ASSERT_EQ(mnode.deleteTimes, 9);
 }

source/dnode/mnode/impl/test/show/CMakeLists.txt

@@ -5,7 +5,9 @@ target_link_libraries(
     PUBLIC sut
 )
 
+if(NOT TD_WINDOWS)
     add_test(
         NAME showTest
         COMMAND showTest
     )
+endif(NOT TD_WINDOWS)

source/dnode/mnode/sdb/inc/sdb.h

@@ -187,6 +187,7 @@ typedef struct SSdb {
 typedef struct SSdbIter {
   TdFilePtr file;
   int64_t   total;
+  char     *name;
 } SSdbIter;
 
 typedef struct {
@@ -380,8 +381,13 @@ SSdbRow *sdbAllocRow(int32_t objSize);
 void    *sdbGetRowObj(SSdbRow *pRow);
 void     sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
 
-int32_t sdbReadSnapshot(SSdb *pSdb, SSdbIter **ppIter, char **ppBuf, int32_t *len);
+int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter);
-int32_t sdbApplySnapshot(SSdb *pSdb, char *pBuf, int32_t len);
+int32_t sdbStopRead(SSdb *pSdb, SSdbIter *pIter);
+int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len);
+
+int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter);
+int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply);
+int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len);
 
 const char *sdbTableName(ESdbType type);
 void        sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper);

source/dnode/mnode/sdb/src/sdb.c

@@ -71,6 +71,7 @@ void sdbCleanup(SSdb *pSdb) {
   }
 
   if (pSdb->tmpDir != NULL) {
+    taosRemoveDir(pSdb->tmpDir);
     taosMemoryFreeClear(pSdb->tmpDir);
   }
 

source/dnode/mnode/sdb/src/sdbFile.c

@@ -445,168 +445,166 @@ int32_t sdbDeploy(SSdb *pSdb) {
   return 0;
 }
 
-static SSdbIter *sdbOpenIter(SSdb *pSdb) {
+static SSdbIter *sdbCreateIter(SSdb *pSdb) {
-  char datafile[PATH_MAX] = {0};
-  char tmpfile[PATH_MAX] = {0};
-  snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
-  snprintf(tmpfile, sizeof(tmpfile), "%s%ssdb.data", pSdb->tmpDir, TD_DIRSEP);
-
-  taosThreadMutexLock(&pSdb->filelock);
-  if (taosCopyFile(datafile, tmpfile) != 0) {
-    taosThreadMutexUnlock(&pSdb->filelock);
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    mError("failed to copy file %s to %s since %s", datafile, tmpfile, terrstr());
-    return NULL;
-  }
-  taosThreadMutexUnlock(&pSdb->filelock);
-
   SSdbIter *pIter = taosMemoryCalloc(1, sizeof(SSdbIter));
   if (pIter == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
-  pIter->file = taosOpenFile(tmpfile, TD_FILE_READ);
+  char name[PATH_MAX + 100] = {0};
-  if (pIter->file == NULL) {
+  snprintf(name, sizeof(name), "%s%ssdb.data.%" PRIu64, pSdb->tmpDir, TD_DIRSEP, (uint64_t)pIter);
-    terrno = TAOS_SYSTEM_ERROR(errno);
+  pIter->name = strdup(name);
-    mError("failed to read file:%s since %s", tmpfile, terrstr());
+  if (pIter->name == NULL) {
     taosMemoryFree(pIter);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
   return pIter;
 }
 
-static void sdbCloseIter(SSdb *pSdb, SSdbIter *pIter) {
+static void sdbCloseIter(SSdbIter *pIter) {
   if (pIter == NULL) return;
+
   if (pIter->file != NULL) {
     taosCloseFile(&pIter->file);
+    pIter->file = NULL;
   }
 
-  char tmpfile[PATH_MAX] = {0};
+  if (pIter->name != NULL) {
-  snprintf(tmpfile, sizeof(tmpfile), "%s%ssdb.data", pSdb->tmpDir, TD_DIRSEP);
+    taosRemoveFile(pIter->name);
-  taosRemoveFile(tmpfile);
+    taosMemoryFree(pIter->name);
+    pIter->name = NULL;
+  }
 
+  mInfo("sdbiter:%p, is closed, total:%" PRId64, pIter, pIter->total);
   taosMemoryFree(pIter);
-  mInfo("sdbiter:%p, is closed", pIter);
 }
 
-static SSdbIter *sdbGetIter(SSdb *pSdb, SSdbIter **ppIter) {
+int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) {
-  SSdbIter *pIter = NULL;
+  SSdbIter *pIter = sdbCreateIter(pSdb);
-  if (ppIter != NULL) pIter = *ppIter;
-
-  if (pIter == NULL) {
-    pIter = sdbOpenIter(pSdb);
-    if (pIter != NULL) {
-      mInfo("sdbiter:%p, is created to read snapshot", pIter);
-      *ppIter = pIter;
-    } else {
-      mError("failed to create sdbiter to read snapshot since %s", terrstr());
-      *ppIter = NULL;
-      return NULL;
-    }
-  } else {
-    mInfo("sdbiter:%p, continue to read snapshot, total:%" PRId64, pIter, pIter->total);
-  }
-
-  return pIter;
-}
-
-int32_t sdbReadSnapshot(SSdb *pSdb, SSdbIter **ppIter, char **ppBuf, int32_t *len) {
-  SSdbIter *pIter = sdbGetIter(pSdb, ppIter);
   if (pIter == NULL) return -1;
 
+  char datafile[PATH_MAX] = {0};
+  snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
+
+  taosThreadMutexLock(&pSdb->filelock);
+  if (taosCopyFile(datafile, pIter->name) < 0) {
+    taosThreadMutexUnlock(&pSdb->filelock);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    mError("failed to copy file %s to %s since %s", datafile, pIter->name, terrstr());
+    sdbCloseIter(pIter);
+    return -1;
+  }
+  taosThreadMutexUnlock(&pSdb->filelock);
+
+  pIter->file = taosOpenFile(pIter->name, TD_FILE_READ);
+  if (pIter->file == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    mError("failed to open file:%s since %s", pIter->name, terrstr());
+    sdbCloseIter(pIter);
+    return -1;
+  }
+
+  *ppIter = pIter;
+  mInfo("sdbiter:%p, is created to read snapshot, file:%s", pIter, pIter->name);
+  return 0;
+}
+
+int32_t sdbStopRead(SSdb *pSdb, SSdbIter *pIter) {
+  sdbCloseIter(pIter);
+  return 0;
+}
+
+int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) {
   int32_t maxlen = 100;
-  char   *pBuf = taosMemoryCalloc(1, maxlen);
+  void   *pBuf = taosMemoryCalloc(1, maxlen);
   if (pBuf == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    sdbCloseIter(pSdb, pIter);
     return -1;
   }
 
   int32_t readlen = taosReadFile(pIter->file, pBuf, maxlen);
-  if (readlen < 0 || (readlen == 0 && errno != 0)) {
+  if (readlen < 0 || readlen > maxlen) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("sdbiter:%p, failed to read snapshot since %s, total:%" PRId64, pIter, terrstr(), pIter->total);
     *ppBuf = NULL;
     *len = 0;
-    *ppIter = NULL;
-    sdbCloseIter(pSdb, pIter);
     taosMemoryFree(pBuf);
     return -1;
   } else if (readlen == 0) {
     mInfo("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total);
     *ppBuf = NULL;
     *len = 0;
-    *ppIter = NULL;
-    sdbCloseIter(pSdb, pIter);
     taosMemoryFree(pBuf);
     return 0;
-  } else if ((readlen < maxlen && errno != 0) || readlen == maxlen) {
+  } else {  // (readlen <= maxlen)
     pIter->total += readlen;
     mInfo("sdbiter:%p, read:%d bytes from snapshot, total:%" PRId64, pIter, readlen, pIter->total);
     *ppBuf = pBuf;
     *len = readlen;
     return 0;
-  } else if (readlen < maxlen && errno == 0) {
-    mInfo("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total);
-    *ppBuf = pBuf;
-    *len = readlen;
-    *ppIter = NULL;
-    sdbCloseIter(pSdb, pIter);
-    return 0;
-  } else {
-    // impossible
-    mError("sdbiter:%p, read:%d bytes from snapshot, total:%" PRId64, pIter, readlen, pIter->total);
-    *ppBuf = NULL;
-    *len = 0;
-    *ppIter = NULL;
-    sdbCloseIter(pSdb, pIter);
-    taosMemoryFree(pBuf);
-    return -1;
   }
 }
 
-int32_t sdbApplySnapshot(SSdb *pSdb, char *pBuf, int32_t len) {
+int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) {
-  char datafile[PATH_MAX] = {0};
+  SSdbIter *pIter = sdbCreateIter(pSdb);
-  char tmpfile[PATH_MAX] = {0};
+  if (pIter == NULL) return -1;
-  snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
-  snprintf(tmpfile, sizeof(tmpfile), "%s%ssdb.data", pSdb->tmpDir, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  pIter->file = taosOpenFile(pIter->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
+  if (pIter->file == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    mError("failed to write %s since %s", tmpfile, terrstr());
+    mError("failed to open %s since %s", pIter->name, terrstr());
     return -1;
   }
 
-  int32_t writelen = taosWriteFile(pFile, pBuf, len);
+  *ppIter = pIter;
+  mInfo("sdbiter:%p, is created to write snapshot, file:%s", pIter, pIter->name);
+  return 0;
+}
+
+int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply) {
+  int32_t code = 0;
+
+  if (!isApply) {
+    sdbCloseIter(pIter);
+    mInfo("sdbiter:%p, not apply to sdb", pIter);
+    return 0;
+  }
+
+  taosFsyncFile(pIter->file);
+  taosCloseFile(&pIter->file);
+  pIter->file = NULL;
+
+  char datafile[PATH_MAX] = {0};
+  snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
+  if (taosRenameFile(pIter->name, datafile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    mError("sdbiter:%p, failed to rename file %s to %s since %s", pIter, pIter->name, datafile, terrstr());
+    sdbCloseIter(pIter);
+    return -1;
+  }
+
+  sdbCloseIter(pIter);
+  if (sdbReadFile(pSdb) != 0) {
+    mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr());
+    return -1;
+  }
+
+  mInfo("sdbiter:%p, successfully applyed to sdb", pIter);
+  return 0;
+}
+
+int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) {
+  int32_t writelen = taosWriteFile(pIter->file, pBuf, len);
   if (writelen != len) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    mError("failed to write %s since %s", tmpfile, terrstr());
+    mError("failed to write len:%d since %s, total:%" PRId64, len, terrstr(), pIter->total);
-    taosCloseFile(&pFile);
-    return -1;
-  }
-
-  if (taosFsyncFile(pFile) != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    mError("failed to fsync %s since %s", tmpfile, terrstr());
-    taosCloseFile(&pFile);
-    return -1;
-  }
-
-  (void)taosCloseFile(&pFile);
-
-  if (taosRenameFile(tmpfile, datafile) != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    mError("failed to rename file %s to %s since %s", tmpfile, datafile, terrstr());
-    return -1;
-  }
-
-  if (sdbReadFile(pSdb) != 0) {
-    mError("failed to read from %s since %s", datafile, terrstr());
     return -1;
   }
 
+  pIter->total += writelen;
+  mInfo("sdbiter:%p, write:%d bytes to snapshot, total:%" PRId64, pIter, writelen, pIter->total);
   return 0;
 }

source/dnode/qnode/src/qnode.c

@@ -40,37 +40,46 @@ void qndClose(SQnode *pQnode) {
   taosMemoryFree(pQnode);
 }
 
-int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; }
+int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { 
+  SMsgCb* pCb = &pQnode->msgCb;
 
-int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) {
+  pLoad->numOfQueryInQueue = pCb->qsizeFp(pCb->mgmt, pQnode->qndId, QUERY_QUEUE);
+  pLoad->numOfFetchInQueue = pCb->qsizeFp(pCb->mgmt, pQnode->qndId, FETCH_QUEUE);
+  pLoad->waitTimeInQueryQUeue = qWorkerGetWaitTimeInQueue(pQnode->pQuery, QUERY_QUEUE);
+  pLoad->waitTimeInFetchQUeue = qWorkerGetWaitTimeInQueue(pQnode->pQuery, FETCH_QUEUE);
+  
+  return 0; 
+}
+
+int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) {
   int32_t     code = -1;
   SReadHandle handle = {.pMsgCb = &pQnode->msgCb};
   qTrace("message in qnode queue is processing");
 
   switch (pMsg->msgType) {
     case TDMT_VND_QUERY:
-      code = qWorkerProcessQueryMsg(&handle, pQnode->pQuery, pMsg);
+      code = qWorkerProcessQueryMsg(&handle, pQnode->pQuery, pMsg, ts);
       break;
     case TDMT_VND_QUERY_CONTINUE:
-      code = qWorkerProcessCQueryMsg(&handle, pQnode->pQuery, pMsg);
+      code = qWorkerProcessCQueryMsg(&handle, pQnode->pQuery, pMsg, ts);
       break;
     case TDMT_VND_FETCH:
-      code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg);
+      code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg, ts);
       break;
     case TDMT_VND_FETCH_RSP:
-      code = qWorkerProcessFetchRsp(pQnode, pQnode->pQuery, pMsg);
+      code = qWorkerProcessFetchRsp(pQnode, pQnode->pQuery, pMsg, ts);
       break;
     case TDMT_VND_CANCEL_TASK:
-      code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg);
+      code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts);
       break;
     case TDMT_VND_DROP_TASK:
-      code = qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg);
+      code = qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg, ts);
       break;
     case TDMT_VND_CONSUME:
       // code =  tqProcessConsumeReq(pQnode->pTq, pMsg);
       // break;
     case TDMT_VND_QUERY_HEARTBEAT:
-      code = qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg);
+      code = qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg, ts);
       break;
     default:
       qError("unknown msg type:%d in qnode queue", pMsg->msgType);

source/dnode/vnode/inc/vnode.h

@@ -106,7 +106,9 @@ tsdbReaderT  tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STab
 int32_t      tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo);
 bool         isTsdbCacheLastRow(tsdbReaderT *pReader);
 int32_t      tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list);
+void *       tsdbGetIdx(SMeta *pMeta);
 int64_t      tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
+
 bool    tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
 void    tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
 int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg ***pBlockStatis, bool *allHave);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -103,6 +103,7 @@ SArray*         metaGetSmaTbUids(SMeta* pMeta);
 int32_t         metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppReader, int64_t sver, int64_t ever);
 int32_t         metaSnapshotReaderClose(SMetaSnapshotReader* pReader);
 int32_t         metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData);
+void*           metaGetIdx(SMeta* pMeta);
 
 int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
 int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);

source/dnode/vnode/src/meta/metaTable.c

@@ -880,3 +880,11 @@ _err:
   metaULock(pMeta);
   return -1;
 }
+// refactor later
+void *metaGetIdx(SMeta *pMeta) {
+#ifdef USE_INVERTED_INDEX
+  return pMeta->pTagIvtIdx;
+#else
+  return pMeta->pTagIdx;
+#endif
+}

source/dnode/vnode/src/tq/tq.c

@@ -235,6 +235,15 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
       }
     }
   }
+  while (1) {
+    pIter = taosHashIterate(pTq->pStreamTasks, pIter);
+    if (pIter == NULL) break;
+    SStreamTask* pTask = (SStreamTask*)pIter;
+    if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
+      int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd);
+      ASSERT(code == 0);
+    }
+  }
   return 0;
 }
 

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -248,8 +248,8 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
     }
 
     taosArrayPush(pTableCheckInfo, &info);
-    tsdbDebug("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReadHandle, info.tableId,
+    tsdbDebug("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReadHandle, info.tableId, info.lastKey,
-              info.lastKey, pTsdbReadHandle->idStr);
+              pTsdbReadHandle->idStr);
   }
 
   // TODO  group table according to the tag value.
@@ -352,13 +352,16 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle,
     }
 
     if (level == TSDB_RETENTION_L0) {
-      tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L0);
+      tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle,
+                TSDB_RETENTION_L0);
       return VND_RSMA0(pVnode);
     } else if (level == TSDB_RETENTION_L1) {
-      tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L1);
+      tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle,
+                TSDB_RETENTION_L1);
       return VND_RSMA1(pVnode);
     } else {
-      tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L2);
+      tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle,
+                TSDB_RETENTION_L2);
       return VND_RSMA2(pVnode);
     }
   }
@@ -1324,7 +1327,6 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
 
   if ((ascScan && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
       (!ascScan && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
-
     bool cacheDataInFileBlockHole = (ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
                                     (!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey));
     if (cacheDataInFileBlockHole) {
@@ -1992,8 +1994,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
 
   STimeWindow* pWin = &blockInfo.window;
   tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64
-            " rows:%d, start:%d, end:%d, %s", pTsdbReadHandle, pCheckInfo->tableId, pWin->skey, pWin->ekey, blockInfo.rows,
+            " rows:%d, start:%d, end:%d, %s",
-            cur->pos, endPos, pTsdbReadHandle->idStr);
+            pTsdbReadHandle, pCheckInfo->tableId, pWin->skey, pWin->ekey, blockInfo.rows, cur->pos, endPos,
+            pTsdbReadHandle->idStr);
 
   // compared with the data from in-memory buffer, to generate the correct timestamp array list
   int32_t numOfRows = 0;
@@ -2112,7 +2115,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           }
 
           // still assign data into current row
-          numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
+          numOfRows +=
+              mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols,
                                  pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend);
 
           if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -2178,8 +2182,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
        * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT
        * copy them all to result buffer, since it may be overlapped with file data block.
        */
-      if (node == NULL ||
+      if (node == NULL || ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && ascScan) ||
-          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && ascScan) ||
           ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && !ascScan)) {
         // no data in cache or data in cache is greater than the ekey of time window, load data from file block
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -2819,6 +2822,12 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
   return numOfRows;
 }
 
+void* tsdbGetIdx(SMeta* pMeta) {
+  if (pMeta == NULL) {
+    return NULL;
+  }
+  return metaGetIdx(pMeta);
+}
 int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
   SMCtbCursor* pCur = metaOpenCtbCursor(pMeta, uid);
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -191,9 +191,9 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
   switch (pMsg->msgType) {
     case TDMT_VND_QUERY:
-      return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg);
+      return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg, 0);
     case TDMT_VND_QUERY_CONTINUE:
-      return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg);
+      return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg, 0);
     default:
       vError("unknown msg type:%d in query queue", pMsg->msgType);
       return TSDB_CODE_VND_APP_ERROR;
@@ -206,13 +206,16 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
   int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
   switch (pMsg->msgType) {
     case TDMT_VND_FETCH:
-      return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg);
+      return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0);
     case TDMT_VND_FETCH_RSP:
-      return qWorkerProcessFetchRsp(pVnode, pVnode->pQuery, pMsg);
+      return qWorkerProcessFetchRsp(pVnode, pVnode->pQuery, pMsg, 0);
     case TDMT_VND_CANCEL_TASK:
-      return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg);
+      return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0);
     case TDMT_VND_DROP_TASK:
-      return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg);
+      return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg, 0);
+    case TDMT_VND_QUERY_HEARTBEAT:
+      return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg, 0);
+
     case TDMT_VND_TABLE_META:
       return vnodeGetTableMeta(pVnode, pMsg);
     case TDMT_VND_CONSUME:
@@ -231,9 +234,6 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
       return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg);
     case TDMT_VND_TASK_RECOVER_RSP:
       return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg);
-
-    case TDMT_VND_QUERY_HEARTBEAT:
-      return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg);
     default:
       vError("unknown msg type:%d in fetch queue", pMsg->msgType);
       return TSDB_CODE_VND_APP_ERROR;
@@ -260,7 +260,7 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
 
     SMsgHead *pHead = pMsg->pCont;
 
-    char  logBuf[512];
+    char  logBuf[512] = {0};
     char *syncNodeStr = sync2SimpleStr(pVnode->sync);
     snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr);
     syncRpcMsgLog2(logBuf, pMsg);
@@ -678,6 +678,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
   int32_t        nRows;
   int32_t        tsize, ret;
   SEncoder       encoder = {0};
+  SArray        *newTbUids = NULL;
   terrno = TSDB_CODE_SUCCESS;
 
   pRsp->code = 0;
@@ -698,6 +699,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
   }
 
   submitRsp.pArray = taosArrayInit(pSubmitReq->numOfBlocks, sizeof(SSubmitBlkRsp));
+  newTbUids = taosArrayInit(pSubmitReq->numOfBlocks, sizeof(int64_t));
   if (!submitRsp.pArray) {
     pRsp->code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
@@ -727,6 +729,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
           goto _exit;
         }
       }
+      taosArrayPush(newTbUids, &createTbReq.uid);
 
       submitBlkRsp.uid = createTbReq.uid;
       submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2);
@@ -754,8 +757,10 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
     submitRsp.affectedRows += submitBlkRsp.affectedRows;
     taosArrayPush(submitRsp.pArray, &submitBlkRsp);
   }
+  tqUpdateTbUidList(pVnode->pTq, newTbUids, true);
 
 _exit:
+  taosArrayDestroy(newTbUids);
   tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret);
   pRsp->pCont = rpcMallocCont(tsize);
   pRsp->contLen = tsize;

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -80,7 +80,7 @@ void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta)
   }
 
   if (cbMeta.index > beginIndex) {
-    char logBuf[256];
+    char logBuf[256] = {0};
     snprintf(
         logBuf, sizeof(logBuf),
         "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
@@ -115,7 +115,7 @@ void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta)
     tmsgPutToQueue(&(pVnode->msgCb), APPLY_QUEUE, &applyMsg);
 
   } else {
-    char logBuf[256];
+    char logBuf[256] = {0};
     snprintf(logBuf, sizeof(logBuf),
              "==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, "
              "beginIndex :%ld\n",
@@ -126,7 +126,7 @@ void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta)
 }
 
 void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
-  char logBuf[256];
+  char logBuf[256] = {0};
   snprintf(logBuf, sizeof(logBuf),
            "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
            cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
@@ -134,7 +134,7 @@ void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMet
 }
 
 void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
-  char logBuf[256];
+  char logBuf[256] = {0};
   snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
            pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
   syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
@@ -142,14 +142,13 @@ void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta
 
 SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
   SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
+  memset(pFsm, 0, sizeof(*pFsm));
   pFsm->data = pVnode;
   pFsm->FpCommitCb = vnodeSyncCommitMsg;
   pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg;
   pFsm->FpRollBackCb = vnodeSyncRollBackMsg;
   pFsm->FpGetSnapshot = vnodeSyncGetSnapshot;
   pFsm->FpRestoreFinishCb = NULL;
-  pFsm->FpSnapshotRead = NULL;
-  pFsm->FpSnapshotApply = NULL;
   pFsm->FpReConfigCb = NULL;
 
   return pFsm;

source/libs/catalog/src/ctgDbg.c

@@ -71,6 +71,16 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
     qDebug("empty db vgroup");
   }
 
+  if (pResult->pDbInfo && taosArrayGetSize(pResult->pDbInfo) > 0) {
+    num = taosArrayGetSize(pResult->pDbInfo);
+    for (int32_t i = 0; i < num; ++i) {
+      SDbInfo *pDb = taosArrayGet(pResult->pDbInfo, i);
+      qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, dbId:%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->dbId);
+    }
+  } else {
+    qDebug("empty db info");
+  }
+
   if (pResult->pTableHash && taosArrayGetSize(pResult->pTableHash) > 0) {
     num = taosArrayGetSize(pResult->pTableHash);
     for (int32_t i = 0; i < num; ++i) {
@@ -127,6 +137,7 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps
   SCatalogReq req = {0};
   req.pTableMeta = taosArrayInit(2, sizeof(SName));
   req.pDbVgroup = taosArrayInit(2, TSDB_DB_FNAME_LEN);
+  req.pDbInfo = taosArrayInit(2, TSDB_DB_FNAME_LEN);
   req.pTableHash = taosArrayInit(2, sizeof(SName));
   req.pUdf = taosArrayInit(2, TSDB_FUNC_NAME_LEN);
   req.pDbCfg = taosArrayInit(2, TSDB_DB_FNAME_LEN);
@@ -149,9 +160,11 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps
   strcpy(dbFName, "1.db1");
   taosArrayPush(req.pDbVgroup, dbFName);
   taosArrayPush(req.pDbCfg, dbFName);
+  taosArrayPush(req.pDbInfo, dbFName);
   strcpy(dbFName, "1.db2");
   taosArrayPush(req.pDbVgroup, dbFName);
   taosArrayPush(req.pDbCfg, dbFName);
+  taosArrayPush(req.pDbInfo, dbFName);
 
   strcpy(funcName, "udf1");
   taosArrayPush(req.pUdf, funcName);

source/libs/catalog/test/catalogTests.cpp

@@ -41,7 +41,6 @@
 namespace {
 
 extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
-extern "C" int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action);
 extern "C" int32_t ctgdEnableDebug(char *option);
 extern "C" int32_t ctgdGetStatNum(char *option, void *res);
 

source/libs/command/src/explain.c

@@ -560,8 +560,10 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
         EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
       }
+      if (pAggNode->pAggFuncs) {
         EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pAggNode->pAggFuncs->length);
         EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+      }
       EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->totalRowSize);
       if (pAggNode->pGroupKeys) {
         EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);

source/libs/executor/inc/executorimpl.h

@@ -156,13 +156,11 @@ typedef struct STaskAttr {
 } STaskAttr;
 
 struct SOperatorInfo;
-struct SAggSupporter;
+//struct SAggSupporter;
-struct SOptrBasicInfo;
+//struct SOptrBasicInfo;
 
-typedef void (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, struct SAggSupporter* pSup,
+typedef int32_t (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, char** result, int32_t* length);
-                                   struct SOptrBasicInfo* pInfo, char** result, int32_t* length);
+typedef int32_t (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, char* result, int32_t length);
-typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, struct SAggSupporter* pSup,
-                                   struct SOptrBasicInfo* pInfo, char* result, int32_t length);
 
 typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr);
 typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr);
@@ -388,6 +386,10 @@ typedef struct SStreamBlockScanInfo {
   SNode*          pCondition;
   SArray*         tsArray;
   SUpdateInfo*    pUpdateInfo;
+
+  SExprInfo*      pPseudoExpr;
+  int32_t         numOfPseudoExpr;
+
   int32_t         primaryTsIndex;    // primary time stamp slot id
   void*           pDataReader;
   SReadHandle     readHandle;
@@ -395,7 +397,6 @@ typedef struct SStreamBlockScanInfo {
   EStreamScanMode scanMode;
   SOperatorInfo* pOperatorDumy;
   SInterval      interval;     // if the upstream is an interval operator, the interval info is also kept here.
-  SCatchSupporter childAggSup;
   SArray*        childIds;
   SessionWindowSupporter sessionSup;
   bool            assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
@@ -437,18 +438,21 @@ typedef struct SAggSupporter {
 typedef struct STimeWindowSupp {
   int8_t           calTrigger;
   int64_t          waterMark;
+  TSKEY            maxTs;
   SColumnInfoData  timeWindowData;     // query time window info for scalar function execution.
 } STimeWindowAggSupp;
 
 typedef struct SIntervalAggOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo     binfo;              // basic info
+  SAggSupporter      aggSup;             // aggregate supporter
+
   SGroupResInfo      groupResInfo;       // multiple results build supporter
   SInterval          interval;           // interval info
   int32_t            primaryTsIndex;     // primary time stamp slot id from result of downstream operator.
   STimeWindow        win;                // query time range
   bool               timeWindowInterpo;  // interpolation needed or not
   char**             pRow;               // previous row/tuple of already processed datablock
-  SAggSupporter      aggSup;             // aggregate supporter
   STableQueryInfo*   pCurrent;           // current tableQueryInfo struct
   int32_t            order;              // current SSDataBlock scan order
   EOPTR_EXEC_MODEL   execModel;          // operator execution model [batch model|stream model]
@@ -459,19 +463,23 @@ typedef struct SIntervalAggOperatorInfo {
 } SIntervalAggOperatorInfo;
 
 typedef struct SStreamFinalIntervalOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo     binfo;              // basic info
+  SAggSupporter      aggSup;             // aggregate supporter
+
   SGroupResInfo      groupResInfo;       // multiple results build supporter
   SInterval          interval;           // interval info
   int32_t            primaryTsIndex;     // primary time stamp slot id from result of downstream operator.
-  SAggSupporter      aggSup;             // aggregate supporter
   int32_t            order;              // current SSDataBlock scan order
   STimeWindowAggSupp twAggSup;
   SArray*            pChildren;
 } SStreamFinalIntervalOperatorInfo;
 
 typedef struct SAggOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo     binfo;
   SAggSupporter      aggSup;
+
   STableQueryInfo   *current;
   uint64_t           groupId;
   SGroupResInfo      groupResInfo;
@@ -484,8 +492,10 @@ typedef struct SAggOperatorInfo {
 } SAggOperatorInfo;
 
 typedef struct SProjectOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo     binfo;
   SAggSupporter      aggSup;
+
   SSDataBlock*       existDataBlock;
   SArray*            pPseudoColInfo;
   SLimit             limit;
@@ -509,7 +519,10 @@ typedef struct SFillOperatorInfo {
 } SFillOperatorInfo;
 
 typedef struct SGroupbyOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo  binfo;
+  SAggSupporter   aggSup;
+
   SArray*         pGroupCols;     // group by columns, SArray<SColumn>
   SArray*         pGroupColVals;  // current group column values, SArray<SGroupKeys>
   SNode*          pCondition;
@@ -517,7 +530,6 @@ typedef struct SGroupbyOperatorInfo {
   char*           keyBuf;       // group by keys for hash
   int32_t         groupKeyLen;  // total group by column width
   SGroupResInfo   groupResInfo;
-  SAggSupporter   aggSup;
   SExprInfo*      pScalarExprInfo;
   int32_t         numOfScalarExpr;  // the number of scalar expression in group operator
   SqlFunctionCtx* pScalarFuncCtx;
@@ -554,8 +566,10 @@ typedef struct SWindowRowsSup {
 } SWindowRowsSup;
 
 typedef struct SSessionAggOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo     binfo;
   SAggSupporter      aggSup;
+
   SGroupResInfo      groupResInfo;
   SWindowRowsSup     winSup;
   bool               reptScan;        // next round scan
@@ -568,6 +582,7 @@ typedef struct SResultWindowInfo {
   SResultRowPosition pos;
   STimeWindow win;
   bool isOutput;
+  bool isClosed;
 } SResultWindowInfo;
 
 typedef struct SStreamSessionAggOperatorInfo {
@@ -593,8 +608,10 @@ typedef struct STimeSliceOperatorInfo {
 } STimeSliceOperatorInfo;
 
 typedef struct SStateWindowOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo     binfo;
   SAggSupporter      aggSup;
+
   SGroupResInfo      groupResInfo;
   SWindowRowsSup     winSup;
   SColumn            stateCol;  // start row index
@@ -606,8 +623,10 @@ typedef struct SStateWindowOperatorInfo {
 } SStateWindowOperatorInfo;
 
 typedef struct SSortedMergeOperatorInfo {
-
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo   binfo;
+  SAggSupporter    aggSup;
+
   SArray*          pSortInfo;
   int32_t          numOfSources;
   SSortHandle     *pSortHandle;
@@ -619,7 +638,6 @@ typedef struct SSortedMergeOperatorInfo {
   int32_t          numOfResPerPage;
   char**           groupVal;
   SArray          *groupInfo;
-  SAggSupporter    aggSup;
 } SSortedMergeOperatorInfo;
 
 typedef struct SSortOperatorInfo {
@@ -737,10 +755,11 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
                                        SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition,
                                        SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataReader, SReadHandle* pHandle,
+
-                                            uint64_t uid, SSDataBlock* pResBlock, SArray* pColList,
+SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle,
-                                            SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pCondition,
+    SArray* pTableIdList, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo,
-                                            SOperatorInfo* pOperatorDumy);
+    STimeWindowAggSupp* pTwSup, int16_t tsColId);
+
 
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols,
                                       SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* fillVal,
@@ -780,24 +799,37 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo);
 void    doDestroyTask(SExecTaskInfo* pTaskInfo);
 int32_t getMaximumIdleDurationSec();
 
+/*
+ * ops:     root operator
+ * data:    *data save the result of encode, need to be freed by caller
+ * length:  *length save the length of *data
+ * return:  result code, 0 means success
+ */
+int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length);
+
+/*
+ * ops:    root operator, created by caller
+ * data:   save the result of decode
+ * length: the length of data
+ * return: result code, 0 means success
+ */
+int32_t decodeOperator(SOperatorInfo* ops, char* data, int32_t length);
+
 void    setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
 int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
                                EOPTR_EXEC_MODEL model);
 int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo** pRes, int32_t* capacity,
                                    int32_t* resNum);
 
-bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasicInfo* pInfo, char* result,
+int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result, int32_t length);
-                        int32_t length);
+int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length);
-void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasicInfo* pInfo, char** result,
+
-                        int32_t* length);
 STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts,
                                        SInterval* pInterval, int32_t precision, STimeWindow* win);
 int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn,
     int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item,
     int32_t order);
 int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
-int32_t initCacheSupporter(SCatchSupporter* pCatchSup, size_t rowSize, const char* pKey,
-    const char* pDir);
 int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey);
 SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize);
 SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap,

source/libs/executor/src/executil.c

@@ -233,7 +233,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int
 
 void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) {
   if (pGroupResInfo->pRows != NULL) {
-    taosArrayDestroy(pGroupResInfo->pRows);
+    taosArrayDestroyP(pGroupResInfo->pRows, taosMemoryFree);
   }
 
   pGroupResInfo->pRows = pArrayList;

source/libs/executor/src/executorimpl.c

@@ -28,13 +28,13 @@
 #include "ttime.h"
 
 #include "executorimpl.h"
+#include "index.h"
 #include "query.h"
 #include "tcompare.h"
 #include "tcompression.h"
 #include "thash.h"
 #include "ttypes.h"
 #include "vnode.h"
-#include "index.h"
 
 #define IS_MAIN_SCAN(runtime)          ((runtime)->scanFlag == MAIN_SCAN)
 #define IS_REVERSE_SCAN(runtime)       ((runtime)->scanFlag == REVERSE_SCAN)
@@ -2601,6 +2601,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
         pStart += sizeof(int32_t) * numOfRows;
 
         if (colLen[i] > 0) {
+          taosMemoryFreeClear(pColInfoData->pData);
           pColInfoData->pData = taosMemoryMalloc(colLen[i]);
         }
       } else {
@@ -2758,6 +2759,7 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
                pExchangeInfo->loadInfo.totalRows);
         pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
         completed += 1;
+        taosMemoryFreeClear(pDataInfo->pRsp);
         continue;
       }
 
@@ -2765,6 +2767,7 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
       code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data,
                                        pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL);
       if (code != 0) {
+        taosMemoryFreeClear(pDataInfo->pRsp);      
         goto _error;
       }
 
@@ -2785,10 +2788,12 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
         pDataInfo->status = EX_SOURCE_DATA_NOT_READY;
         code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i);
         if (code != TSDB_CODE_SUCCESS) {
+          taosMemoryFreeClear(pDataInfo->pRsp);        
           goto _error;
         }
       }
 
+      taosMemoryFreeClear(pDataInfo->pRsp);
       return pExchangeInfo->pResult;
     }
 
@@ -2891,6 +2896,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
 
       pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
       pExchangeInfo->current += 1;      
+      taosMemoryFreeClear(pDataInfo->pRsp);
       continue;
     }
 
@@ -2916,6 +2922,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     }
 
     pOperator->resultInfo.totalRows += pRes->info.rows;
+    taosMemoryFreeClear(pDataInfo->pRsp);    
     return pExchangeInfo->pResult;
   }
 }
@@ -3491,17 +3498,24 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
   return (rows == 0) ? NULL : pInfo->pRes;
 }
 
-void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasicInfo* pInfo, char** result,
+int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length) {
-                        int32_t* length) {
+  if(result == NULL || length == NULL){
+    return TSDB_CODE_TSC_INVALID_INPUT;
+  }
+  SOptrBasicInfo* pInfo = (SOptrBasicInfo*)(pOperator->info);
+  SAggSupporter* pSup = (SAggSupporter*)POINTER_SHIFT(pOperator->info, sizeof(SOptrBasicInfo));
   int32_t size = taosHashGetSize(pSup->pResultRowHashTable);
   size_t  keyLen = sizeof(uint64_t) * 2;  // estimate the key length
-  int32_t totalSize = sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize);
+  int32_t totalSize = sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize);
-  *result = taosMemoryCalloc(1, totalSize);
+
+  *result = (char*)taosMemoryCalloc(1, totalSize);
   if (*result == NULL) {
-    longjmp(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
+    return TSDB_CODE_OUT_OF_MEMORY;
   }
-  *(int32_t*)(*result) = size;
+
   int32_t offset = sizeof(int32_t);
+  *(int32_t*)(*result + offset) = size;
+  offset += sizeof(int32_t);
 
   // prepare memory
   SResultRowPosition* pos = &pInfo->resultRowInfo.cur;
@@ -3523,12 +3537,11 @@ void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi
     // recalculate the result size
     int32_t realTotalSize = offset + sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize;
     if (realTotalSize > totalSize) {
-      char* tmp = taosMemoryRealloc(*result, realTotalSize);
+      char* tmp = (char*)taosMemoryRealloc(*result, realTotalSize);
       if (tmp == NULL) {
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
         taosMemoryFree(*result);
         *result = NULL;
-        longjmp(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
+        return TSDB_CODE_OUT_OF_MEMORY;
       } else {
         *result = tmp;
       }
@@ -3548,17 +3561,18 @@ void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi
     pIter = taosHashIterate(pSup->pResultRowHashTable, pIter);
   }
 
-  if (length) {
+  *(int32_t*)(*result) = offset;
   *length = offset;
-  }
+
-  return;
+  return TDB_CODE_SUCCESS;
 }
 
-bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasicInfo* pInfo, char* result,
+int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result, int32_t length) {
-                        int32_t length) {
+  if(result == NULL || length <= 0){
-  if (!result || length <= 0) {
+    return TSDB_CODE_TSC_INVALID_INPUT;
-    return false;
   }
+  SOptrBasicInfo* pInfo = (SOptrBasicInfo*)(pOperator->info);
+  SAggSupporter* pSup = (SAggSupporter*)POINTER_SHIFT(pOperator->info, sizeof(SOptrBasicInfo));
 
   //  int32_t size = taosHashGetSize(pSup->pResultRowHashTable);
   int32_t count = *(int32_t*)(result);
@@ -3571,7 +3585,7 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi
     uint64_t    tableGroupId = *(uint64_t*)(result + offset);
     SResultRow* resultRow = getNewResultRow_rv(pSup->pResultBuf, tableGroupId, pSup->resultRowSize);
     if (!resultRow) {
-      longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_INVALID_INPUT);
+      return TSDB_CODE_TSC_INVALID_INPUT;
     }
 
     // add a new result set for a new group
@@ -3581,7 +3595,7 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi
     offset += keyLen;
     int32_t valueLen = *(int32_t*)(result + offset);
     if (valueLen != pSup->resultRowSize) {
-      longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_INVALID_INPUT);
+      return TSDB_CODE_TSC_INVALID_INPUT;
     }
     offset += sizeof(int32_t);
     int32_t pageId = resultRow->pageId;
@@ -3600,9 +3614,9 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi
   }
 
   if (offset != length) {
-    longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_INVALID_INPUT);
+    return TSDB_CODE_TSC_INVALID_INPUT;
   }
-  return true;
+  return TDB_CODE_SUCCESS;
 }
 
 enum {
@@ -4434,9 +4448,11 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT
 }
 
 static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                      STableListInfo* pTableGroupInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond);
+                                      STableListInfo* pTableGroupInfo, uint64_t queryId, uint64_t taskId,
+                                      SNode* pTagCond);
 
-static int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STableListInfo* pListInfo, SNode* pTagCond);
+static int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STableListInfo* pListInfo,
+                            SNode* pTagCond);
 static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo);
 static SArray* extractColumnInfo(SNodeList* pNodeList);
 
@@ -4473,7 +4489,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
 
-      tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond);
+      tsdbReaderT pDataReader =
+          doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond);
       if (pDataReader == NULL && terrno != 0) {
         return NULL;
       }
@@ -4492,9 +4509,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
       SScanPhysiNode*      pScanPhyNode = (SScanPhysiNode*)pPhyNode;  // simple child table.
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
-
+      STimeWindowAggSupp twSup = {.waterMark = pTableScanNode->watermark,
-      int32_t numOfCols = 0;
+          .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
-
       tsdbReaderT pDataReader = NULL;
       if (pHandle->vnode) {
         pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond);
@@ -4503,24 +4519,15 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       }
 
       if (pDataReader == NULL && terrno != 0) {
-        qDebug("pDataReader is     NULL");
+        qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo));
         // return NULL;
       } else {
-        qDebug("pDataReader is not NULL");
+        qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo));
       }
-
-      SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
-      SOperatorInfo*      pOperatorDumy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo);
-
       SArray* tableIdList = extractTableIdList(pTableListInfo);
+      SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle,
+          tableIdList, pTableScanNode, pTaskInfo, &twSup, pTableScanNode->tsColId);
 
-      SSDataBlock* pResBlock = createResDataBlock(pDescNode);
-      SArray*      pCols =
-          extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, pTaskInfo, COL_MATCH_FROM_COL_ID);
-
-      SOperatorInfo* pOperator =
-          createStreamScanOperatorInfo(pHandle->reader, pDataReader, pHandle, pScanPhyNode->uid, pResBlock, pCols,
-                                       tableIdList, pTaskInfo, pScanPhyNode->node.pConditions, pOperatorDumy);
       taosArrayDestroy(tableIdList);
       return pOperator;
     } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
@@ -4602,8 +4609,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       pOptr = createGroupOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions,
                                       pScalarExprInfo, numOfScalarExpr, pTaskInfo);
     } else {
-      pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pScalarExprInfo, numOfScalarExpr,
+      pOptr =
-                                          pTaskInfo);
+          createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pScalarExprInfo, numOfScalarExpr, pTaskInfo);
     }
   } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
     SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
@@ -4619,7 +4626,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
                           .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision};
 
     STimeWindowAggSupp as = {.waterMark = pIntervalPhyNode->window.watermark,
-                             .calTrigger = pIntervalPhyNode->window.triggerType};
+                             .calTrigger = pIntervalPhyNode->window.triggerType,
+                             .maxTs = INT64_MIN};
 
     int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
     pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo);
@@ -4751,11 +4759,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
       TSWAP(pCond->twindows[i].skey, pCond->twindows[i].ekey);
     }
   }
-  taosqsort(pCond->twindows,
+  taosqsort(pCond->twindows, pCond->numOfTWindows, sizeof(STimeWindow), pCond, compareTimeWindow);
-            pCond->numOfTWindows,
-            sizeof(STimeWindow),
-            pCond,
-            compareTimeWindow);
 
   pCond->type = BLOCK_LOAD_OFFSET_SEQ_ORDER;
   //  pCond->type = pTableScanNode->scanFlag;
@@ -4915,20 +4919,24 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
   return pList;
 }
 
-int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid,
+int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STableListInfo* pListInfo,
-                           STableListInfo* pListInfo, SNode* pTagCond) {
+                     SNode* pTagCond) {
   int32_t code = TSDB_CODE_SUCCESS;
   pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo));
 
   if (tableType == TSDB_SUPER_TABLE) {
     if (pTagCond) {
+      SIndexMetaArg metaArg = {.metaHandle = tsdbGetIdx(metaHandle), .suid = tableUid};
+
       SArray* res = taosArrayInit(8, sizeof(uint64_t));
-      code = doFilterTag(pTagCond, res);
+      code = doFilterTag(pTagCond, &metaArg, res);
       if (code != TSDB_CODE_SUCCESS) {
-        qError("doFilterTag error:%d", code);
+        qError("failed  to  get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid);
         taosArrayDestroy(res);
         terrno = code;
         return code;
+      } else {
+        qDebug("sucess to  get tableIds, size: %d, suid: %" PRIu64 "", (int)taosArrayGetSize(res), tableUid);
       }
       for (int i = 0; i < taosArrayGetSize(res); i++) {
         STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)};
@@ -4960,7 +4968,8 @@ SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) {
 
 tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) {
-  int32_t  code = getTableList(pHandle->meta, pTableScanNode->scan.tableType, pTableScanNode->scan.uid, pTableListInfo, pTagCond);
+  int32_t code =
+      getTableList(pHandle->meta, pTableScanNode->scan.tableType, pTableScanNode->scan.uid, pTableListInfo, pTagCond);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -4984,6 +4993,91 @@ _error:
   return NULL;
 }
 
+int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t *length){
+  int32_t code = TDB_CODE_SUCCESS;
+  char *pCurrent = NULL;
+  int32_t currLength = 0;
+  if(ops->fpSet.encodeResultRow){
+    if(result == NULL || length == NULL){
+      return TSDB_CODE_TSC_INVALID_INPUT;
+    }
+    code = ops->fpSet.encodeResultRow(ops, &pCurrent, &currLength);
+
+    if(code != TDB_CODE_SUCCESS){
+      if(*result != NULL){
+        taosMemoryFree(*result);
+        *result = NULL;
+      }
+      return code;
+    }
+
+    if(*result == NULL){
+      *result = (char*)taosMemoryCalloc(1, currLength + sizeof(int32_t));
+      if (*result == NULL) {
+        taosMemoryFree(pCurrent);
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      memcpy(*result + sizeof(int32_t), pCurrent, currLength);
+      *(int32_t*)(*result) = currLength + sizeof(int32_t);
+    }else{
+      int32_t sizePre = *(int32_t*)(*result);
+      char* tmp = (char*)taosMemoryRealloc(*result, sizePre + currLength);
+      if (tmp == NULL) {
+        taosMemoryFree(pCurrent);
+        taosMemoryFree(*result);
+        *result = NULL;
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      *result = tmp;
+      memcpy(*result + sizePre, pCurrent, currLength);
+      *(int32_t*)(*result) += currLength;
+    }
+    taosMemoryFree(pCurrent);
+    *length = *(int32_t*)(*result);
+  }
+
+  for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
+    code = encodeOperator(ops->pDownstream[i], result, length);
+    if(code != TDB_CODE_SUCCESS){
+      return code;
+    }
+  }
+  return TDB_CODE_SUCCESS;
+}
+
+int32_t decodeOperator(SOperatorInfo* ops, char* result, int32_t length){
+  int32_t code = TDB_CODE_SUCCESS;
+  if(ops->fpSet.decodeResultRow){
+    if(result == NULL || length <= 0){
+      return TSDB_CODE_TSC_INVALID_INPUT;
+    }
+    char* data = result + 2 * sizeof(int32_t);
+    int32_t dataLength = *(int32_t*)(result + sizeof(int32_t));
+    code = ops->fpSet.decodeResultRow(ops, data, dataLength - sizeof(int32_t));
+    if(code != TDB_CODE_SUCCESS){
+      return code;
+    }
+
+    int32_t totalLength = *(int32_t*)result;
+    if(totalLength == dataLength + sizeof(int32_t)) { // the last data
+      result = NULL;
+      length = 0;
+    }else{
+      result += dataLength;
+      *(int32_t*)(result) = totalLength - dataLength;
+      length = totalLength - dataLength;
+    }
+  }
+
+  for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
+    code = decodeOperator(ops->pDownstream[i], result, length);
+    if(code != TDB_CODE_SUCCESS){
+      return code;
+    }
+  }
+  return TDB_CODE_SUCCESS;
+}
+
 int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
                                EOPTR_EXEC_MODEL model) {
   uint64_t queryId = pPlan->id.queryId;
@@ -4995,8 +5089,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
     goto _complete;
   }
 
-  (*pTaskInfo)->pRoot =
+  (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
-      createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond);
+                                           &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond);
   if (NULL == (*pTaskInfo)->pRoot) {
     code = terrno;
     goto _complete;
@@ -5181,20 +5275,6 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t initCacheSupporter(SCatchSupporter* pCatchSup, size_t rowSize, const char* pKey, const char* pDir) {
-  pCatchSup->keySize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY);
-  pCatchSup->pKeyBuf = taosMemoryCalloc(1, pCatchSup->keySize);
-  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
-  pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK);
-  if (pCatchSup->pKeyBuf == NULL || pCatchSup->pWindowHashTable == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  int32_t pageSize = rowSize * 32;
-  int32_t bufSize = pageSize * 4096;
-  return createDiskbasedBuf(&pCatchSup->pDataBuf, pageSize, bufSize, pKey, pDir);
-}
-
 int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey) {
   pSup->keySize = sizeof(int64_t) + sizeof(TSKEY);
   pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize);
@@ -5212,5 +5292,5 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey) {
   if (bufSize <= pageSize) {
     bufSize = pageSize * 4;
   }
-  return createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, "/tmp/");
+  return createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, TD_TMP_DIR_PATH);
 }

source/libs/executor/src/scanoperator.c

@@ -158,7 +158,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
   return false;
 }
 
-static void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock);
+static void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock);
 
 static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
                              uint32_t* status) {
@@ -250,7 +250,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
 
   // currently only the tbname pseudo column
   if (pTableScanInfo->numOfPseudoExpr > 0) {
-    addTagPseudoColumnData(pTableScanInfo, pBlock);
+    addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr, pTableScanInfo->numOfPseudoExpr, pBlock);
   }
 
   int64_t st = taosGetTimestampMs();
@@ -287,18 +287,18 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction
             compareTimeWindow);
 }
 
-void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) {
+void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock) {
   // currently only the tbname pseudo column
-  if (pTableScanInfo->numOfPseudoExpr == 0) {
+  if (numOfPseudoExpr == 0) {
     return;
   }
 
   SMetaReader mr = {0};
-  metaReaderInit(&mr, pTableScanInfo->readHandle.meta, 0);
+  metaReaderInit(&mr, pHandle->meta, 0);
   metaGetTableEntryByUid(&mr, pBlock->info.uid);
 
-  for (int32_t j = 0; j < pTableScanInfo->numOfPseudoExpr; ++j) {
+  for (int32_t j = 0; j < numOfPseudoExpr; ++j) {
-    SExprInfo* pExpr = &pTableScanInfo->pPseudoExpr[j];
+    SExprInfo* pExpr = &pPseudoExpr[j];
 
     int32_t dstSlotId = pExpr->base.resSchema.slotId;
 
@@ -309,7 +309,7 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock)
 
     // this is to handle the tbname
     if (fmIsScanPseudoColumnFunc(functionId)) {
-      setTbNameColData(pTableScanInfo->readHandle.meta, pBlock, pColInfoData, functionId);
+      setTbNameColData(pHandle->meta, pBlock, pColInfoData, functionId);
     } else {  // these are tags
       const char* p = NULL;
       if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
@@ -755,91 +755,6 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool inverti
   return NULL;
 }
 
-void static setSupKeyBuf(SCatchSupporter* pSup, int64_t groupId, int64_t childId, TSKEY ts) {
-  int64_t* pKey = (int64_t*)pSup->pKeyBuf;
-  pKey[0] = groupId;
-  pKey[1] = childId;
-  pKey[2] = ts;
-}
-
-static int32_t catchWidonwInfo(SSDataBlock* pDataBlock, SCatchSupporter* pSup, int32_t pageId, int32_t tsIndex,
-                               int64_t childId) {
-  SColumnInfoData* pColDataInfo = taosArrayGet(pDataBlock->pDataBlock, tsIndex);
-  TSKEY*           tsCols = (int64_t*)pColDataInfo->pData;
-  for (int32_t i = 0; i < pDataBlock->info.rows; i++) {
-    setSupKeyBuf(pSup, pDataBlock->info.groupId, childId, tsCols[i]);
-    SWindowPosition* p1 = (SWindowPosition*)taosHashGet(pSup->pWindowHashTable, pSup->pKeyBuf, pSup->keySize);
-    if (p1 == NULL) {
-      SWindowPosition pos = {.pageId = pageId, .rowId = i};
-      int32_t code = taosHashPut(pSup->pWindowHashTable, pSup->pKeyBuf, pSup->keySize, &pos, sizeof(SWindowPosition));
-      if (code != TSDB_CODE_SUCCESS) {
-        return code;
-      }
-    } else {
-      p1->pageId = pageId;
-      p1->rowId = i;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t catchDatablock(SSDataBlock* pDataBlock, SCatchSupporter* pSup, int32_t tsIndex, int64_t childId) {
-  int32_t start = 0;
-  int32_t stop = 0;
-  int32_t pageSize = getBufPageSize(pSup->pDataBuf);
-  while (start < pDataBlock->info.rows) {
-    blockDataSplitRows(pDataBlock, pDataBlock->info.hasVarCol, start, &stop, pageSize);
-    SSDataBlock* pDB = blockDataExtractBlock(pDataBlock, start, stop - start + 1);
-    if (pDB == NULL) {
-      return terrno;
-    }
-    int32_t pageId = -1;
-    void*   pPage = getNewBufPage(pSup->pDataBuf, pDataBlock->info.groupId, &pageId);
-    if (pPage == NULL) {
-      blockDataDestroy(pDB);
-      return terrno;
-    }
-    int32_t size = blockDataGetSize(pDB) + sizeof(int32_t) + pDB->info.numOfCols * sizeof(int32_t);
-    assert(size <= pageSize);
-    blockDataToBuf(pPage, pDB);
-    setBufPageDirty(pPage, true);
-    releaseBufPage(pSup->pDataBuf, pPage);
-    blockDataDestroy(pDB);
-    start = stop + 1;
-    int32_t code = catchWidonwInfo(pDB, pSup, pageId, tsIndex, childId);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static SSDataBlock* getDataFromCatch(SStreamBlockScanInfo* pInfo) {
-  SSDataBlock* pBlock = pInfo->pUpdateRes;
-  if (pInfo->updateResIndex < pBlock->info.rows) {
-    blockDataCleanup(pInfo->pRes);
-    SCatchSupporter* pCSup = &pInfo->childAggSup;
-    SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, 0);
-    TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
-    int32_t          size = taosArrayGetSize(pInfo->childIds);
-    for (int32_t i = 0; i < size; i++) {
-      int64_t id = *(int64_t*)taosArrayGet(pInfo->childIds, i);
-      setSupKeyBuf(pCSup, pBlock->info.groupId, id, tsCols[pInfo->updateResIndex]);
-      SWindowPosition* pos = (SWindowPosition*)taosHashGet(pCSup->pWindowHashTable, pCSup->pKeyBuf, pCSup->keySize);
-      void*            buf = getBufPage(pCSup->pDataBuf, pos->pageId);
-      SSDataBlock*     pDB = createOneDataBlock(pInfo->pRes, false);
-      blockDataFromBuf(pDB, buf);
-      SSDataBlock* pSub = blockDataExtractBlock(pDB, pos->rowId, 1);
-      blockDataMerge(pInfo->pRes, pSub);
-      blockDataDestroy(pDB);
-      blockDataDestroy(pSub);
-    }
-    pInfo->updateResIndex++;
-    return pInfo->pRes;
-  }
-  return NULL;
-}
-
 static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
   // NOTE: this operator does never check if current status is done or not
   SExecTaskInfo*        pTaskInfo = pOperator->pTaskInfo;
@@ -910,8 +825,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
         pInfo->pRes->info.groupId = groupId;
       }
 
-      int32_t numOfCols = pInfo->pRes->info.numOfCols;
+      for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
-      for (int32_t i = 0; i < numOfCols; ++i) {
         SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
         if (!pColMatchInfo->output) {
           continue;
@@ -941,10 +855,16 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
         pTaskInfo->code = terrno;
         return NULL;
       }
+
       rows = pBlockInfo->rows;
+
+      // currently only the tbname pseudo column
+      if (pInfo->numOfPseudoExpr > 0) {
+        addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes);
+      }
+
       doFilter(pInfo->pCondition, pInfo->pRes, NULL);
       blockDataUpdateTsWindow(pInfo->pRes, 0);
-
       break;
     }
 
@@ -972,10 +892,9 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
   }
 }
 
-SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataReader, SReadHandle* pHandle,
+SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle,
-                                            uint64_t uid, SSDataBlock* pResBlock, SArray* pColList,
+    SArray* pTableIdList, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo,
-                                            SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pCondition,
+    STimeWindowAggSupp* pTwSup, int16_t tsColId) {
-                                            SOperatorInfo* pOperatorDumy) {
   SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
   SOperatorInfo*        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -983,22 +902,28 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR
     goto _error;
   }
 
-  STableScanInfo* pSTInfo = (STableScanInfo*)pOperatorDumy->info;
+  SScanPhysiNode* pScanPhyNode = &pTableScanNode->scan;
 
-  int32_t numOfOutput = taosArrayGetSize(pColList);
+  SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
+  SOperatorInfo*      pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo);
 
-  SArray* pColIds = taosArrayInit(4, sizeof(int16_t));
+  STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info;
+
+  int32_t numOfCols = 0;
+  pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, pTaskInfo, COL_MATCH_FROM_COL_ID);
+
+  int32_t numOfOutput = taosArrayGetSize(pInfo->pColMatchInfo);
+  SArray* pColIds = taosArrayInit(numOfOutput, sizeof(int16_t));
   for (int32_t i = 0; i < numOfOutput; ++i) {
-    SColMatchInfo* id = taosArrayGet(pColList, i);
+    SColMatchInfo* id = taosArrayGet(pInfo->pColMatchInfo, i);
+
     int16_t colId = id->colId;
     taosArrayPush(pColIds, &colId);
   }
 
-  pInfo->pColMatchInfo = pColList;
-
   // set the extract column id to streamHandle
-  tqReadHandleSetColIdList((STqReadHandle*)streamReadHandle, pColIds);
+  tqReadHandleSetColIdList((STqReadHandle*)pHandle->reader, pColIds);
-  int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList);
+  int32_t code = tqReadHandleSetTbUidList(pHandle->reader, pTableIdList);
   if (code != 0) {
     goto _error;
   }
@@ -1014,37 +939,39 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR
     goto _error;
   }
 
-  pInfo->primaryTsIndex = 0;  // TODO(liuyao) get it from physical plan
+  pInfo->primaryTsIndex = tsColId;
   if (pSTInfo->interval.interval > 0) {
-    pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, 10000);  // TODO(liuyao) get watermark from physical plan
+    pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark);
   } else {
     pInfo->pUpdateInfo = NULL;
   }
 
+  // create the pseduo columns info
+  if (pTableScanNode->scan.pScanPseudoCols != NULL) {
+    pInfo->pPseudoExpr = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pInfo->numOfPseudoExpr);
+  }
+
   pInfo->readHandle        = *pHandle;
-  pInfo->tableUid = uid;
+  pInfo->tableUid          = pScanPhyNode->uid;
-  pInfo->streamBlockReader = streamReadHandle;
+  pInfo->streamBlockReader = pHandle->reader;
-  pInfo->pRes = pResBlock;
+  pInfo->pRes              = createResDataBlock(pDescNode);
-  pInfo->pCondition = pCondition;
+  pInfo->pCondition        = pScanPhyNode->node.pConditions;
   pInfo->pDataReader       = pDataReader;
   pInfo->scanMode          = STREAM_SCAN_FROM_READERHANDLE;
-  pInfo->pOperatorDumy = pOperatorDumy;
+  pInfo->pOperatorDumy     = pTableScanDummy;
   pInfo->interval          = pSTInfo->interval;
   pInfo->sessionSup        = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
 
-  initCacheSupporter(&pInfo->childAggSup, 1024, "StreamFinalInterval",
-                     "/tmp/");  // TODO(liuyao) get row size from phy plan
-
   pOperator->name         = "StreamBlockScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
   pOperator->blocking     = false;
   pOperator->status       = OP_NOT_OPENED;
   pOperator->info         = pInfo;
-  pOperator->numOfExprs = pResBlock->info.numOfCols;
+  pOperator->numOfExprs   = pInfo->pRes->info.numOfCols;
   pOperator->pTaskInfo    = pTaskInfo;
 
-  pOperator->fpSet =
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL,
-      createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL);
+      NULL, operatorDummyCloseFn, NULL, NULL, NULL);
 
   return pOperator;
 

source/libs/executor/src/timewindowoperator.c

@@ -622,18 +622,103 @@ static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowInd
   }
 }
 
-static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
+typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
-                               uint64_t tableGroupId) {
+
+int32_t binarySearch(void* keyList, int num, TSKEY key, int order,
+    __get_value_fn_t getValuefn) {
+  int    firstPos = 0, lastPos = num - 1, midPos = -1;
+  int    numOfRows = 0;
+
+  if (num <= 0) return -1;
+  if (order == TSDB_ORDER_DESC) {
+    // find the first position which is smaller or equal than the key
+    while (1) {
+      if (key >= getValuefn(keyList, lastPos)) return lastPos;
+      if (key == getValuefn(keyList, firstPos)) return firstPos;
+      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;
+
+      numOfRows = lastPos - firstPos + 1;
+      midPos = (numOfRows >> 1) + firstPos;
+
+      if (key < getValuefn(keyList, midPos)) {
+        lastPos = midPos - 1;
+      } else if (key > getValuefn(keyList, midPos)) {
+        firstPos = midPos + 1;
+      } else {
+        break;
+      }
+    }
+
+  } else {
+    // find the first position which is bigger or equal than the key
+    while (1) {
+      if (key <= getValuefn(keyList, firstPos)) return firstPos;
+      if (key == getValuefn(keyList, lastPos)) return lastPos;
+
+      if (key > getValuefn(keyList, lastPos)) {
+        lastPos = lastPos + 1;
+        if (lastPos >= num)
+          return -1;
+        else
+          return lastPos;
+      }
+
+      numOfRows = lastPos - firstPos + 1;
+      midPos = (numOfRows >> 1) + firstPos;
+
+      if (key < getValuefn(keyList, midPos)) {
+        lastPos = midPos - 1;
+      } else if (key > getValuefn(keyList, midPos)) {
+        firstPos = midPos + 1;
+      } else {
+        break;
+      }
+    }
+  }
+
+  return midPos;
+}
+
+int64_t getReskey(void* data, int32_t index) {
+  SArray* res = (SArray*) data;
+  SResKeyPos* pos = taosArrayGetP(res, index);
+  return *(int64_t*)pos->key;
+}
+
+static int32_t saveResult(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
+  int32_t size = taosArrayGetSize(pUpdated);
+  int32_t index = binarySearch(pUpdated, size, result->win.skey, TSDB_ORDER_DESC, getReskey);
+  if (index == -1) {
+    index = 0;
+  } else {
+    TSKEY resTs = getReskey(pUpdated, index);
+    if (resTs < result->win.skey) {
+      index++;
+    } else {
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+  
+  SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
+  if (newPos == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  newPos->groupId = groupId;
+  newPos->pos = (SResultRowPosition){.pageId = result->pageId, .offset = result->offset};
+  *(int64_t*)newPos->key = result->win.skey;
+  if (taosArrayInsert(pUpdated, index, &newPos) == NULL ){
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
+                               uint64_t tableGroupId, SArray* pUpdated) {
   SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
 
   SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
   int32_t        numOfOutput = pOperatorInfo->numOfExprs;
 
-  SArray* pUpdated = NULL;
-  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
-    pUpdated = taosArrayInit(4, POINTER_BYTES);
-  }
-
   int32_t step = 1;
   bool    ascScan = (pInfo->order == TSDB_ORDER_ASC);
 
@@ -663,13 +748,10 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
     longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
-  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
+  if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM &&
-    SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
+        (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||
-    pos->groupId = tableGroupId;
+         pInfo->twAggSup.calTrigger == 0) ) {
-    pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
+    saveResult(pResult, tableGroupId, pUpdated);
-    *(int64_t*)pos->key = pResult->win.skey;
-
-    taosArrayPush(pUpdated, &pos);
   }
 
   int32_t forwardStep = 0;
@@ -742,13 +824,10 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
       longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
     }
 
-    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
+    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM &&
-      SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
+          (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||
-      pos->groupId = tableGroupId;
+           pInfo->twAggSup.calTrigger == 0) ) {
-      pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
+      saveResult(pResult, tableGroupId, pUpdated);
-      *(int64_t*)pos->key = pResult->win.skey;
-
-      taosArrayPush(pUpdated, &pos);
     }
 
     ekey = ascScan? nextWin.ekey:nextWin.skey;
@@ -769,7 +848,6 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
     saveDataBlockLastRow(pInfo->pRow, pBlock->pDataBlock, rowIndex, pBlock->info.numOfCols);
   }
 
-  return pUpdated;
   //  updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false);
 }
 
@@ -799,7 +877,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
     STableQueryInfo* pTableQueryInfo = pInfo->pCurrent;
 
     setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window);
-    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId);
+    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId, NULL);
 
 #if 0  // test for encode/decode result info
     if(pOperator->encodeResultRow){
@@ -1067,7 +1145,7 @@ void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData,
 }
 
 static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo,
-    SInterval* pIntrerval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock,
+    SInterval* pInterval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock,
     SArray* pUpWins) {
   SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
   TSKEY *tsCols = (TSKEY*)pColDataInfo->pData;
@@ -1075,8 +1153,8 @@ static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo,
   for (int32_t i = 0; i < pBlock->info.rows; i += step) {
     SResultRowInfo dumyInfo;
     dumyInfo.cur.pageId = -1;
-    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pIntrerval,
+    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval,
-                                        pIntrerval->precision, NULL);
+                                        pInterval->precision, NULL);
     step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i,
         win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
     doClearWindow(pSup, pBinfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput);
@@ -1086,6 +1164,39 @@ static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo,
   }
 }
 
+static int32_t closeIntervalWindow(SHashObj *pHashMap, STimeWindowAggSupp *pSup,
+    SInterval* pInterval, SArray* closeWins) {
+  void *pIte = NULL;
+  size_t keyLen = 0;
+  while((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
+    void* key = taosHashGetKey(pIte, &keyLen);
+    uint64_t groupId = *(uint64_t*) key;
+    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
+    TSKEY ts = *(uint64_t*) ((char*)key + sizeof(uint64_t));
+    SResultRowInfo dumyInfo;
+    dumyInfo.cur.pageId = -1;
+    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval,
+        pInterval->precision, NULL);
+    if (win.ekey < pSup->maxTs - pSup->waterMark) {
+      char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
+      SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
+      taosHashRemove(pHashMap, keyBuf, keyLen);
+      SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
+      if (pos == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      pos->groupId = groupId;
+      pos->pos = *(SResultRowPosition*) pIte;
+      *(int64_t*)pos->key = ts;
+      if (!taosArrayPush(closeWins, &pos)) {
+        taosMemoryFree(pos);
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   SIntervalAggOperatorInfo* pInfo = pOperator->info;
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -1106,7 +1217,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 
   SOperatorInfo* downstream = pOperator->pDownstream[0];
 
-  SArray* pUpdated = NULL;
+  SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
+  SArray* pClosed = taosArrayInit(4, POINTER_BYTES);
+
   while (1) {
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
     if (pBlock == NULL) {
@@ -1128,10 +1241,19 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
       continue;
     }
 
-    pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId);
+    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId, pUpdated);
+    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
   }
-
+  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup,
-  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset);
+      &pInfo->interval, pClosed);
+  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed,
+      pInfo->binfo.rowCellInfoOffset);
+  if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER__WINDOW_CLOSE) {
+    taosArrayAddAll(pUpdated, pClosed);
+  }
+  taosArrayDestroy(pClosed);
+  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated,
+      pInfo->binfo.rowCellInfoOffset);
 
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
@@ -1935,63 +2057,6 @@ _error:
   return NULL;
 }
 
-typedef int64_t (*__get_value_fn_t)(void* data, int32_t index);
-
-int32_t binarySearch(void* keyList, int num, TSKEY key, int order,
-    __get_value_fn_t getValuefn) {
-  int    firstPos = 0, lastPos = num - 1, midPos = -1;
-  int    numOfRows = 0;
-
-  if (num <= 0) return -1;
-  if (order == TSDB_ORDER_DESC) {
-    // find the first position which is smaller than the key
-    while (1) {
-      if (key >= getValuefn(keyList, lastPos)) return lastPos;
-      if (key == getValuefn(keyList, firstPos)) return firstPos;
-      if (key < getValuefn(keyList, firstPos)) return firstPos - 1;
-
-      numOfRows = lastPos - firstPos + 1;
-      midPos = (numOfRows >> 1) + firstPos;
-
-      if (key < getValuefn(keyList, midPos)) {
-        lastPos = midPos - 1;
-      } else if (key > getValuefn(keyList, midPos)) {
-        firstPos = midPos + 1;
-      } else {
-        break;
-      }
-    }
-
-  } else {
-    // find the first position which is bigger than the key
-    while (1) {
-      if (key <= getValuefn(keyList, firstPos)) return firstPos;
-      if (key == getValuefn(keyList, lastPos)) return lastPos;
-
-      if (key > getValuefn(keyList, lastPos)) {
-        lastPos = lastPos + 1;
-        if (lastPos >= num)
-          return -1;
-        else
-          return lastPos;
-      }
-
-      numOfRows = lastPos - firstPos + 1;
-      midPos = (numOfRows >> 1) + firstPos;
-
-      if (key < getValuefn(keyList, midPos)) {
-        lastPos = midPos - 1;
-      } else if (key > getValuefn(keyList, midPos)) {
-        firstPos = midPos + 1;
-      } else {
-        break;
-      }
-    }
-  }
-
-  return midPos;
-}
-
 int64_t getSessionWindowEndkey(void* data, int32_t index) {
   SArray* pWinInfos = (SArray*) data;
   SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index);
@@ -2223,12 +2288,14 @@ static void doStreamSessionWindowAggImpl(SOperatorInfo* pOperator,
     if (winNum > 0) {
       compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pTaskInfo, pStUpdated, pStDeleted);
     }
-
+    pCurWin->isClosed = false;
+    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
       code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &(pCurWin->win.skey), sizeof(TSKEY));
       if (code != TSDB_CODE_SUCCESS) {
         longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
       }
       pCurWin->isOutput = true;
+    }
     i += winRows;
   }
 }
@@ -2325,6 +2392,37 @@ bool isFinalSession(SStreamSessionAggOperatorInfo* pInfo) {
   return pInfo->pChildren != NULL;
 }
 
+int32_t closeSessionWindow(SArray *pWins, STimeWindowAggSupp *pTwSup, SArray *pClosed,
+    int8_t calTrigger) {
+  // Todo(liuyao) save window to tdb
+  int32_t size =  taosArrayGetSize(pWins);
+  for (int32_t i = 0; i < size; i++) {
+    SResultWindowInfo *pSeWin = taosArrayGet(pWins, i);
+    if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) {
+      if (!pSeWin->isClosed) {
+        SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
+        if (pos == NULL) {
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
+        pos->groupId = 0;
+        pos->pos = pSeWin->pos;
+        *(int64_t*)pos->key = pSeWin->win.ekey;
+        if (!taosArrayPush(pClosed, &pos)) {
+          taosMemoryFree(pos);
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
+        pSeWin->isClosed = true;
+        if (calTrigger == STREAM_TRIGGER__WINDOW_CLOSE) {
+          pSeWin->isOutput = true;
+        }
+      }
+      continue;
+    }
+    break;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static SSDataBlock* doStreamSessionWindowAgg(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -2377,13 +2475,21 @@ static SSDataBlock* doStreamSessionWindowAgg(SOperatorInfo* pOperator) {
       doStreamSessionWindowAggImpl(pOperator, pBlock, NULL, NULL);
     }
     doStreamSessionWindowAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted);
+    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
   }
-
   // restore the value
   pOperator->status = OP_RES_TO_RETURN;
+  
+  SArray* pClosed = taosArrayInit(16, POINTER_BYTES);
+  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed,
+      pInfo->twAggSup.calTrigger);
   SArray* pUpdated = taosArrayInit(16, POINTER_BYTES);
   copyUpdateResult(pStUpdated, pUpdated, pBInfo->pRes->info.groupId);
   taosHashCleanup(pStUpdated);
+  if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER__WINDOW_CLOSE) {
+    taosArrayAddAll(pUpdated, pClosed);
+  }
+
   finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
       pInfo->binfo.rowCellInfoOffset);
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);

source/libs/function/CMakeLists.txt

@@ -14,7 +14,7 @@ target_include_directories(
 
 target_link_libraries(
     function
-    PRIVATE os util common nodes scalar catalog qcom transport
+    PRIVATE os util common nodes scalar qcom transport
     PUBLIC uv_a
 )
 

source/libs/function/inc/functionMgtInt.h

@@ -45,8 +45,6 @@ extern "C" {
 #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 
 #define FUNC_UDF_ID_START 5000
-#define FUNC_AGGREGATE_UDF_ID 5001
-#define FUNC_SCALAR_UDF_ID    5002
 
 extern const int funcMgtUdfNum;
 

source/libs/function/src/builtins.c

@@ -746,7 +746,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   }
 
   uint8_t resType;
-  if (IS_SIGNED_NUMERIC_TYPE(colType)) {
+  if (IS_SIGNED_NUMERIC_TYPE(colType) || TSDB_DATA_TYPE_BOOL == colType) {
     resType = TSDB_DATA_TYPE_BIGINT;
   } else {
     resType = TSDB_DATA_TYPE_DOUBLE;

source/libs/function/src/functionMgt.c

@@ -16,7 +16,6 @@
 #include "functionMgt.h"
 
 #include "builtins.h"
-#include "catalog.h"
 #include "functionMgtInt.h"
 #include "taos.h"
 #include "taoserror.h"
@@ -65,35 +64,19 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
   return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification);
 }
 
-static int32_t getUdfInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) {
-  SFuncInfo funcInfo = {0};
-  int32_t    code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &funcInfo);
-  if (TSDB_CODE_SUCCESS != code) {
-    return code;
-  }
-
-  pFunc->funcType = FUNCTION_TYPE_UDF;
-  pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == funcInfo.funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID;
-  pFunc->node.resType.type = funcInfo.outputType;
-  pFunc->node.resType.bytes = funcInfo.outputLen;
-  pFunc->udfBufSize = funcInfo.bufSize;
-  tFreeSFuncInfo(&funcInfo);
-  return TSDB_CODE_SUCCESS;
-}
-
 int32_t fmFuncMgtInit() {
   taosThreadOnce(&functionHashTableInit, doInitFunctionTable);
   return initFunctionCode;
 }
 
-int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) {
+int32_t fmGetFuncInfo(SFunctionNode* pFunc, char* pMsg, int32_t msgLen) {
   void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFunc->functionName, strlen(pFunc->functionName));
   if (NULL != pVal) {
     pFunc->funcId = *(int32_t*)pVal;
     pFunc->funcType = funcMgtBuiltins[pFunc->funcId].type;
-    return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, pParam->pErrBuf, pParam->errBufLen);
+    return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, pMsg, msgLen);
   }
-  return getUdfInfo(pParam, pFunc);
+  return TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION;
 }
 
 bool fmIsBuiltinFunc(const char* pFunc) {

source/libs/index/inc/indexCache.h

@@ -74,7 +74,7 @@ void     indexCacheIteratorDestroy(Iterate* iiter);
 int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid);
 
 // int indexCacheGet(void *cache, uint64_t *rst);
-int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* tr, STermValueType* s);
+int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s);
 
 void indexCacheRef(IndexCache* cache);
 void indexCacheUnRef(IndexCache* cache);

source/libs/index/inc/indexTfile.h

@@ -105,7 +105,7 @@ TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName);
 TFileReader* tfileReaderOpen(char* path, uint64_t suid, int64_t version, const char* colName);
 TFileReader* tfileReaderCreate(WriterCtx* ctx);
 void         tfileReaderDestroy(TFileReader* reader);
-int          tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr);
+int          tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr);
 void         tfileReaderRef(TFileReader* reader);
 void         tfileReaderUnRef(TFileReader* reader);
 
@@ -120,7 +120,7 @@ int          tfileWriterFinish(TFileWriter* tw);
 IndexTFile* indexTFileCreate(const char* path);
 void        indexTFileDestroy(IndexTFile* tfile);
 int         indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid);
-int         indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTempResult* tr);
+int         indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* tr);
 
 Iterate* tfileIteratorCreate(TFileReader* reader);
 void     tfileIteratorDestroy(Iterate* iterator);

source/libs/index/inc/indexUtil.h

@@ -66,7 +66,7 @@ extern "C" {
  *        [1, 4, 5]
  * output:[4, 5]
  */
-void iIntersection(SArray *interResults, SArray *finalResult);
+void iIntersection(SArray *in, SArray *out);
 
 /* multi sorted result union
  * input: [1, 2, 4, 5]
@@ -74,7 +74,7 @@ void iIntersection(SArray *interResults, SArray *finalResult);
  *        [1, 4, 5]
  * output:[1, 2, 3, 4, 5]
  */
-void iUnion(SArray *interResults, SArray *finalResult);
+void iUnion(SArray *in, SArray *out);
 
 /* see example
  * total:   [1, 2, 4, 5, 7, 8]
@@ -92,19 +92,24 @@ typedef struct {
   uint64_t data;
 } SIdxVerdata;
 
+/*
+ * index temp result
+ *
+ */
 typedef struct {
   SArray *total;
-  SArray *added;
+  SArray *add;
-  SArray *deled;
+  SArray *del;
-} SIdxTempResult;
+} SIdxTRslt;
 
-SIdxTempResult *sIdxTempResultCreate();
+SIdxTRslt *idxTRsltCreate();
 
-void sIdxTempResultClear(SIdxTempResult *tr);
+void idxTRsltClear(SIdxTRslt *tr);
 
-void sIdxTempResultDestroy(SIdxTempResult *tr);
+void idxTRsltDestroy(SIdxTRslt *tr);
+
+void idxTRsltMergeTo(SIdxTRslt *tr, SArray *out);
 
-void sIdxTempResultMergeTo(SArray *result, SIdxTempResult *tr);
 #ifdef __cplusplus
 }
 #endif

source/libs/index/src/index.c

@@ -29,7 +29,7 @@
 #include "lucene++/Lucene_c.h"
 #endif
 
-#define INDEX_NUM_OF_THREADS 1
+#define INDEX_NUM_OF_THREADS 5
 #define INDEX_QUEUE_SIZE     200
 
 #define INDEX_DATA_BOOL_NULL      0x02
@@ -85,7 +85,7 @@ static int  indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp
 static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch);
 
 // merge cache and tfile by opera type
-static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTempResult* helper);
+static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTRslt* helper);
 
 // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
 // int32_t        indexSerialKey(ICacheKey* key, char* buf);
@@ -201,6 +201,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
     char      buf[128] = {0};
     ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType};
     int32_t   sz = indexSerialCacheKey(&key, buf);
+    indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
 
     IndexCache** cache = taosHashGet(index->colObj, buf, sz);
     assert(*cache != NULL);
@@ -328,6 +329,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
   char      buf[128] = {0};
   ICacheKey key = {
       .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType};
+  indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType);
   int32_t sz = indexSerialCacheKey(&key, buf);
 
   taosThreadMutexLock(&sIdx->mtx);
@@ -341,7 +343,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
 
   int64_t st = taosGetTimestampUs();
 
-  SIdxTempResult* tr = sIdxTempResultCreate();
+  SIdxTRslt* tr = idxTRsltCreate();
   if (0 == indexCacheSearch(cache, query, tr, &s)) {
     if (s == kTypeDeletion) {
       indexInfo("col: %s already drop by", term->colName);
@@ -363,12 +365,12 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
   int64_t cost = taosGetTimestampUs() - st;
   indexInfo("search cost: %" PRIu64 "us", cost);
 
-  sIdxTempResultMergeTo(*result, tr);
+  idxTRsltMergeTo(tr, *result);
 
-  sIdxTempResultDestroy(tr);
+  idxTRsltDestroy(tr);
   return 0;
 END:
-  sIdxTempResultDestroy(tr);
+  idxTRsltDestroy(tr);
   return -1;
 }
 static void indexInterResultsDestroy(SArray* results) {
@@ -404,18 +406,18 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType
   return 0;
 }
 
-static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdxTempResult* tr) {
+static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdxTRslt* tr) {
   int32_t sz = taosArrayGetSize(result);
   if (sz > 0) {
     TFileValue* lv = taosArrayGetP(result, sz - 1);
     if (tfv != NULL && strcmp(lv->colVal, tfv->colVal) != 0) {
-      sIdxTempResultMergeTo(lv->tableId, tr);
+      idxTRsltMergeTo(tr, lv->tableId);
-      sIdxTempResultClear(tr);
+      idxTRsltClear(tr);
 
       taosArrayPush(result, &tfv);
     } else if (tfv == NULL) {
       // handle last iterator
-      sIdxTempResultMergeTo(lv->tableId, tr);
+      idxTRsltMergeTo(tr, lv->tableId);
     } else {
       // temp result saved in help
       tfileValueDestroy(tfv);
@@ -424,7 +426,7 @@ static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdx
     taosArrayPush(result, &tfv);
   }
 }
-static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTempResult* tr) {
+static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTRslt* tr) {
   char*       colVal = (cv != NULL) ? cv->colVal : tv->colVal;
   TFileValue* tfv = tfileValueCreate(colVal);
 
@@ -434,9 +436,9 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateVal
     uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0);
     uint32_t ver = cv->ver;
     if (cv->type == ADD_VALUE) {
-      INDEX_MERGE_ADD_DEL(tr->deled, tr->added, id)
+      INDEX_MERGE_ADD_DEL(tr->del, tr->add, id)
     } else if (cv->type == DEL_VALUE) {
-      INDEX_MERGE_ADD_DEL(tr->added, tr->deled, id)
+      INDEX_MERGE_ADD_DEL(tr->add, tr->del, id)
     }
   }
   if (tv != NULL) {
@@ -489,7 +491,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
   bool cn = cacheIter ? cacheIter->next(cacheIter) : false;
   bool tn = tfileIter ? tfileIter->next(tfileIter) : false;
 
-  SIdxTempResult* tr = sIdxTempResultCreate();
+  SIdxTRslt* tr = idxTRsltCreate();
   while (cn == true || tn == true) {
     IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL;
     IterateValue* tv = (tn == true) ? tfileIter->getValue(tfileIter) : NULL;
@@ -515,7 +517,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
     }
   }
   indexMayMergeTempToFinalResult(result, NULL, tr);
-  sIdxTempResultDestroy(tr);
+  idxTRsltDestroy(tr);
 
   int ret = indexGenTFile(sIdx, pCache, result);
   indexDestroyFinalResult(result);

source/libs/index/src/indexCache.c

@@ -36,32 +36,31 @@ static char*   indexCacheTermGet(const void* pData);
 
 static MemTable* indexInternalCacheCreate(int8_t type);
 
-static int32_t cacheSearchTerm(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchTerm(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchPrefix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchPrefix(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchSuffix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchSuffix(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchRegex(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRegex(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchLessThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessThan(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
 /*comm func of compare, used in (LE/LT/GE/GT compare)*/
-static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s,
+static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s, RangeType type);
-                                      RangeType type);
+static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s);
-static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s);
 
-static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
+static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s,
                                            RangeType type);
 
-static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s) = {
+static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTRslt* tr, STermValueType* s) = {
     {cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual,
      cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange},
     {cacheSearchTerm_JSON, cacheSearchPrefix_JSON, cacheSearchSuffix_JSON, cacheSearchRegex_JSON,
@@ -71,7 +70,7 @@ static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTemp
 static void doMergeWork(SSchedMsg* msg);
 static bool indexCacheIteratorNext(Iterate* itera);
 
-static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   if (cache == NULL) {
     return 0;
   }
@@ -93,11 +92,11 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr
     CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
     if (0 == strcmp(c->colVal, pCt->colVal)) {
       if (c->operaType == ADD_VALUE) {
-        INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+        INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
         // taosArrayPush(result, &c->uid);
         *s = kTypeValue;
       } else if (c->operaType == DEL_VALUE) {
-        INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+        INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
       }
     } else {
       break;
@@ -108,20 +107,19 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr
   tSkipListDestroyIter(iter);
   return 0;
 }
-static int32_t cacheSearchPrefix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchPrefix(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   // impl later
   return 0;
 }
-static int32_t cacheSearchSuffix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchSuffix(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   // impl later
   return 0;
 }
-static int32_t cacheSearchRegex(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchRegex(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   // impl later
   return 0;
 }
-static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
+static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s, RangeType type) {
-                                      RangeType type) {
   if (cache == NULL) {
     return 0;
   }
@@ -133,6 +131,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempRes
 
   CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm));
   pCt->colVal = term->colVal;
+  pCt->colType = term->colType;
   pCt->version = atomic_load_64(&pCache->version);
 
   char* key = indexCacheTermGet(pCt);
@@ -147,11 +146,11 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempRes
     TExeCond   cond = cmpFn(c->colVal, pCt->colVal, pCt->colType);
     if (cond == MATCH) {
       if (c->operaType == ADD_VALUE) {
-        INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+        INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
         // taosArrayPush(result, &c->uid);
         *s = kTypeValue;
       } else if (c->operaType == DEL_VALUE) {
-        INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+        INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
       }
     } else if (cond == CONTINUE) {
       continue;
@@ -163,20 +162,20 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempRes
   tSkipListDestroyIter(iter);
   return TSDB_CODE_SUCCESS;
 }
-static int32_t cacheSearchLessThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchLessThan(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return cacheSearchCompareFunc(cache, term, tr, s, LT);
 }
-static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return cacheSearchCompareFunc(cache, term, tr, s, LE);
 }
-static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return cacheSearchCompareFunc(cache, term, tr, s, GT);
 }
-static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return cacheSearchCompareFunc(cache, term, tr, s, GE);
 }
 
-static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   if (cache == NULL) {
     return 0;
   }
@@ -204,11 +203,11 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResul
 
     if (0 == strcmp(c->colVal, pCt->colVal)) {
       if (c->operaType == ADD_VALUE) {
-        INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+        INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
         // taosArrayPush(result, &c->uid);
         *s = kTypeValue;
       } else if (c->operaType == DEL_VALUE) {
-        INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+        INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
       }
     } else {
       break;
@@ -222,32 +221,32 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResul
 
   return TSDB_CODE_SUCCESS;
 }
-static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return TSDB_CODE_SUCCESS;
 }
-static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return TSDB_CODE_SUCCESS;
 }
-static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return TSDB_CODE_SUCCESS;
 }
-static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return cacheSearchCompareFunc_JSON(cache, term, tr, s, LT);
 }
-static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return cacheSearchCompareFunc_JSON(cache, term, tr, s, LE);
 }
-static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return cacheSearchCompareFunc_JSON(cache, term, tr, s, GT);
 }
-static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return cacheSearchCompareFunc_JSON(cache, term, tr, s, GE);
 }
-static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s,
+static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s,
                                            RangeType type) {
   if (cache == NULL) {
     return 0;
@@ -289,11 +288,11 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe
     TExeCond cond = cmpFn(p + skip, term->colVal, dType);
     if (cond == MATCH) {
       if (c->operaType == ADD_VALUE) {
-        INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+        INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
         // taosArrayPush(result, &c->uid);
         *s = kTypeValue;
       } else if (c->operaType == DEL_VALUE) {
-        INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+        INDEX_MERGE_ADD_DEL(tr->add, tr->del, c->uid)
       }
     } else if (cond == CONTINUE) {
       continue;
@@ -309,7 +308,7 @@ static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTe
 
   return TSDB_CODE_SUCCESS;
 }
-static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) {
+static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   // impl later
   return 0;
 }
@@ -568,7 +567,7 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u
   return 0;
 }
 
-static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTempResult* tr, STermValueType* s) {
+static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, STermValueType* s) {
   if (mem == NULL) {
     return 0;
   }
@@ -582,7 +581,7 @@ static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTempResu
     return cacheSearch[0][qtype](mem, term, tr, s);
   }
 }
-int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) {
+int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) {
   int64_t st = taosGetTimestampUs();
   if (cache == NULL) {
     return 0;
@@ -597,10 +596,10 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result
   indexMemRef(imm);
   taosThreadMutexUnlock(&pCache->mtx);
 
-  int ret = indexQueryMem(mem, query, result, s);
+  int ret = (mem && mem->mem) ? indexQueryMem(mem, query, result, s) : 0;
   if (ret == 0 && *s != kTypeDeletion) {
     // continue search in imm
-    ret = indexQueryMem(imm, query, result, s);
+    ret = (imm && imm->mem) ? indexQueryMem(imm, query, result, s) : 0;
   }
 
   indexMemUnRef(mem);
@@ -709,7 +708,7 @@ static int32_t indexCacheJsonTermCompare(const void* l, const void* r) {
   return cmp;
 }
 static MemTable* indexInternalCacheCreate(int8_t type) {
-  int ttype = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : type;
+  int ttype = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY;
   int32_t (*cmpFn)(const void* l, const void* r) =
       INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? indexCacheJsonTermCompare : indexCacheTermCompare;
 

source/libs/index/src/indexFilter.c

@@ -37,12 +37,15 @@ typedef struct SIFParam {
   int64_t       suid;  // add later
   char          dbName[TSDB_DB_NAME_LEN];
   char          colName[TSDB_COL_NAME_LEN];
+
+  SIndexMetaArg arg;
 } SIFParam;
 
 typedef struct SIFCtx {
   int32_t       code;
   SHashObj *    pRes;    /* element is SIFParam */
   bool          noExec;  // true: just iterate condition tree, and add hint to executor plan
+  SIndexMetaArg arg;
   // SIdxFltStatus st;
 } SIFCtx;
 
@@ -257,7 +260,9 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu
   return TSDB_CODE_QRY_INVALID_INPUT;
 }
 static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
-  SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName),
+#ifdef USE_INVERTED_INDEX
+  SIndexMetaArg *arg = &output->arg;
+  SIndexTerm *   tm = indexTermCreate(arg->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName),
                                    right->condValue, strlen(right->condValue));
   if (tm == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
@@ -268,9 +273,13 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
 
   SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST);
   indexMultiTermQueryAdd(mtm, tm, qtype);
-  int ret = indexSearch(NULL, mtm, output->result);
+  int ret = indexSearch(arg->metaHandle, mtm, output->result);
+  indexDebug("index filter data size: %d", (int)taosArrayGetSize(output->result));
   indexMultiTermQueryDestroy(mtm);
   return ret;
+#else
+  return 0;
+#endif
 }
 
 static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
@@ -372,6 +381,8 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
   SIFParam *params = NULL;
   SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
 
+  // ugly code, refactor later
+  output->arg = ctx->arg;
   sif_func_t operFn = sifGetOperFn(node->opType);
   if (ctx->noExec && operFn == NULL) {
     output->status = SFLT_NOT_INDEX;
@@ -423,7 +434,7 @@ _return:
 
 static EDealRes sifWalkFunction(SNode *pNode, void *context) {
   SFunctionNode *node = (SFunctionNode *)pNode;
-  SIFParam       output = {0};
+  SIFParam       output = {.result = taosArrayInit(8, sizeof(uint64_t))};
 
   SIFCtx *ctx = context;
   ctx->code = sifExecFunction(node, ctx, &output);
@@ -439,7 +450,8 @@ static EDealRes sifWalkFunction(SNode *pNode, void *context) {
 }
 static EDealRes sifWalkLogic(SNode *pNode, void *context) {
   SLogicConditionNode *node = (SLogicConditionNode *)pNode;
-  SIFParam             output = {0};
+
+  SIFParam output = {.result = taosArrayInit(8, sizeof(uint64_t))};
 
   SIFCtx *ctx = context;
   ctx->code = sifExecLogic(node, ctx, &output);
@@ -455,7 +467,7 @@ static EDealRes sifWalkLogic(SNode *pNode, void *context) {
 }
 static EDealRes sifWalkOper(SNode *pNode, void *context) {
   SOperatorNode *node = (SOperatorNode *)pNode;
-  SIFParam       output = {0};
+  SIFParam       output = {.result = taosArrayInit(8, sizeof(uint64_t))};
 
   SIFCtx *ctx = context;
   ctx->code = sifExecOper(node, ctx, &output);
@@ -507,8 +519,9 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
   int32_t code = 0;
-  SIFCtx  ctx = {.code = 0, .noExec = false};
+  SIFCtx  ctx = {.code = 0, .noExec = false, .arg = pDst->arg};
   ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+
   if (NULL == ctx.pRes) {
     indexError("index-filter failed to taosHashInit");
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
@@ -523,7 +536,9 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
       indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode));
       SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
     }
+    if (res->result != NULL) {
       taosArrayAddAll(pDst->result, res->result);
+    }
 
     sifFreeParam(res);
     taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES);
@@ -561,7 +576,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) {
   SIF_RET(code);
 }
 
-int32_t doFilterTag(const SNode *pFilterNode, SArray *result) {
+int32_t doFilterTag(const SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result) {
   if (pFilterNode == NULL) {
     return TSDB_CODE_SUCCESS;
   }
@@ -570,10 +585,12 @@ int32_t doFilterTag(const SNode *pFilterNode, SArray *result) {
   // todo move to the initialization function
   // SIF_ERR_RET(filterInitFromNode((SNode *)pFilterNode, &filter, 0));
 
-  SIFParam param = {0};
+  SArray * output = taosArrayInit(8, sizeof(uint64_t));
+  SIFParam param = {.arg = *metaArg, .result = output};
   SIF_ERR_RET(sifCalculate((SNode *)pFilterNode, &param));
 
   taosArrayAddAll(result, param.result);
+  // taosArrayAddAll(result, param.result);
   sifFreeParam(&param);
   SIF_RET(TSDB_CODE_SUCCESS);
 }

source/libs/index/src/indexFst.c

@@ -1324,7 +1324,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
       if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) {
         taosArrayPop(sws->inp);
       }
-      // streamStateDestroy(p);
+      streamStateDestroy(p);
       continue;
     }
     FstTransition trn;

source/libs/index/src/indexFstUtil.c

@@ -93,7 +93,7 @@ FstSlice fstSliceCreate(uint8_t* data, uint64_t len) {
 // just shallow copy
 FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) {
   FstString* str = s->str;
-  str->ref++;
+  atomic_add_fetch_32(&str->ref, 1);
 
   FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start};
   return t;
@@ -101,6 +101,7 @@ FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) {
 FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) {
   int32_t tlen = end - start + 1;
   int32_t slen;
+
   uint8_t* data = fstSliceData(s, &slen);
   assert(tlen <= slen);
 
@@ -129,8 +130,9 @@ uint8_t* fstSliceData(FstSlice* s, int32_t* size) {
 }
 void fstSliceDestroy(FstSlice* s) {
   FstString* str = s->str;
-  str->ref--;
+
-  if (str->ref == 0) {
+  int32_t ref = atomic_sub_fetch_32(&str->ref, 1);
+  if (ref == 0) {
     taosMemoryFree(str->data);
     taosMemoryFree(str);
     s->str = NULL;

source/libs/index/src/indexTfile.c

@@ -60,31 +60,31 @@ static void    tfileGenFileFullName(char* fullname, const char* path, uint64_t s
 /*
  * search from  tfile
  */
-static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
 
-static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype);
+static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, RangeType ctype);
 
-static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
-static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr);
 
-static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype);
+static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr, RangeType ctype);
 
-static int32_t (*tfSearch[][QUERY_MAX])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = {
+static int32_t (*tfSearch[][QUERY_MAX])(void* reader, SIndexTerm* tem, SIdxTRslt* tr) = {
     {tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, tfSearchLessEqual,
      tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange},
     {tfSearchTerm_JSON, tfSearchPrefix_JSON, tfSearchSuffix_JSON, tfSearchRegex_JSON, tfSearchLessThan_JSON,
@@ -211,16 +211,16 @@ void tfileReaderDestroy(TFileReader* reader) {
   }
   // T_REF_INC(reader);
   fstDestroy(reader->fst);
-  writerCtxDestroy(reader->ctx, reader->remove);
   if (reader->remove) {
     indexInfo("%s is removed", reader->ctx->file.buf);
   } else {
     indexInfo("%s is not removed", reader->ctx->file.buf);
   }
+  writerCtxDestroy(reader->ctx, reader->remove);
 
   taosMemoryFree(reader);
 }
-static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   int      ret = 0;
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
@@ -243,7 +243,7 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
   return 0;
 }
 
-static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
   char*    p = tem->colVal;
   uint64_t sz = tem->nColVal;
@@ -279,7 +279,7 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr)
   }
   return 0;
 }
-static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
 
   int      ret = 0;
@@ -298,7 +298,7 @@ static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr)
   fstSliceDestroy(&key);
   return 0;
 }
-static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
 
   int      ret = 0;
@@ -319,7 +319,7 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr)
   return 0;
 }
 
-static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) {
+static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, RangeType type) {
   int                  ret = 0;
   char*                p = tem->colVal;
   int                  skip = 0;
@@ -358,19 +358,19 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult
   fstStreamBuilderDestroy(sb);
   return TSDB_CODE_SUCCESS;
 }
-static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return tfSearchCompareFunc(reader, tem, tr, LT);
 }
-static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return tfSearchCompareFunc(reader, tem, tr, LE);
 }
-static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return tfSearchCompareFunc(reader, tem, tr, GT);
 }
-static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return tfSearchCompareFunc(reader, tem, tr, GE);
 }
-static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
   int      ret = 0;
   char*    p = tem->colVal;
@@ -399,7 +399,7 @@ static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr)
   fstSliceDestroy(&key);
   return 0;
 }
-static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   int   ret = 0;
   char* p = indexPackJsonData(tem);
   int   sz = strlen(p);
@@ -424,36 +424,36 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTempResult*
   // deprecate api
   return TSDB_CODE_SUCCESS;
 }
-static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   // impl later
   return TSDB_CODE_SUCCESS;
 }
-static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   // impl later
   return TSDB_CODE_SUCCESS;
 }
-static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchRegex_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   // impl later
   return TSDB_CODE_SUCCESS;
 }
-static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchLessThan_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return tfSearchCompareFunc_JSON(reader, tem, tr, LT);
 }
-static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchLessEqual_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return tfSearchCompareFunc_JSON(reader, tem, tr, LE);
 }
-static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchGreaterThan_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return tfSearchCompareFunc_JSON(reader, tem, tr, GT);
 }
-static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchGreaterEqual_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return tfSearchCompareFunc_JSON(reader, tem, tr, GE);
 }
-static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+static int32_t tfSearchRange_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   // impl later
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype) {
+static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr, RangeType ctype) {
   int ret = 0;
   int skip = 0;
 
@@ -501,7 +501,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
   fstStreamBuilderDestroy(sb);
   return TSDB_CODE_SUCCESS;
 }
-int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) {
+int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr) {
   SIndexTerm*     term = query->term;
   EIndexQueryType qtype = query->qType;
   int             ret = 0;
@@ -673,7 +673,7 @@ void indexTFileDestroy(IndexTFile* tfile) {
   taosMemoryFree(tfile);
 }
 
-int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTempResult* result) {
+int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* result) {
   int ret = -1;
   if (tfile == NULL) {
     return ret;

source/libs/index/src/indexUtil.c

@@ -36,24 +36,24 @@ static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) {
   return s;
 }
 
-void iIntersection(SArray *inters, SArray *final) {
+void iIntersection(SArray *in, SArray *out) {
-  int32_t sz = (int32_t)taosArrayGetSize(inters);
+  int32_t sz = (int32_t)taosArrayGetSize(in);
   if (sz <= 0) {
     return;
   }
   MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex));
   for (int i = 0; i < sz; i++) {
-    SArray *t = taosArrayGetP(inters, i);
+    SArray *t = taosArrayGetP(in, i);
     mi[i].len = (int32_t)taosArrayGetSize(t);
     mi[i].idx = 0;
   }
 
-  SArray *base = taosArrayGetP(inters, 0);
+  SArray *base = taosArrayGetP(in, 0);
   for (int i = 0; i < taosArrayGetSize(base); i++) {
     uint64_t tgt = *(uint64_t *)taosArrayGet(base, i);
     bool     has = true;
-    for (int j = 1; j < taosArrayGetSize(inters); j++) {
+    for (int j = 1; j < taosArrayGetSize(in); j++) {
-      SArray *oth = taosArrayGetP(inters, j);
+      SArray *oth = taosArrayGetP(in, j);
       int     mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
       if (mid >= 0 && mid < mi[j].len) {
         uint64_t val = *(uint64_t *)taosArrayGet(oth, mid);
@@ -64,33 +64,33 @@ void iIntersection(SArray *inters, SArray *final) {
       }
     }
     if (has == true) {
-      taosArrayPush(final, &tgt);
+      taosArrayPush(out, &tgt);
     }
   }
   taosMemoryFreeClear(mi);
 }
-void iUnion(SArray *inters, SArray *final) {
+void iUnion(SArray *in, SArray *out) {
-  int32_t sz = (int32_t)taosArrayGetSize(inters);
+  int32_t sz = (int32_t)taosArrayGetSize(in);
   if (sz <= 0) {
     return;
   }
   if (sz == 1) {
-    taosArrayAddAll(final, taosArrayGetP(inters, 0));
+    taosArrayAddAll(out, taosArrayGetP(in, 0));
     return;
   }
 
   MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex));
   for (int i = 0; i < sz; i++) {
-    SArray *t = taosArrayGetP(inters, i);
+    SArray *t = taosArrayGetP(in, i);
     mi[i].len = (int32_t)taosArrayGetSize(t);
     mi[i].idx = 0;
   }
   while (1) {
-    uint64_t mVal = UINT_MAX;
+    uint64_t mVal = UINT64_MAX;
     int      mIdx = -1;
 
     for (int j = 0; j < sz; j++) {
-      SArray *t = taosArrayGetP(inters, j);
+      SArray *t = taosArrayGetP(in, j);
       if (mi[j].idx >= mi[j].len) {
         continue;
       }
@@ -102,13 +102,13 @@ void iUnion(SArray *inters, SArray *final) {
     }
     if (mIdx != -1) {
       mi[mIdx].idx++;
-      if (taosArrayGetSize(final) > 0) {
+      if (taosArrayGetSize(out) > 0) {
-        uint64_t lVal = *(uint64_t *)taosArrayGetLast(final);
+        uint64_t lVal = *(uint64_t *)taosArrayGetLast(out);
         if (lVal == mVal) {
           continue;
         }
       }
-      taosArrayPush(final, &mVal);
+      taosArrayPush(out, &mVal);
     } else {
       break;
     }
@@ -158,41 +158,44 @@ int verdataCompare(const void *a, const void *b) {
   return cmp;
 }
 
-SIdxTempResult *sIdxTempResultCreate() {
+SIdxTRslt *idxTRsltCreate() {
-  SIdxTempResult *tr = taosMemoryCalloc(1, sizeof(SIdxTempResult));
+  SIdxTRslt *tr = taosMemoryCalloc(1, sizeof(SIdxTRslt));
 
   tr->total = taosArrayInit(4, sizeof(uint64_t));
-  tr->added = taosArrayInit(4, sizeof(uint64_t));
+  tr->add = taosArrayInit(4, sizeof(uint64_t));
-  tr->deled = taosArrayInit(4, sizeof(uint64_t));
+  tr->del = taosArrayInit(4, sizeof(uint64_t));
   return tr;
 }
-void sIdxTempResultClear(SIdxTempResult *tr) {
+void idxTRsltClear(SIdxTRslt *tr) {
   if (tr == NULL) {
     return;
   }
   taosArrayClear(tr->total);
-  taosArrayClear(tr->added);
+  taosArrayClear(tr->add);
-  taosArrayClear(tr->deled);
+  taosArrayClear(tr->del);
 }
-void sIdxTempResultDestroy(SIdxTempResult *tr) {
+void idxTRsltDestroy(SIdxTRslt *tr) {
   if (tr == NULL) {
     return;
   }
   taosArrayDestroy(tr->total);
-  taosArrayDestroy(tr->added);
+  taosArrayDestroy(tr->add);
-  taosArrayDestroy(tr->deled);
+  taosArrayDestroy(tr->del);
 }
-void sIdxTempResultMergeTo(SArray *result, SIdxTempResult *tr) {
+void idxTRsltMergeTo(SIdxTRslt *tr, SArray *result) {
   taosArraySort(tr->total, uidCompare);
-  taosArraySort(tr->added, uidCompare);
+  taosArraySort(tr->add, uidCompare);
-  taosArraySort(tr->deled, uidCompare);
+  taosArraySort(tr->del, uidCompare);
 
+  if (taosArrayGetSize(tr->total) == 0 || taosArrayGetSize(tr->add) == 0) {
+    SArray *t = taosArrayGetSize(tr->total) == 0 ? tr->add : tr->total;
+    taosArrayAddAll(result, t);
+  } else {
     SArray *arrs = taosArrayInit(2, sizeof(void *));
     taosArrayPush(arrs, &tr->total);
-  taosArrayPush(arrs, &tr->added);
+    taosArrayPush(arrs, &tr->add);
-
     iUnion(arrs, result);
     taosArrayDestroy(arrs);
-
+  }
-  iExcept(result, tr->deled);
+  iExcept(result, tr->del);
 }

source/libs/index/test/CMakeLists.txt

@@ -1,74 +1,74 @@
-add_executable(indexTest "")
+add_executable(idxTest "")
-add_executable(fstTest "")
+add_executable(idxFstTest "")
-add_executable(fstUT "")
+add_executable(idxFstUT "")
-add_executable(UtilUT "")
+add_executable(idxUtilUT "")
-add_executable(jsonUT "")
+add_executable(idxJsonUT "")
 
-target_sources(indexTest
+target_sources(idxTest
   PRIVATE 
   "indexTests.cc" 
 )
-target_sources(fstTest
+target_sources(idxFstTest
   PRIVATE 
   "fstTest.cc" 
 )
 
-target_sources(fstUT
+target_sources(idxFstUT
   PRIVATE 
   "fstUT.cc" 
 )
-target_sources(UtilUT
+target_sources(idxUtilUT
   PRIVATE 
   "utilUT.cc" 
 )
 
-target_sources(jsonUT
+target_sources(idxJsonUT
   PRIVATE 
   "jsonUT.cc" 
 )
-target_include_directories ( indexTest
+target_include_directories (idxTest
   PUBLIC
   "${TD_SOURCE_DIR}/include/libs/index" 
   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 ) 
-target_include_directories ( fstTest 
+target_include_directories (idxFstTest 
   PUBLIC
   "${TD_SOURCE_DIR}/include/libs/index" 
   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 ) 
 
-target_include_directories ( fstUT 
+target_include_directories (idxFstUT 
   PUBLIC
   "${TD_SOURCE_DIR}/include/libs/index" 
   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 ) 
 
-target_include_directories ( UtilUT 
+target_include_directories (idxUtilUT 
   PUBLIC
   "${TD_SOURCE_DIR}/include/libs/index" 
   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 ) 
 
-target_include_directories (jsonUT
+target_include_directories (idxJsonUT
   PUBLIC
   "${TD_SOURCE_DIR}/include/libs/index" 
   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 ) 
-target_link_libraries (indexTest
+target_link_libraries (idxTest
   os  
   util
   common
   gtest_main
   index
 )
-target_link_libraries (fstTest
+target_link_libraries (idxFstTest
   os  
   util
   common
   gtest_main
   index
 )
-target_link_libraries (fstUT
+target_link_libraries (idxFstUT
   os  
   util
   common
@@ -76,7 +76,7 @@ target_link_libraries (fstUT
   index
 )
 
-target_link_libraries (UtilUT
+target_link_libraries (idxUtilUT
   os  
   util
   common
@@ -84,7 +84,7 @@ target_link_libraries (UtilUT
   index
 )
 
-target_link_libraries (jsonUT
+target_link_libraries (idxJsonUT
   os  
   util
   common
@@ -92,19 +92,21 @@ target_link_libraries (jsonUT
   index
 )
 
+if(NOT TD_WINDOWS)
   add_test(
     NAME idxtest
-  COMMAND indexTest
+    COMMAND idxTest
   )
   add_test(
     NAME idxJsonUT
-  COMMAND jsonUT 
+    COMMAND idxJsonUT 
   )
+endif(NOT TD_WINDOWS)
 add_test(
   NAME idxUtilUT 
-  COMMAND UtilUT 
+  COMMAND idxUtilUT 
 )
 add_test(
   NAME idxFstUT 
-  COMMAND fstUT 
+  COMMAND idxFstUT 
 )

source/libs/index/test/indexTests.cc

@@ -272,7 +272,7 @@ void validateFst() {
   }
   delete m;
 }
-static std::string logDir = "/tmp/log";
+static std::string logDir = TD_TMP_DIR_PATH "log";
 
 static void initLog() {
   const char*   defaultLogFileNamePrefix = "taoslog";
@@ -411,12 +411,12 @@ class TFileObj {
       //
       //
     }
-    SIdxTempResult* tr = sIdxTempResultCreate();
+    SIdxTRslt* tr = idxTRsltCreate();
 
     int ret = tfileReaderSearch(reader_, query, tr);
 
-    sIdxTempResultMergeTo(result, tr);
+    idxTRsltMergeTo(tr, result);
-    sIdxTempResultDestroy(tr);
+    idxTRsltDestroy(tr);
     return ret;
   }
   ~TFileObj() {
@@ -531,11 +531,11 @@ class CacheObj {
     indexCacheDebug(cache);
   }
   int Get(SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) {
-    SIdxTempResult* tr = sIdxTempResultCreate();
+    SIdxTRslt* tr = idxTRsltCreate();
 
     int ret = indexCacheSearch(cache, query, tr, s);
-    sIdxTempResultMergeTo(result, tr);
+    idxTRsltMergeTo(tr, result);
-    sIdxTempResultDestroy(tr);
+    idxTRsltDestroy(tr);
 
     if (ret != 0) {
       std::cout << "failed to get from cache:" << ret << std::endl;
@@ -916,7 +916,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
   }
 }
 TEST_F(IndexEnv2, testEmptyIndexOpen) {
-  std::string path = "/tmp/test";
+  std::string path = TD_TMP_DIR_PATH "test";
   if (index->Init(path) != 0) {
     std::cout << "failed to init index" << std::endl;
     exit(1);

source/libs/index/test/utilUT.cc

@@ -226,6 +226,22 @@ TEST_F(UtilEnv, 04union) {
   iUnion(src, rslt);
   assert(taosArrayGetSize(rslt) == 12);
 }
+TEST_F(UtilEnv, 05unionExcept) {
+  clearSourceArray(src);
+  clearFinalArray(rslt);
+
+  uint64_t arr2[] = {7};
+  SArray * f = (SArray *)taosArrayGetP(src, 1);
+  for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
+    taosArrayPush(f, &arr2[i]);
+  }
+
+  iUnion(src, rslt);
+
+  SArray *ept = taosArrayInit(0, sizeof(uint64_t));
+  iExcept(rslt, ept);
+  EXPECT_EQ(taosArrayGetSize(rslt), 1);
+}
 TEST_F(UtilEnv, 01Except) {
   SArray *total = taosArrayInit(4, sizeof(uint64_t));
   {
@@ -308,16 +324,36 @@ TEST_F(UtilEnv, 01Except) {
   ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 100);
 }
 TEST_F(UtilEnv, testFill) {
-  for (int i = 0; i < 10000000; i++) {
+  for (int i = 0; i < 1000000; i++) {
     int64_t val = i;
     char    buf[65] = {0};
     indexInt2str(val, buf, 1);
     EXPECT_EQ(val, taosStr2int64(buf));
   }
-  for (int i = 0; i < 10000000; i++) {
+  for (int i = 0; i < 1000000; i++) {
     int64_t val = 0 - i;
     char    buf[65] = {0};
     indexInt2str(val, buf, -1);
     EXPECT_EQ(val, taosStr2int64(buf));
   }
 }
+TEST_F(UtilEnv, TempResult) {
+  SIdxTRslt *relt = idxTRsltCreate();
+
+  SArray *f = taosArrayInit(0, sizeof(uint64_t));
+
+  uint64_t val = UINT64_MAX - 1;
+  taosArrayPush(relt->add, &val);
+  idxTRsltMergeTo(relt, f);
+  EXPECT_EQ(taosArrayGetSize(f), 1);
+}
+TEST_F(UtilEnv, TempResultExcept) {
+  SIdxTRslt *relt = idxTRsltCreate();
+
+  SArray *f = taosArrayInit(0, sizeof(uint64_t));
+
+  uint64_t val = UINT64_MAX;
+  taosArrayPush(relt->add, &val);
+  idxTRsltMergeTo(relt, f);
+  EXPECT_EQ(taosArrayGetSize(f), 1);
+}

source/libs/nodes/src/nodesCodeFuncs.c

@@ -1130,6 +1130,9 @@ static const char* jkTableScanPhysiPlanOffset = "Offset";
 static const char* jkTableScanPhysiPlanSliding = "Sliding";
 static const char* jkTableScanPhysiPlanIntervalUnit = "intervalUnit";
 static const char* jkTableScanPhysiPlanSlidingUnit = "slidingUnit";
+static const char* jkTableScanPhysiPlanTriggerType = "triggerType";
+static const char* jkTableScanPhysiPlanWatermark = "watermark";
+static const char* jkTableScanPhysiPlanTsColId = "tsColId";
 
 static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
   const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj;
@@ -1171,6 +1174,15 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanTriggerType, pNode->triggerType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanWatermark, pNode->watermark);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanTsColId, pNode->tsColId);
+  }
 
   return code;
 }
@@ -1221,6 +1233,15 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
     tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit, code);
     ;
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    tjsonGetNumberValue(pJson, jkTableScanPhysiPlanTriggerType, pNode->triggerType, code);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    tjsonGetNumberValue(pJson, jkTableScanPhysiPlanWatermark, pNode->watermark, code);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    tjsonGetNumberValue(pJson, jkTableScanPhysiPlanTsColId, pNode->tsColId, code);
+  }
 
   return code;
 }
@@ -2775,6 +2796,150 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkDatabaseOptionsBuffer = "Buffer";
+static const char* jkDatabaseOptionsCachelast = "Cachelast";
+static const char* jkDatabaseOptionsCompressionLevel = "CompressionLevel";
+static const char* jkDatabaseOptionsDaysPerFileNode = "DaysPerFileNode";
+static const char* jkDatabaseOptionsDaysPerFile = "DaysPerFile";
+static const char* jkDatabaseOptionsFsyncPeriod = "FsyncPeriod";
+static const char* jkDatabaseOptionsMaxRowsPerBlock = "MaxRowsPerBlock";
+static const char* jkDatabaseOptionsMinRowsPerBlock = "MinRowsPerBlock";
+static const char* jkDatabaseOptionsKeep = "Keep";
+static const char* jkDatabaseOptionsPages = "Pages";
+static const char* jkDatabaseOptionsPagesize = "Pagesize";
+static const char* jkDatabaseOptionsPrecision = "Precision";
+static const char* jkDatabaseOptionsReplica = "Replica";
+static const char* jkDatabaseOptionsStrict = "Strict";
+static const char* jkDatabaseOptionsWalLevel = "WalLevel";
+static const char* jkDatabaseOptionsNumOfVgroups = "NumOfVgroups";
+static const char* jkDatabaseOptionsSingleStable = "SingleStable";
+static const char* jkDatabaseOptionsRetentions = "Retentions";
+static const char* jkDatabaseOptionsSchemaless = "Schemaless";
+
+static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) {
+  const SDatabaseOptions* pNode = (const SDatabaseOptions*)pObj;
+
+  int32_t code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsBuffer, pNode->buffer);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCachelast, pNode->cachelast);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCompressionLevel, pNode->compressionLevel);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkDatabaseOptionsDaysPerFileNode, nodeToJson, pNode->pDaysPerFile);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsDaysPerFile, pNode->daysPerFile);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsFsyncPeriod, pNode->fsyncPeriod);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsMaxRowsPerBlock, pNode->maxRowsPerBlock);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsMinRowsPerBlock, pNode->minRowsPerBlock);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodeListToJson(pJson, jkDatabaseOptionsKeep, pNode->pKeep);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsPages, pNode->pages);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsPagesize, pNode->pagesize);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddStringToObject(pJson, jkDatabaseOptionsPrecision, pNode->precisionStr);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsReplica, pNode->replica);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsStrict, pNode->strict);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsWalLevel, pNode->walLevel);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsNumOfVgroups, pNode->numOfVgroups);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsSingleStable, pNode->singleStable);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodeListToJson(pJson, jkDatabaseOptionsRetentions, pNode->pRetentions);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsSchemaless, pNode->schemaless);
+  }
+
+  return code;
+}
+
+static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) {
+  SDatabaseOptions* pNode = (SDatabaseOptions*)pObj;
+
+  int32_t code = tjsonGetIntValue(pJson, jkDatabaseOptionsBuffer, &pNode->buffer);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCachelast, &pNode->cachelast);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCompressionLevel, &pNode->compressionLevel);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkDatabaseOptionsDaysPerFileNode, (SNode**)&pNode->pDaysPerFile);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkDatabaseOptionsDaysPerFile, &pNode->daysPerFile);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkDatabaseOptionsFsyncPeriod, &pNode->fsyncPeriod);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkDatabaseOptionsMaxRowsPerBlock, &pNode->maxRowsPerBlock);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkDatabaseOptionsMinRowsPerBlock, &pNode->minRowsPerBlock);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeList(pJson, jkDatabaseOptionsKeep, &pNode->pKeep);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkDatabaseOptionsPages, &pNode->pages);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkDatabaseOptionsPagesize, &pNode->pagesize);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetStringValue(pJson, jkDatabaseOptionsPrecision, pNode->precisionStr);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsReplica, &pNode->replica);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsStrict, &pNode->strict);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsWalLevel, &pNode->walLevel);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkDatabaseOptionsNumOfVgroups, &pNode->numOfVgroups);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsSingleStable, &pNode->singleStable);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeList(pJson, jkDatabaseOptionsRetentions, &pNode->pRetentions);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsSchemaless, &pNode->schemaless);
+  }
+
+  return code;
+}
+
 static const char* jkDataBlockDescDataBlockId = "DataBlockId";
 static const char* jkDataBlockDescSlots = "Slots";
 static const char* jkDataBlockTotalRowSize = "TotalRowSize";
@@ -2977,6 +3142,130 @@ static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkAlterDatabaseStmtDbName = "DbName";
+static const char* jkAlterDatabaseStmtOptions = "Options";
+
+static int32_t alterDatabaseStmtToJson(const void* pObj, SJson* pJson) {
+  const SAlterDatabaseStmt* pNode = (const SAlterDatabaseStmt*)pObj;
+
+  int32_t code = tjsonAddStringToObject(pJson, jkAlterDatabaseStmtDbName, pNode->dbName);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkAlterDatabaseStmtOptions, nodeToJson, pNode->pOptions);
+  }
+
+  return code;
+}
+
+static int32_t jsonToAlterDatabaseStmt(const SJson* pJson, void* pObj) {
+  SAlterDatabaseStmt* pNode = (SAlterDatabaseStmt*)pObj;
+
+  int32_t code = tjsonGetStringValue(pJson, jkAlterDatabaseStmtDbName, pNode->dbName);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkAlterDatabaseStmtOptions, (SNode**)&pNode->pOptions);
+  }
+
+  return code;
+}
+
+static const char* jkAlterTableStmtDbName = "DbName";
+static const char* jkAlterTableStmtTableName = "TableName";
+static const char* jkAlterTableStmtAlterType = "AlterType";
+static const char* jkAlterTableStmtColName = "ColName";
+static const char* jkAlterTableStmtNewColName = "NewColName";
+static const char* jkAlterTableStmtOptions = "Options";
+static const char* jkAlterTableStmtNewDataType = "NewDataType";
+static const char* jkAlterTableStmtNewTagVal = "NewTagVal";
+
+static int32_t alterTableStmtToJson(const void* pObj, SJson* pJson) {
+  const SAlterTableStmt* pNode = (const SAlterTableStmt*)pObj;
+
+  int32_t code = tjsonAddStringToObject(pJson, jkAlterTableStmtDbName, pNode->dbName);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddStringToObject(pJson, jkAlterTableStmtTableName, pNode->tableName);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkAlterTableStmtAlterType, pNode->alterType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddStringToObject(pJson, jkAlterTableStmtColName, pNode->colName);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddStringToObject(pJson, jkAlterTableStmtNewColName, pNode->newColName);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkAlterTableStmtOptions, nodeToJson, pNode->pOptions);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkAlterTableStmtNewDataType, dataTypeToJson, &pNode->dataType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkAlterTableStmtOptions, nodeToJson, pNode->pVal);
+  }
+
+  return code;
+}
+
+static int32_t jsonToAlterTableStmt(const SJson* pJson, void* pObj) {
+  SAlterTableStmt* pNode = (SAlterTableStmt*)pObj;
+
+  int32_t code = tjsonGetStringValue(pJson, jkAlterTableStmtDbName, pNode->dbName);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetStringValue(pJson, jkAlterTableStmtTableName, pNode->tableName);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkAlterTableStmtAlterType, &pNode->alterType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetStringValue(pJson, jkAlterTableStmtColName, pNode->colName);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetStringValue(pJson, jkAlterTableStmtNewColName, pNode->newColName);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pOptions);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonToObject(pJson, jkAlterTableStmtNewDataType, jsonToDataType, &pNode->dataType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pVal);
+  }
+
+  return code;
+}
+
+static const char* jkAlterDnodeStmtDnodeId = "DnodeId";
+static const char* jkAlterDnodeStmtConfig = "Config";
+static const char* jkAlterDnodeStmtValue = "Value";
+
+static int32_t alterDnodeStmtToJson(const void* pObj, SJson* pJson) {
+  const SAlterDnodeStmt* pNode = (const SAlterDnodeStmt*)pObj;
+
+  int32_t code = tjsonAddIntegerToObject(pJson, jkAlterDnodeStmtDnodeId, pNode->dnodeId);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtConfig, pNode->config);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtValue, pNode->value);
+  }
+
+  return code;
+}
+
+static int32_t jsonToAlterDnodeStmt(const SJson* pJson, void* pObj) {
+  SAlterDnodeStmt* pNode = (SAlterDnodeStmt*)pObj;
+
+  int32_t code = tjsonGetIntValue(pJson, jkAlterDnodeStmtDnodeId, &pNode->dnodeId);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetStringValue(pJson, jkAlterDnodeStmtConfig, pNode->config);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetStringValue(pJson, jkAlterDnodeStmtValue, pNode->value);
+  }
+
+  return code;
+}
+
 static const char* jkCreateTopicStmtTopicName = "TopicName";
 static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName";
 static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists";
@@ -3061,6 +3350,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       break;
     case QUERY_NODE_DOWNSTREAM_SOURCE:
       return downstreamSourceNodeToJson(pObj, pJson);
+    case QUERY_NODE_DATABASE_OPTIONS:
+      return databaseOptionsToJson(pObj, pJson);
     case QUERY_NODE_LEFT_VALUE:
       return TSDB_CODE_SUCCESS;  // SLeftValueNode has no fields to serialize.
     case QUERY_NODE_SET_OPERATOR:
@@ -3069,8 +3360,17 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       return selectStmtToJson(pObj, pJson);
     case QUERY_NODE_VNODE_MODIF_STMT:
     case QUERY_NODE_CREATE_DATABASE_STMT:
+      break;
+    case QUERY_NODE_ALTER_DATABASE_STMT:
+      return alterDatabaseStmtToJson(pObj, pJson);
     case QUERY_NODE_CREATE_TABLE_STMT:
+      break;
+    case QUERY_NODE_ALTER_TABLE_STMT:
+      return alterTableStmtToJson(pObj, pJson);
     case QUERY_NODE_USE_DATABASE_STMT:
+      break;
+    case QUERY_NODE_ALTER_DNODE_STMT:
+      return alterDnodeStmtToJson(pObj, pJson);
     case QUERY_NODE_SHOW_DATABASES_STMT:
     case QUERY_NODE_SHOW_TABLES_STMT:
       break;
@@ -3177,12 +3477,20 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToSlotDescNode(pJson, pObj);
     case QUERY_NODE_DOWNSTREAM_SOURCE:
       return jsonToDownstreamSourceNode(pJson, pObj);
+    case QUERY_NODE_DATABASE_OPTIONS:
+      return jsonToDatabaseOptions(pJson, pObj);
     case QUERY_NODE_LEFT_VALUE:
       return TSDB_CODE_SUCCESS;  // SLeftValueNode has no fields to deserialize.
     case QUERY_NODE_SET_OPERATOR:
       return jsonToSetOperator(pJson, pObj);
     case QUERY_NODE_SELECT_STMT:
       return jsonToSelectStmt(pJson, pObj);
+    case QUERY_NODE_ALTER_DATABASE_STMT:
+      return jsonToAlterDatabaseStmt(pJson, pObj);
+    case QUERY_NODE_ALTER_TABLE_STMT:
+      return jsonToAlterTableStmt(pJson, pObj);
+    case QUERY_NODE_ALTER_DNODE_STMT:
+      return jsonToAlterDnodeStmt(pJson, pObj);
     case QUERY_NODE_CREATE_TOPIC_STMT:
       return jsonToCreateTopicStmt(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_SCAN:

source/libs/parser/inc/parAst.h

@@ -34,7 +34,6 @@ typedef struct SAstCreateContext {
   int16_t        placeholderNo;
   SArray*        pPlaceholderValues;
   int32_t        errCode;
-  SParseMetaCache* pMetaCache;
 } SAstCreateContext;
 
 typedef enum EDatabaseOptionType {
@@ -75,7 +74,7 @@ typedef struct SAlterOption {
 
 extern SToken nil_token;
 
-int32_t initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt);
+void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt);
 
 SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode);
 SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SToken* pEnd, SNode* pNode);

source/libs/parser/inc/parInt.h

@@ -26,6 +26,7 @@ extern "C" {
 
 int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery);
 int32_t parse(SParseContext* pParseCxt, SQuery** pQuery);
+int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery);
 int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery);
 int32_t translate(SParseContext* pParseCxt, SQuery* pQuery);
 int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);

source/libs/parser/inc/parUtil.h

@@ -24,12 +24,12 @@ extern "C" {
 #include "os.h"
 #include "query.h"
 
-#define parserFatal(param, ...) qFatal("PARSER: " param, __VA_ARGS__)
+#define parserFatal(param, ...) qFatal("PARSER: " param, ##__VA_ARGS__)
-#define parserError(param, ...) qError("PARSER: " param, __VA_ARGS__)
+#define parserError(param, ...) qError("PARSER: " param, ##__VA_ARGS__)
-#define parserWarn(param, ...)  qWarn("PARSER: " param, __VA_ARGS__)
+#define parserWarn(param, ...)  qWarn("PARSER: " param, ##__VA_ARGS__)
-#define parserInfo(param, ...)  qInfo("PARSER: " param, __VA_ARGS__)
+#define parserInfo(param, ...)  qInfo("PARSER: " param, ##__VA_ARGS__)
-#define parserDebug(param, ...) qDebug("PARSER: " param, __VA_ARGS__)
+#define parserDebug(param, ...) qDebug("PARSER: " param, ##__VA_ARGS__)
-#define parserTrace(param, ...) qTrace("PARSER: " param, __VA_ARGS__)
+#define parserTrace(param, ...) qTrace("PARSER: " param, ##__VA_ARGS__)
 
 #define PK_TS_COL_INTERNAL_NAME "_rowts"
 
@@ -42,7 +42,10 @@ typedef struct SParseMetaCache {
   SHashObj* pTableMeta;    // key is tbFName, element is STableMeta*
   SHashObj* pDbVgroup;     // key is dbFName, element is SArray<SVgroupInfo>*
   SHashObj* pTableVgroup;  // key is tbFName, element is SVgroupInfo*
-  SHashObj* pDbCfg;        // key is tbFName, element is SDbCfgInfo
+  SHashObj* pDbCfg;        // key is tbFName, element is SDbCfgInfo*
+  SHashObj* pDbInfo;       // key is tbFName, element is SDbInfo*
+  SHashObj* pUserAuth;     // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass
+  SHashObj* pUdf;          // key is funcName, element is SFuncInfo*
 } SParseMetaCache;
 
 int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...);
@@ -62,12 +65,22 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
 int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);
 int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache);
 int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
+int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache);
+int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
+int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache);
+int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache);
+int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type,
+                               SParseMetaCache* pMetaCache);
+int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache);
 int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta);
-int32_t getDBVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo);
+int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo);
-int32_t getTableHashVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup);
+int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup);
-int32_t getDBVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,
+int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,
                                 int32_t* pTableNum);
-int32_t getDBCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo);
+int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo);
+int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDb, AUTH_TYPE type,
+                             bool* pPass);
+int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo);
 
 #ifdef __cplusplus
 }

source/libs/parser/src/parAstCreater.c

@@ -38,7 +38,7 @@
 
 SToken nil_token = {.type = TK_NK_NIL, .n = 0, .z = NULL};
 
-int32_t initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
+void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
   memset(pCxt, 0, sizeof(SAstCreateContext));
   pCxt->pQueryCxt = pParseCxt;
   pCxt->msgBuf.buf = pParseCxt->pMsg;
@@ -48,13 +48,6 @@ int32_t initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt)
   pCxt->placeholderNo = 0;
   pCxt->pPlaceholderValues = NULL;
   pCxt->errCode = TSDB_CODE_SUCCESS;
-  if (pParseCxt->async) {
-    pCxt->pMetaCache = taosMemoryCalloc(1, sizeof(SParseMetaCache));
-    if (NULL == pCxt->pMetaCache) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
 }
 
 static void copyStringFormStringToken(SToken* pToken, char* pBuf, int32_t len) {
@@ -472,13 +465,6 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTa
     strncpy(realTable->table.tableAlias, pTableName->z, pTableName->n);
   }
   strncpy(realTable->table.tableName, pTableName->z, pTableName->n);
-  if (NULL != pCxt->pMetaCache) {
-    if (TSDB_CODE_SUCCESS != reserveTableMetaInCache(pCxt->pQueryCxt->acctId, realTable->table.dbName,
-                                                     realTable->table.tableName, pCxt->pMetaCache)) {
-      nodesDestroyNode(realTable);
-      CHECK_OUT_OF_MEM(NULL);
-    }
-  }
   return (SNode*)realTable;
 }
 

source/libs/parser/src/parAstParser.c

@@ -13,11 +13,12 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "functionMgt.h"
 #include "os.h"
-#include "parInt.h"
-
 #include "parAst.h"
+#include "parInt.h"
 #include "parToken.h"
+#include "systable.h"
 
 typedef void* (*FMalloc)(size_t);
 typedef void (*FFree)(void*);
@@ -82,8 +83,386 @@ abort_parse:
     (*pQuery)->pRoot = cxt.pRootNode;
     (*pQuery)->placeholderNum = cxt.placeholderNo;
     TSWAP((*pQuery)->pPlaceholderValues, cxt.pPlaceholderValues);
-    TSWAP((*pQuery)->pMetaCache, cxt.pMetaCache);
   }
   taosArrayDestroy(cxt.pPlaceholderValues);
   return cxt.errCode;
 }
+
+typedef struct SCollectMetaKeyCxt {
+  SParseContext*   pParseCxt;
+  SParseMetaCache* pMetaCache;
+} SCollectMetaKeyCxt;
+
+static void destroyCollectMetaKeyCxt(SCollectMetaKeyCxt* pCxt) {
+  if (NULL != pCxt->pMetaCache) {
+    // TODO
+  }
+}
+
+typedef struct SCollectMetaKeyFromExprCxt {
+  SCollectMetaKeyCxt* pComCxt;
+  int32_t             errCode;
+} SCollectMetaKeyFromExprCxt;
+
+static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt);
+
+static EDealRes collectMetaKeyFromFunction(SCollectMetaKeyFromExprCxt* pCxt, SFunctionNode* pFunc) {
+  if (fmIsBuiltinFunc(pFunc->functionName)) {
+    return TSDB_CODE_SUCCESS;
+  }
+  return reserveUdfInCache(pFunc->functionName, pCxt->pComCxt->pMetaCache);
+}
+
+static EDealRes collectMetaKeyFromRealTable(SCollectMetaKeyFromExprCxt* pCxt, SRealTableNode* pRealTable) {
+  pCxt->errCode = reserveTableMetaInCache(pCxt->pComCxt->pParseCxt->acctId, pRealTable->table.dbName,
+                                          pRealTable->table.tableName, pCxt->pComCxt->pMetaCache);
+  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
+    pCxt->errCode = reserveTableVgroupInCache(pCxt->pComCxt->pParseCxt->acctId, pRealTable->table.dbName,
+                                              pRealTable->table.tableName, pCxt->pComCxt->pMetaCache);
+  }
+  if (TSDB_CODE_SUCCESS == pCxt->errCode) {
+    pCxt->errCode = reserveUserAuthInCache(pCxt->pComCxt->pParseCxt->acctId, pCxt->pComCxt->pParseCxt->pUser,
+                                           pRealTable->table.dbName, AUTH_TYPE_READ, pCxt->pComCxt->pMetaCache);
+  }
+  return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
+}
+
+static EDealRes collectMetaKeyFromTempTable(SCollectMetaKeyFromExprCxt* pCxt, STempTableNode* pTempTable) {
+  pCxt->errCode = collectMetaKeyFromQuery(pCxt->pComCxt, pTempTable->pSubquery);
+  return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
+}
+
+static EDealRes collectMetaKeyFromExprImpl(SNode* pNode, void* pContext) {
+  SCollectMetaKeyFromExprCxt* pCxt = pContext;
+  switch (nodeType(pNode)) {
+    case QUERY_NODE_FUNCTION:
+      return collectMetaKeyFromFunction(pCxt, (SFunctionNode*)pNode);
+    case QUERY_NODE_REAL_TABLE:
+      return collectMetaKeyFromRealTable(pCxt, (SRealTableNode*)pNode);
+    case QUERY_NODE_TEMP_TABLE:
+      return collectMetaKeyFromTempTable(pCxt, (STempTableNode*)pNode);
+    default:
+      break;
+  }
+  return DEAL_RES_CONTINUE;
+}
+
+static int32_t collectMetaKeyFromExprs(SCollectMetaKeyCxt* pCxt, SNodeList* pList) {
+  SCollectMetaKeyFromExprCxt cxt = {.pComCxt = pCxt, .errCode = TSDB_CODE_SUCCESS};
+  nodesWalkExprs(pList, collectMetaKeyFromExprImpl, &cxt);
+  return cxt.errCode;
+}
+
+static int32_t collectMetaKeyFromSetOperator(SCollectMetaKeyCxt* pCxt, SSetOperator* pStmt) {
+  int32_t code = collectMetaKeyFromQuery(pCxt, pStmt->pLeft);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = collectMetaKeyFromQuery(pCxt, pStmt->pRight);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = collectMetaKeyFromExprs(pCxt, pStmt->pOrderByList);
+  }
+  return code;
+}
+
+static int32_t collectMetaKeyFromSelect(SCollectMetaKeyCxt* pCxt, SSelectStmt* pStmt) {
+  SCollectMetaKeyFromExprCxt cxt = {.pComCxt = pCxt, .errCode = TSDB_CODE_SUCCESS};
+  nodesWalkSelectStmt(pStmt, SQL_CLAUSE_FROM, collectMetaKeyFromExprImpl, &cxt);
+  return cxt.errCode;
+}
+
+static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTableStmt* pStmt) {
+  if (NULL == pStmt->pTags) {
+    return reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
+  } else {
+    return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
+  }
+}
+
+static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCreateMultiTableStmt* pStmt) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  SNode*  pNode = NULL;
+  FOREACH(pNode, pStmt->pSubTables) {
+    SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
+    code =
+        reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName, pCxt->pMetaCache);
+    if (TSDB_CODE_SUCCESS == code) {
+      code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache);
+    }
+    if (TSDB_CODE_SUCCESS != code) {
+      break;
+    }
+  }
+  return code;
+}
+
+static int32_t collectMetaKeyFromAlterTable(SCollectMetaKeyCxt* pCxt, SAlterTableStmt* pStmt) {
+  int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
+  }
+  return code;
+}
+
+static int32_t collectMetaKeyFromUseDatabase(SCollectMetaKeyCxt* pCxt, SUseDatabaseStmt* pStmt) {
+  return reserveDbVgVersionInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromCreateIndex(SCollectMetaKeyCxt* pCxt, SCreateIndexStmt* pStmt) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if (INDEX_TYPE_SMA == pStmt->indexType) {
+    code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->tableName, pCxt->pMetaCache);
+    if (TSDB_CODE_SUCCESS == code) {
+      code =
+          reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->tableName, pCxt->pMetaCache);
+    }
+  }
+  return code;
+}
+
+static int32_t collectMetaKeyFromCreateTopic(SCollectMetaKeyCxt* pCxt, SCreateTopicStmt* pStmt) {
+  if (NULL != pStmt->pQuery) {
+    return collectMetaKeyFromQuery(pCxt, pStmt->pQuery);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t collectMetaKeyFromExplain(SCollectMetaKeyCxt* pCxt, SExplainStmt* pStmt) {
+  return collectMetaKeyFromQuery(pCxt, pStmt->pQuery);
+}
+
+static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateStreamStmt* pStmt) {
+  return collectMetaKeyFromQuery(pCxt, pStmt->pQuery);
+}
+
+static int32_t collectMetaKeyFromShowDnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DNODES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowMnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MNODES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowModules(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MODULES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowQnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_QNODES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowSnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_SNODES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowBnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_BNODES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowDatabases(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_DATABASES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowFunctions(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_FUNCTIONS,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowIndexes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_INDEXES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowStables(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_STABLES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowStreams(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_STREAMS,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB,
+                                         TSDB_INS_TABLE_USER_TABLES, pCxt->pMetaCache);
+  if (TSDB_CODE_SUCCESS == code) {
+    if (NULL != pStmt->pDbName) {
+      code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
+    } else {
+      code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, pCxt->pMetaCache);
+    }
+  }
+  return code;
+}
+
+static int32_t collectMetaKeyFromShowUsers(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_USERS,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowLicence(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_LICENCES,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowVgroups(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VGROUPS,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowTopics(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_TOPICS,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromShowTransactions(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
+  return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_TRANS,
+                                 pCxt->pMetaCache);
+}
+
+static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
+  switch (nodeType(pStmt)) {
+    case QUERY_NODE_SET_OPERATOR:
+      return collectMetaKeyFromSetOperator(pCxt, (SSetOperator*)pStmt);
+    case QUERY_NODE_SELECT_STMT:
+      return collectMetaKeyFromSelect(pCxt, (SSelectStmt*)pStmt);
+    case QUERY_NODE_VNODE_MODIF_STMT:
+    case QUERY_NODE_CREATE_DATABASE_STMT:
+    case QUERY_NODE_DROP_DATABASE_STMT:
+    case QUERY_NODE_ALTER_DATABASE_STMT:
+      break;
+    case QUERY_NODE_CREATE_TABLE_STMT:
+      return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt);
+    case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
+      break;
+    case QUERY_NODE_CREATE_MULTI_TABLE_STMT:
+      return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTableStmt*)pStmt);
+    case QUERY_NODE_DROP_TABLE_CLAUSE:
+    case QUERY_NODE_DROP_TABLE_STMT:
+    case QUERY_NODE_DROP_SUPER_TABLE_STMT:
+      break;
+    case QUERY_NODE_ALTER_TABLE_STMT:
+      return collectMetaKeyFromAlterTable(pCxt, (SAlterTableStmt*)pStmt);
+    case QUERY_NODE_CREATE_USER_STMT:
+    case QUERY_NODE_ALTER_USER_STMT:
+    case QUERY_NODE_DROP_USER_STMT:
+      break;
+    case QUERY_NODE_USE_DATABASE_STMT:
+      return collectMetaKeyFromUseDatabase(pCxt, (SUseDatabaseStmt*)pStmt);
+    case QUERY_NODE_CREATE_DNODE_STMT:
+    case QUERY_NODE_DROP_DNODE_STMT:
+    case QUERY_NODE_ALTER_DNODE_STMT:
+      break;
+    case QUERY_NODE_CREATE_INDEX_STMT:
+      return collectMetaKeyFromCreateIndex(pCxt, (SCreateIndexStmt*)pStmt);
+    case QUERY_NODE_DROP_INDEX_STMT:
+    case QUERY_NODE_CREATE_QNODE_STMT:
+    case QUERY_NODE_DROP_QNODE_STMT:
+    case QUERY_NODE_CREATE_BNODE_STMT:
+    case QUERY_NODE_DROP_BNODE_STMT:
+    case QUERY_NODE_CREATE_SNODE_STMT:
+    case QUERY_NODE_DROP_SNODE_STMT:
+    case QUERY_NODE_CREATE_MNODE_STMT:
+    case QUERY_NODE_DROP_MNODE_STMT:
+      break;
+    case QUERY_NODE_CREATE_TOPIC_STMT:
+      return collectMetaKeyFromCreateTopic(pCxt, (SCreateTopicStmt*)pStmt);
+    case QUERY_NODE_DROP_TOPIC_STMT:
+    case QUERY_NODE_DROP_CGROUP_STMT:
+    case QUERY_NODE_ALTER_LOCAL_STMT:
+      break;
+    case QUERY_NODE_EXPLAIN_STMT:
+      return collectMetaKeyFromExplain(pCxt, (SExplainStmt*)pStmt);
+    case QUERY_NODE_DESCRIBE_STMT:
+    case QUERY_NODE_RESET_QUERY_CACHE_STMT:
+    case QUERY_NODE_COMPACT_STMT:
+    case QUERY_NODE_CREATE_FUNCTION_STMT:
+    case QUERY_NODE_DROP_FUNCTION_STMT:
+      break;
+    case QUERY_NODE_CREATE_STREAM_STMT:
+      return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt);
+    case QUERY_NODE_DROP_STREAM_STMT:
+    case QUERY_NODE_MERGE_VGROUP_STMT:
+    case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
+    case QUERY_NODE_SPLIT_VGROUP_STMT:
+    case QUERY_NODE_SYNCDB_STMT:
+    case QUERY_NODE_GRANT_STMT:
+    case QUERY_NODE_REVOKE_STMT:
+    case QUERY_NODE_SHOW_DNODES_STMT:
+      return collectMetaKeyFromShowDnodes(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_MNODES_STMT:
+      return collectMetaKeyFromShowMnodes(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_MODULES_STMT:
+      return collectMetaKeyFromShowModules(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_QNODES_STMT:
+      return collectMetaKeyFromShowQnodes(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_SNODES_STMT:
+      return collectMetaKeyFromShowSnodes(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_BNODES_STMT:
+      return collectMetaKeyFromShowBnodes(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_CLUSTER_STMT:
+      break;
+    case QUERY_NODE_SHOW_DATABASES_STMT:
+      return collectMetaKeyFromShowDatabases(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_FUNCTIONS_STMT:
+      return collectMetaKeyFromShowFunctions(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_INDEXES_STMT:
+      return collectMetaKeyFromShowIndexes(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_STABLES_STMT:
+      return collectMetaKeyFromShowStables(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_STREAMS_STMT:
+      return collectMetaKeyFromShowStreams(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_TABLES_STMT:
+      return collectMetaKeyFromShowTables(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_USERS_STMT:
+      return collectMetaKeyFromShowUsers(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_LICENCE_STMT:
+      return collectMetaKeyFromShowLicence(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_VGROUPS_STMT:
+      return collectMetaKeyFromShowVgroups(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_TOPICS_STMT:
+      return collectMetaKeyFromShowTopics(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_SHOW_CONSUMERS_STMT:
+    case QUERY_NODE_SHOW_SUBSCRIBES_STMT:
+    case QUERY_NODE_SHOW_SMAS_STMT:
+    case QUERY_NODE_SHOW_CONFIGS_STMT:
+    case QUERY_NODE_SHOW_CONNECTIONS_STMT:
+    case QUERY_NODE_SHOW_QUERIES_STMT:
+    case QUERY_NODE_SHOW_VNODES_STMT:
+    case QUERY_NODE_SHOW_APPS_STMT:
+    case QUERY_NODE_SHOW_SCORES_STMT:
+    case QUERY_NODE_SHOW_VARIABLE_STMT:
+    case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
+    case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
+    case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
+      break;
+    case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
+      return collectMetaKeyFromShowTransactions(pCxt, (SShowStmt*)pStmt);
+    case QUERY_NODE_KILL_CONNECTION_STMT:
+    case QUERY_NODE_KILL_QUERY_STMT:
+    case QUERY_NODE_KILL_TRANSACTION_STMT:
+    default:
+      break;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery) {
+  SCollectMetaKeyCxt cxt = {.pParseCxt = pParseCxt, .pMetaCache = taosMemoryCalloc(1, sizeof(SParseMetaCache))};
+  if (NULL == cxt.pMetaCache) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  int32_t code = collectMetaKeyFromQuery(&cxt, pQuery->pRoot);
+  if (TSDB_CODE_SUCCESS == code) {
+    TSWAP(pQuery->pMetaCache, cxt.pMetaCache);
+  }
+  destroyCollectMetaKeyCxt(&cxt);
+  return code;
+}

source/libs/parser/src/parAuthenticator.c

@@ -19,22 +19,29 @@
 
 typedef struct SAuthCxt {
   SParseContext*   pParseCxt;
+  SParseMetaCache* pMetaCache;
   int32_t          errCode;
 } SAuthCxt;
 
 static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt);
 
-static int32_t checkAuth(SParseContext* pCxt, const char* pDbName, AUTH_TYPE type) {
+static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, AUTH_TYPE type) {
-  if (pCxt->isSuperUser) {
+  SParseContext* pParseCxt = pCxt->pParseCxt;
+  if (pParseCxt->isSuperUser) {
     return TSDB_CODE_SUCCESS;
   }
   SName name;
-  tNameSetDbName(&name, pCxt->acctId, pDbName, strlen(pDbName));
+  tNameSetDbName(&name, pParseCxt->acctId, pDbName, strlen(pDbName));
   char dbFname[TSDB_DB_FNAME_LEN] = {0};
   tNameGetFullDbName(&name, dbFname);
+  int32_t code = TSDB_CODE_SUCCESS;
   bool    pass = false;
-  int32_t code =
+  if (NULL != pCxt->pMetaCache) {
-      catalogChkAuth(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, pCxt->pUser, dbFname, type, &pass);
+    code = getUserAuthFromCache(pCxt->pMetaCache, pParseCxt->pUser, dbFname, type, &pass);
+  } else {
+    code = catalogChkAuth(pParseCxt->pCatalog, pParseCxt->pTransporter, &pParseCxt->mgmtEpSet, pParseCxt->pUser,
+                          dbFname, type, &pass);
+  }
   return TSDB_CODE_SUCCESS == code ? (pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
 }
 
@@ -45,7 +52,7 @@ static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) {
 static EDealRes authSelectImpl(SNode* pNode, void* pContext) {
   SAuthCxt* pCxt = pContext;
   if (QUERY_NODE_REAL_TABLE == nodeType(pNode)) {
-    pCxt->errCode = checkAuth(pCxt->pParseCxt, ((SRealTableNode*)pNode)->table.dbName, AUTH_TYPE_READ);
+    pCxt->errCode = checkAuth(pCxt, ((SRealTableNode*)pNode)->table.dbName, AUTH_TYPE_READ);
     return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
   } else if (QUERY_NODE_TEMP_TABLE == nodeType(pNode)) {
     return authSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery);
@@ -79,87 +86,8 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
       return authSetOperator(pCxt, (SSetOperator*)pStmt);
     case QUERY_NODE_SELECT_STMT:
       return authSelect(pCxt, (SSelectStmt*)pStmt);
-    case QUERY_NODE_CREATE_DATABASE_STMT:
+    case QUERY_NODE_DROP_USER_STMT:
-    case QUERY_NODE_DROP_DATABASE_STMT:
-    case QUERY_NODE_ALTER_DATABASE_STMT:
-    case QUERY_NODE_CREATE_TABLE_STMT:
-    case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
-    case QUERY_NODE_CREATE_MULTI_TABLE_STMT:
-    case QUERY_NODE_DROP_TABLE_CLAUSE:
-    case QUERY_NODE_DROP_TABLE_STMT:
-    case QUERY_NODE_DROP_SUPER_TABLE_STMT:
-    case QUERY_NODE_ALTER_TABLE_STMT:
-    case QUERY_NODE_CREATE_USER_STMT:
-    case QUERY_NODE_ALTER_USER_STMT:
-      break;
-    case QUERY_NODE_DROP_USER_STMT: {
       return authDropUser(pCxt, (SDropUserStmt*)pStmt);
-    }
-    case QUERY_NODE_USE_DATABASE_STMT:
-    case QUERY_NODE_CREATE_DNODE_STMT:
-    case QUERY_NODE_DROP_DNODE_STMT:
-    case QUERY_NODE_ALTER_DNODE_STMT:
-    case QUERY_NODE_CREATE_INDEX_STMT:
-    case QUERY_NODE_DROP_INDEX_STMT:
-    case QUERY_NODE_CREATE_QNODE_STMT:
-    case QUERY_NODE_DROP_QNODE_STMT:
-    case QUERY_NODE_CREATE_BNODE_STMT:
-    case QUERY_NODE_DROP_BNODE_STMT:
-    case QUERY_NODE_CREATE_SNODE_STMT:
-    case QUERY_NODE_DROP_SNODE_STMT:
-    case QUERY_NODE_CREATE_MNODE_STMT:
-    case QUERY_NODE_DROP_MNODE_STMT:
-    case QUERY_NODE_CREATE_TOPIC_STMT:
-    case QUERY_NODE_DROP_TOPIC_STMT:
-    case QUERY_NODE_ALTER_LOCAL_STMT:
-    case QUERY_NODE_EXPLAIN_STMT:
-    case QUERY_NODE_DESCRIBE_STMT:
-    case QUERY_NODE_RESET_QUERY_CACHE_STMT:
-    case QUERY_NODE_COMPACT_STMT:
-    case QUERY_NODE_CREATE_FUNCTION_STMT:
-    case QUERY_NODE_DROP_FUNCTION_STMT:
-    case QUERY_NODE_CREATE_STREAM_STMT:
-    case QUERY_NODE_DROP_STREAM_STMT:
-    case QUERY_NODE_MERGE_VGROUP_STMT:
-    case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
-    case QUERY_NODE_SPLIT_VGROUP_STMT:
-    case QUERY_NODE_SYNCDB_STMT:
-    case QUERY_NODE_GRANT_STMT:
-    case QUERY_NODE_REVOKE_STMT:
-    case QUERY_NODE_SHOW_DNODES_STMT:
-    case QUERY_NODE_SHOW_MNODES_STMT:
-    case QUERY_NODE_SHOW_MODULES_STMT:
-    case QUERY_NODE_SHOW_QNODES_STMT:
-    case QUERY_NODE_SHOW_SNODES_STMT:
-    case QUERY_NODE_SHOW_BNODES_STMT:
-    case QUERY_NODE_SHOW_CLUSTER_STMT:
-    case QUERY_NODE_SHOW_DATABASES_STMT:
-    case QUERY_NODE_SHOW_FUNCTIONS_STMT:
-    case QUERY_NODE_SHOW_INDEXES_STMT:
-    case QUERY_NODE_SHOW_STABLES_STMT:
-    case QUERY_NODE_SHOW_STREAMS_STMT:
-    case QUERY_NODE_SHOW_TABLES_STMT:
-    case QUERY_NODE_SHOW_USERS_STMT:
-    case QUERY_NODE_SHOW_LICENCE_STMT:
-    case QUERY_NODE_SHOW_VGROUPS_STMT:
-    case QUERY_NODE_SHOW_TOPICS_STMT:
-    case QUERY_NODE_SHOW_CONSUMERS_STMT:
-    case QUERY_NODE_SHOW_SUBSCRIBES_STMT:
-    case QUERY_NODE_SHOW_SMAS_STMT:
-    case QUERY_NODE_SHOW_CONFIGS_STMT:
-    case QUERY_NODE_SHOW_CONNECTIONS_STMT:
-    case QUERY_NODE_SHOW_QUERIES_STMT:
-    case QUERY_NODE_SHOW_VNODES_STMT:
-    case QUERY_NODE_SHOW_APPS_STMT:
-    case QUERY_NODE_SHOW_SCORES_STMT:
-    case QUERY_NODE_SHOW_VARIABLE_STMT:
-    case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
-    case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
-    case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
-    case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
-    case QUERY_NODE_KILL_CONNECTION_STMT:
-    case QUERY_NODE_KILL_QUERY_STMT:
-    case QUERY_NODE_KILL_TRANSACTION_STMT:
     default:
       break;
   }
@@ -168,6 +96,6 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
 }
 
 int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery) {
-  SAuthCxt cxt = {.pParseCxt = pParseCxt, .errCode = TSDB_CODE_SUCCESS};
+  SAuthCxt cxt = {.pParseCxt = pParseCxt, .pMetaCache = pQuery->pMetaCache, .errCode = TSDB_CODE_SUCCESS};
   return authQuery(&cxt, pQuery->pRoot);
 }

source/libs/parser/src/parTranslater.c

@@ -152,7 +152,7 @@ static int32_t getDBVgInfoImpl(STranslateContext* pCxt, const SName* pName, SArr
   tNameGetFullDbName(pName, fullDbName);
   int32_t code = TSDB_CODE_SUCCESS;
   if (pParCxt->async) {
-    code = getDBVgInfoFromCache(pCxt->pMetaCache, fullDbName, pVgInfo);
+    code = getDbVgInfoFromCache(pCxt->pMetaCache, fullDbName, pVgInfo);
   } else {
     code = collectUseDatabaseImpl(fullDbName, pCxt->pDbs);
     if (TSDB_CODE_SUCCESS == code) {
@@ -177,7 +177,7 @@ static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pNam
   SParseContext* pParCxt = pCxt->pParseCxt;
   int32_t        code = TSDB_CODE_SUCCESS;
   if (pParCxt->async) {
-    code = getTableHashVgroupFromCache(pCxt->pMetaCache, pName, pInfo);
+    code = getTableVgroupFromCache(pCxt->pMetaCache, pName, pInfo);
   } else {
     code = collectUseDatabase(pName, pCxt->pDbs);
     if (TSDB_CODE_SUCCESS == code) {
@@ -205,7 +205,7 @@ static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int
   SParseContext* pParCxt = pCxt->pParseCxt;
   int32_t        code = TSDB_CODE_SUCCESS;
   if (pParCxt->async) {
-    code = getDBVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum);
+    code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum);
   } else {
     code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs);
     if (TSDB_CODE_SUCCESS == code) {
@@ -226,7 +226,7 @@ static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo
   tNameGetFullDbName(&name, dbFname);
   int32_t code = TSDB_CODE_SUCCESS;
   if (pParCxt->async) {
-    code = getDBCfgFromCache(pCxt->pMetaCache, dbFname, pInfo);
+    code = getDbCfgFromCache(pCxt->pMetaCache, dbFname, pInfo);
   } else {
     code = collectUseDatabaseImpl(dbFname, pCxt->pDbs);
     if (TSDB_CODE_SUCCESS == code) {
@@ -239,6 +239,27 @@ static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo
   return code;
 }
 
+static int32_t getUdfInfo(STranslateContext* pCxt, SFunctionNode* pFunc) {
+  SParseContext* pParCxt = pCxt->pParseCxt;
+  SFuncInfo      funcInfo = {0};
+  int32_t        code = TSDB_CODE_SUCCESS;
+  if (pParCxt->async) {
+    code = getUdfInfoFromCache(pCxt->pMetaCache, pFunc->functionName, &funcInfo);
+  } else {
+    code = catalogGetUdfInfo(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pFunc->functionName,
+                             &funcInfo);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    pFunc->funcType = FUNCTION_TYPE_UDF;
+    pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == funcInfo.funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID;
+    pFunc->node.resType.type = funcInfo.outputType;
+    pFunc->node.resType.bytes = funcInfo.outputLen;
+    pFunc->udfBufSize = funcInfo.bufSize;
+    tFreeSFuncInfo(&funcInfo);
+  }
+  return code;
+}
+
 static int32_t initTranslateContext(SParseContext* pParseCxt, SParseMetaCache* pMetaCache, STranslateContext* pCxt) {
   pCxt->pParseCxt = pParseCxt;
   pCxt->errCode = TSDB_CODE_SUCCESS;
@@ -873,12 +894,11 @@ static bool hasInvalidFuncNesting(SNodeList* pParameterList) {
 }
 
 static int32_t getFuncInfo(STranslateContext* pCxt, SFunctionNode* pFunc) {
-  SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog,
+  int32_t code = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len);
-                               .pRpc = pCxt->pParseCxt->pTransporter,
+  if (TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION == code) {
-                               .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet,
+    code = getUdfInfo(pCxt, pFunc);
-                               .pErrBuf = pCxt->msgBuf.buf,
+  }
-                               .errBufLen = pCxt->msgBuf.len};
+  return code;
-  return fmGetFuncInfo(&param, pFunc);
 }
 
 static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
@@ -1212,7 +1232,6 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea
   int32_t code = TSDB_CODE_SUCCESS;
   SArray* vgroupList = NULL;
   if ('\0' != pRealTable->qualDbName[0]) {
-    // todo release after mnode can be processed
     if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) {
       code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList);
     }
@@ -1220,7 +1239,6 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea
     code = getDBVgInfoImpl(pCxt, pName, &vgroupList);
   }
 
-  // todo release after mnode can be processed
   if (TSDB_CODE_SUCCESS == code) {
     code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList);
   }

source/libs/parser/src/parUtil.c

@@ -15,6 +15,9 @@
 
 #include "parUtil.h"
 #include "cJSON.h"
+#include "querynodes.h"
+
+#define USER_AUTH_KEY_MAX_LEN TSDB_USER_LEN + TSDB_DB_FNAME_LEN + 2
 
 static char* getSyntaxErrFormat(int32_t errCode) {
   switch (errCode) {
@@ -255,17 +258,8 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) {
   return pTableMeta->tableInfo;
 }
 
-static uint32_t getTableMetaSize(const STableMeta* pTableMeta) {
-  int32_t totalCols = 0;
-  if (pTableMeta->tableInfo.numOfColumns >= 0) {
-    totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags;
-  }
-
-  return sizeof(STableMeta) + totalCols * sizeof(SSchema);
-}
-
 STableMeta* tableMetaDup(const STableMeta* pTableMeta) {
-  size_t size = getTableMetaSize(pTableMeta);
+  size_t size = TABLE_META_SIZE(pTableMeta);
 
   STableMeta* p = taosMemoryMalloc(size);
   memcpy(p, pTableMeta, size);
@@ -449,6 +443,26 @@ end:
   return retCode;
 }
 
+static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) {
+  return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type);
+}
+
+static int32_t userAuthToStringExt(const char* pUser, const char* pDbFName, AUTH_TYPE type, char* pStr) {
+  return sprintf(pStr, "%s*%s*%d", pUser, pDbFName, type);
+}
+
+static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) {
+  char* p1 = strchr(pStr, '*');
+  strncpy(pUserAuth->user, pStr, p1 - pStr);
+  ++p1;
+  char* p2 = strchr(p1, '*');
+  strncpy(pUserAuth->dbFName, p1, p2 - p1);
+  ++p2;
+  char buf[10] = {0};
+  strncpy(buf, p2, len - (p2 - pStr));
+  pUserAuth->type = taosStr2Int32(buf, NULL, 10);
+}
+
 static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) {
   if (NULL != pTablesHash) {
     *pTables = taosArrayInit(taosHashGetSize(pTablesHash), sizeof(SName));
@@ -503,6 +517,44 @@ static int32_t buildTableVgroupReq(SHashObj* pTableVgroupHash, SArray** pTableVg
 
 static int32_t buildDbCfgReq(SHashObj* pDbCfgHash, SArray** pDbCfg) { return buildDbReq(pDbCfgHash, pDbCfg); }
 
+static int32_t buildUserAuthReq(SHashObj* pUserAuthHash, SArray** pUserAuth) {
+  if (NULL != pUserAuthHash) {
+    *pUserAuth = taosArrayInit(taosHashGetSize(pUserAuthHash), sizeof(SUserAuthInfo));
+    if (NULL == *pUserAuth) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    void* p = taosHashIterate(pUserAuthHash, NULL);
+    while (NULL != p) {
+      size_t        len = 0;
+      char*         pKey = taosHashGetKey(p, &len);
+      SUserAuthInfo userAuth = {0};
+      stringToUserAuth(pKey, len, &userAuth);
+      taosArrayPush(*pUserAuth, &userAuth);
+      p = taosHashIterate(pUserAuthHash, p);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildUdfReq(SHashObj* pUdfHash, SArray** pUdf) {
+  if (NULL != pUdfHash) {
+    *pUdf = taosArrayInit(taosHashGetSize(pUdfHash), TSDB_FUNC_NAME_LEN);
+    if (NULL == *pUdf) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    void* p = taosHashIterate(pUdfHash, NULL);
+    while (NULL != p) {
+      size_t len = 0;
+      char*  pFunc = taosHashGetKey(p, &len);
+      char   func[TSDB_FUNC_NAME_LEN] = {0};
+      strncpy(func, pFunc, len);
+      taosArrayPush(*pUdf, func);
+      p = taosHashIterate(pUdfHash, p);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) {
   int32_t code = buildTableMetaReq(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta);
   if (TSDB_CODE_SUCCESS == code) {
@@ -512,7 +564,13 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog
     code = buildTableVgroupReq(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = buildDbCfgReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbCfg);
+    code = buildDbCfgReq(pMetaCache->pDbCfg, &pCatalogReq->pDbCfg);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildUserAuthReq(pMetaCache->pUserAuth, &pCatalogReq->pUser);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildUdfReq(pMetaCache->pUdf, &pCatalogReq->pUdf);
   }
   return code;
 }
@@ -568,6 +626,31 @@ static int32_t putDbCfgToCache(const SArray* pDbCfgReq, const SArray* pDbCfgData
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUserAuthData, SHashObj* pUserAuth) {
+  int32_t nvgs = taosArrayGetSize(pUserAuthReq);
+  for (int32_t i = 0; i < nvgs; ++i) {
+    SUserAuthInfo* pUser = taosArrayGet(pUserAuthReq, i);
+    char           key[USER_AUTH_KEY_MAX_LEN] = {0};
+    int32_t        len = userAuthToStringExt(pUser->user, pUser->dbFName, pUser->type, key);
+    if (TSDB_CODE_SUCCESS != taosHashPut(pUserAuth, key, len, taosArrayGet(pUserAuthData, i), sizeof(bool))) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHashObj* pUdf) {
+  int32_t num = taosArrayGetSize(pUdfReq);
+  for (int32_t i = 0; i < num; ++i) {
+    char*      pFunc = taosArrayGet(pUdfReq, i);
+    SFuncInfo* pInfo = taosArrayGet(pUdfData, i);
+    if (TSDB_CODE_SUCCESS != taosHashPut(pUdf, pFunc, strlen(pFunc), &pInfo, POINTER_BYTES)) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) {
   int32_t code = putTableMetaToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, pMetaCache->pTableMeta);
   if (TSDB_CODE_SUCCESS == code) {
@@ -579,54 +662,161 @@ int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMet
   if (TSDB_CODE_SUCCESS == code) {
     code = putDbCfgToCache(pCatalogReq->pDbCfg, pMetaData->pDbCfg, pMetaCache->pDbCfg);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = putUserAuthToCache(pCatalogReq->pUser, pMetaData->pUser, pMetaCache->pUserAuth);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = putUdfToCache(pCatalogReq->pUdf, pMetaData->pUdfList, pMetaCache->pUdf);
+  }
   return code;
 }
 
-int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) {
+static int32_t reserveTableReqInCache(int32_t acctId, const char* pDb, const char* pTable, SHashObj** pTables) {
-  if (NULL == pMetaCache->pTableMeta) {
+  if (NULL == *pTables) {
-    pMetaCache->pTableMeta = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+    *pTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
-    if (NULL == pMetaCache->pTableMeta) {
+    if (NULL == *pTables) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
   char    fullName[TSDB_TABLE_FNAME_LEN];
   int32_t len = snprintf(fullName, sizeof(fullName), "%d.%s.%s", acctId, pDb, pTable);
-  return taosHashPut(pMetaCache->pTableMeta, fullName, len, &len, POINTER_BYTES);
+  return taosHashPut(*pTables, fullName, len, &pTables, POINTER_BYTES);
+}
+
+int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) {
+  return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableMeta);
 }
 
 int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) {
   char fullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(pName, fullName);
-  *pMeta = taosHashGet(pMetaCache->pTableMeta, fullName, strlen(fullName));
+  STableMeta** pRes = taosHashGet(pMetaCache->pTableMeta, fullName, strlen(fullName));
-  return NULL == *pMeta ? TSDB_CODE_PAR_INTERNAL_ERROR : TSDB_CODE_SUCCESS;
+  if (NULL == pRes || NULL == *pRes) {
+    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  }
+  *pMeta = tableMetaDup(*pRes);
+  if (NULL == *pMeta) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return TSDB_CODE_SUCCESS;
 }
 
-int32_t getDBVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo) {
+static int32_t reserveDbReqInCache(int32_t acctId, const char* pDb, SHashObj** pDbs) {
-  *pVgInfo = taosHashGet(pMetaCache->pDbVgroup, pDbFName, strlen(pDbFName));
+  if (NULL == *pDbs) {
-  return NULL == *pVgInfo ? TSDB_CODE_PAR_INTERNAL_ERROR : TSDB_CODE_SUCCESS;
+    *pDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+    if (NULL == *pDbs) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  char    fullName[TSDB_TABLE_FNAME_LEN];
+  int32_t len = snprintf(fullName, sizeof(fullName), "%d.%s", acctId, pDb);
+  return taosHashPut(*pDbs, fullName, len, &pDbs, POINTER_BYTES);
 }
 
-int32_t getTableHashVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup) {
+int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache) {
+  return reserveDbReqInCache(acctId, pDb, &pMetaCache->pDbVgroup);
+}
+
+int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo) {
+  SArray** pRes = taosHashGet(pMetaCache->pDbVgroup, pDbFName, strlen(pDbFName));
+  if (NULL == pRes) {
+    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  }
+  // *pRes is null, which is a legal value, indicating that the user DB has not been created
+  if (NULL != *pRes) {
+    *pVgInfo = taosArrayDup(*pRes);
+    if (NULL == *pVgInfo) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) {
+  return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableVgroup);
+}
+
+int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup) {
   char fullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(pName, fullName);
-  SVgroupInfo* pInfo = taosHashGet(pMetaCache->pTableVgroup, fullName, strlen(fullName));
+  SVgroupInfo** pRes = taosHashGet(pMetaCache->pTableVgroup, fullName, strlen(fullName));
-  if (NULL == pInfo) {
+  if (NULL == pRes || NULL == *pRes) {
     return TSDB_CODE_PAR_INTERNAL_ERROR;
   }
-  memcpy(pVgroup, pInfo, sizeof(SVgroupInfo));
+  memcpy(pVgroup, *pRes, sizeof(SVgroupInfo));
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t getDBVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,
+int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache) {
+  return reserveDbReqInCache(acctId, pDb, &pMetaCache->pDbCfg);
+}
+
+int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,
                                 int32_t* pTableNum) {
+  SDbInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName));
+  if (NULL == pRes || NULL == *pRes) {
     return TSDB_CODE_PAR_INTERNAL_ERROR;
   }
-
+  *pVersion = (*pRes)->vgVer;
-int32_t getDBCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo) {
+  *pDbId = (*pRes)->dbId;
-  SDbCfgInfo* pDbCfg = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName));
+  *pTableNum = (*pRes)->tbNum;
-  if (NULL == pDbCfg) {
+  return TSDB_CODE_SUCCESS;
-    return TSDB_CODE_PAR_INTERNAL_ERROR;
+}
-  }
+
-  memcpy(pInfo, pDbCfg, sizeof(SDbCfgInfo));
+int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache) {
+  return reserveDbReqInCache(acctId, pDb, &pMetaCache->pDbCfg);
+}
+
+int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo) {
+  SDbCfgInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName));
+  if (NULL == pRes || NULL == *pRes) {
+    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  }
+  memcpy(pInfo, *pRes, sizeof(SDbCfgInfo));
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type,
+                               SParseMetaCache* pMetaCache) {
+  if (NULL == pMetaCache->pUserAuth) {
+    pMetaCache->pUserAuth = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+    if (NULL == pMetaCache->pUserAuth) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  char    key[USER_AUTH_KEY_MAX_LEN] = {0};
+  int32_t len = userAuthToString(acctId, pUser, pDb, type, key);
+  bool    pass = false;
+  return taosHashPut(pMetaCache->pUserAuth, key, len, &pass, sizeof(pass));
+}
+
+int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type,
+                             bool* pPass) {
+  char    key[USER_AUTH_KEY_MAX_LEN] = {0};
+  int32_t len = userAuthToStringExt(pUser, pDbFName, type, key);
+  bool*   pRes = taosHashGet(pMetaCache->pUserAuth, key, len);
+  if (NULL == pRes) {
+    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  }
+  *pPass = *pRes;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache) {
+  if (NULL == pMetaCache->pUdf) {
+    pMetaCache->pUdf = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+    if (NULL == pMetaCache->pUdf) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  return taosHashPut(pMetaCache->pUdf, pFunc, strlen(pFunc), &pMetaCache, POINTER_BYTES);
+}
+
+int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo) {
+  SFuncInfo** pRes = taosHashGet(pMetaCache->pUdf, pFunc, strlen(pFunc));
+  if (NULL == pRes || NULL == *pRes) {
+    return TSDB_CODE_PAR_INTERNAL_ERROR;
+  }
+  memcpy(pInfo, *pRes, sizeof(SFuncInfo));
   return TSDB_CODE_SUCCESS;
 }