Merge branch '3.0' into opti/TD-30681

.github/workflows/taoskeeper-ci.yml

@@ -0,0 +1,58 @@
+name: TaosKeeper CI
+
+on:
+  push:
+    paths:
+      - tools/keeper/**
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Run unit tests
+
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.18
+
+      - name: Install system dependencies
+        run: |
+          sudo apt update -y
+          sudo apt install -y build-essential cmake libgeos-dev
+
+      - name: Install TDengine
+        run: |
+          mkdir debug
+          cd debug
+          cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true
+          make -j 4
+          sudo make install
+          which taosd
+          which taosadapter
+          which taoskeeper
+
+      - name: Start taosd
+        run: |
+          cp /etc/taos/taos.cfg ./
+          sudo echo "supportVnodes 256" >> taos.cfg
+          nohup sudo taosd -c taos.cfg &
+
+      - name: Start taosadapter
+        run: nohup sudo taosadapter &
+
+      - name: Run tests with coverage
+        working-directory: tools/keeper
+        run: |
+          go mod tidy
+          go test -v -coverpkg=./... -coverprofile=coverage.out ./...
+          go tool cover -func=coverage.out
+
+      - name: Clean up
+        if: always()
+        run: |
+          if pgrep taosd; then sudo pkill taosd; fi
+          if pgrep taosadapter; then sudo pkill taosadapter; fi

README-CN.md

@@ -348,7 +348,7 @@ TDengine 提供了丰富的应用程序开发接口，其中包括 C/C++、Java
 
 # 成为社区贡献者
 
-点击 [这里](https://www.taosdata.com/cn/contributor/)，了解如何成为 TDengine 的贡献者。
+点击 [这里](https://www.taosdata.com/contributor)，了解如何成为 TDengine 的贡献者。
 
 # 加入技术交流群
 

cmake/cmake.define

@@ -1,6 +1,7 @@
 cmake_minimum_required(VERSION 3.0)
 set(CMAKE_VERBOSE_MAKEFILE FALSE)
 set(TD_BUILD_TAOSA_INTERNAL FALSE)
+set(TD_BUILD_KEEPER_INTERNAL FALSE)
 
 # set output directory
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
@@ -57,6 +58,19 @@ IF(TD_BUILD_HTTP)
     ADD_DEFINITIONS(-DHTTP_EMBEDDED)
 ENDIF()
 
+IF("${BUILD_KEEPER}" STREQUAL "")
+    SET(TD_BUILD_KEEPER FALSE)
+ELSEIF(${BUILD_KEEPER} MATCHES "false")
+    SET(TD_BUILD_KEEPER FALSE)
+ELSEIF(${BUILD_KEEPER} MATCHES "true")
+    SET(TD_BUILD_KEEPER TRUE)
+ELSEIF(${BUILD_KEEPER} MATCHES "internal")
+    SET(TD_BUILD_KEEPER FALSE)
+    SET(TD_BUILD_KEEPER_INTERNAL TRUE)
+ELSE()
+    SET(TD_BUILD_KEEPER FALSE)
+ENDIF()
+
 IF("${BUILD_TOOLS}" STREQUAL "")
     IF(TD_LINUX)
         IF(TD_ARM_32)

docs/en/04-get-started/01-docker.md

@@ -75,9 +75,9 @@ taos>
 
 ## TDegnine Graphic User Interface
 
-From TDengine 3.3.0.0, there is a new componenet called `taos-explorer` added in the TDengine docker image. You can use it to manage the databases, super tables, child tables, and data in your TDengine system. There are also some features only available in TDengine Enterprise Edition, please contact TDengine sales team in case you need these features.
+From TDengine 3.3.0.0, there is a new component called `taos-explorer` added in the TDengine docker image. You can use it to manage the databases, super tables, child tables, and data in your TDengine system. There are also some features only available in TDengine Enterprise Edition, please contact TDengine sales team in case you need these features.
 
-To use taos-explorer in the container, you need to access the host port mapped from container port 6060. Assuming the host name is abc.com, and the port used on host is 6060, you need to access `http://abc.com:6060`. taos-explorer uses port 6060 by default in the container. When you use it the first time, you need to register with your enterprise email, then can logon using your user name and password in the TDengine database management system.
+To use taos-explorer in the container, you need to access the host port mapped from container port 6060. Assuming the host name is abc.com, and the port used on host is 6060, you need to access `http://abc.com:6060`. taos-explorer uses port 6060 by default in the container. The default username and password to log in to the TDengine Database Management System is "root/taosdata".
 
 ## Test data insert performance
 

docs/en/14-reference/01-components/10-taosbenchmark

@@ -364,6 +364,9 @@ The configuration parameters for specifying super table tag columns and data col
 - **min**: The minimum value of the column/label of the data type. The generated value will equal or large than the minimum value.
 
 - **max**: The maximum value of the column/label of the data type. The generated value will less than the maximum value.
+
+- **scalingFactor**: Floating-point precision enhancement factor, which takes effect only when the data type is float/double. It has a valid range of positive integers from 1 to 1,000,000. It is used to enhance the precision of generated floating-point numbers, particularly when the min or max values are small. This property enhances the precision after the decimal point by powers of 10: scalingFactor of 10 indicates an enhancement of 1 decimal precision, 100 indicates an enhancement of 2 decimal precision, and so on.
+
 - **fun**: This column of data is filled with functions. Currently, only the sin and cos functions are supported. The input parameter is the timestamp and converted to an angle value. The conversion formula is: angle x=input time column ts value % 360. At the same time, it supports coefficient adjustment and random fluctuation factor adjustment, presented in a fixed format expression, such as fun="10\*sin(x)+100\*random(5)", where x represents the angle, ranging from 0 to 360 degrees, and the growth step size is consistent with the time column step size. 10 represents the coefficient of multiplication, 100 represents the coefficient of addition or subtraction, and 5 represents the fluctuation range within a random range of 5%. The currently supported data types are int, bigint, float, and double. Note: The expression is fixed and cannot be reversed.
 
 - **values**: The value field of the nchar/binary column/label, which will be chosen randomly from the values.

docs/en/14-reference/03-taos-sql/21-node.md

@@ -27,11 +27,15 @@ The preceding SQL command shows all dnodes in the cluster with the ID, endpoint,
 ## Delete a DNODE
 
 ```sql
-DROP DNODE dnode_id
+DROP DNODE dnode_id [force] [unsafe]
 ```
 
 Note that deleting a dnode does not stop its process. You must stop the process after the dnode is deleted.
 
+Only online node is allowed to be deleted. Drop is executed with force option if the offline node need to be deleted.
+
+Drop is executed with unsafe option if the node with single replica is offline, and the data on it is not able to be restored.
+
 ## Modify Dnode Configuration
 
 ```sql

docs/examples/java/src/main/java/com/taos/example/ConsumerLoopFull.java

@@ -1,8 +1,9 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.taosdata.jdbc.TSDBDriver;
 import com.taosdata.jdbc.tmq.*;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.*;
 import java.time.Duration;
@@ -60,7 +61,7 @@ public class ConsumerLoopFull {
 // ANCHOR_END: create_consumer
     }
 
-    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: poll_data_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -73,7 +74,7 @@ public class ConsumerLoopFull {
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
             }
         } catch (Exception ex) {
@@ -91,7 +92,7 @@ public class ConsumerLoopFull {
 // ANCHOR_END: poll_data_code_piece
     }
 
-    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: consumer_seek
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -99,7 +100,7 @@ public class ConsumerLoopFull {
             consumer.subscribe(topics);
             System.out.println("Subscribe topics successfully.");
             Set<TopicPartition> assignment = consumer.assignment();
-            System.out.println("Now assignment: " + JSON.toJSONString(assignment));
+            System.out.println("Now assignment: " + JsonUtil.getObjectMapper().writeValueAsString(assignment));
 
             ConsumerRecords<ResultBean> records = ConsumerRecords.emptyRecord();
             // make sure we have got some data
@@ -125,7 +126,7 @@ public class ConsumerLoopFull {
     }
 
 
-    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: commit_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -135,7 +136,7 @@ public class ConsumerLoopFull {
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
                 if (!records.isEmpty()) {
                     // after processing the data, commit the offset manually

docs/examples/java/src/main/java/com/taos/example/ConsumerLoopImp.java

@@ -1,7 +1,7 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
 import com.taosdata.jdbc.TSDBDriver;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -31,7 +31,11 @@ public class ConsumerLoopImp {
         final AbsConsumerLoop consumerLoop = new AbsConsumerLoop() {
             @Override
             public void process(ResultBean result) {
-                System.out.println("data: " + JSON.toJSONString(result));
+                try{
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(result));
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
             }
         };
 

docs/examples/java/src/main/java/com/taos/example/WsConsumerLoopFull.java

@@ -1,8 +1,9 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.taosdata.jdbc.TSDBDriver;
 import com.taosdata.jdbc.tmq.*;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.*;
 import java.time.Duration;
@@ -60,7 +61,7 @@ public class WsConsumerLoopFull {
 // ANCHOR_END: create_consumer
     }
 
-    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: poll_data_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -73,7 +74,7 @@ public class WsConsumerLoopFull {
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
             }
         } catch (Exception ex) {
@@ -91,7 +92,7 @@ public class WsConsumerLoopFull {
 // ANCHOR_END: poll_data_code_piece
     }
 
-    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: consumer_seek
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -99,7 +100,7 @@ public class WsConsumerLoopFull {
             consumer.subscribe(topics);
             System.out.println("Subscribe topics successfully.");
             Set<TopicPartition> assignment = consumer.assignment();
-            System.out.println("Now assignment: " + JSON.toJSONString(assignment));
+            System.out.println("Now assignment: " + JsonUtil.getObjectMapper().writeValueAsString(assignment));
 
             ConsumerRecords<ResultBean> records = ConsumerRecords.emptyRecord();
             // make sure we have got some data
@@ -125,7 +126,7 @@ public class WsConsumerLoopFull {
     }
 
 
-    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException {
+    public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
 // ANCHOR: commit_code_piece
         List<String> topics = Collections.singletonList("topic_meters");
         try {
@@ -135,7 +136,7 @@ public class WsConsumerLoopFull {
                 for (ConsumerRecord<ResultBean> record : records) {
                     ResultBean bean = record.value();
                     // Add your data processing logic here
-                    System.out.println("data: " + JSON.toJSONString(bean));
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
                 }
                 if (!records.isEmpty()) {
                     // after processing the data, commit the offset manually

docs/examples/java/src/main/java/com/taos/example/WsConsumerLoopImp.java

@@ -1,7 +1,7 @@
 package com.taos.example;
 
-import com.alibaba.fastjson.JSON;
 import com.taosdata.jdbc.TSDBDriver;
+import com.taosdata.jdbc.utils.JsonUtil;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -28,7 +28,11 @@ public abstract class WsConsumerLoopImp {
         final AbsConsumerLoop consumerLoop = new AbsConsumerLoop() {
             @Override
             public void process(ResultBean result) {
-                System.out.println("data: " + JSON.toJSONString(result));
+                try{
+                    System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(result));
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
             }
         };
 

docs/examples/java/src/main/java/com/taos/example/highvolume/DataBaseMonitor.java

@@ -13,6 +13,9 @@ public class DataBaseMonitor {
     public DataBaseMonitor init() throws SQLException {
         if (conn == null) {
             String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
+            if (jdbcURL == null || jdbcURL == ""){
+                jdbcURL = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+            }
             conn = DriverManager.getConnection(jdbcURL);
             stmt = conn.createStatement();
         }

docs/examples/java/src/main/java/com/taos/example/highvolume/SQLWriter.java

@@ -69,6 +69,9 @@ public class SQLWriter {
      */
     private static Connection getConnection() throws SQLException {
         String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
+        if (jdbcURL == null || jdbcURL == ""){
+            jdbcURL = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+        }
         return DriverManager.getConnection(jdbcURL);
     }
 

docs/examples/java/src/test/java/com/taos/test/TestAll.java

@@ -17,6 +17,37 @@ public class TestAll {
                 stmt.execute("drop database if exists " + dbName);
             }
         }
+        waitTransaction();
+    }
+
+    public void dropTopic(String topicName) throws SQLException {
+        String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+        try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
+            try (Statement stmt = conn.createStatement()) {
+                stmt.execute("drop topic if exists " + topicName);
+            }
+        }
+        waitTransaction();
+    }
+
+    public void waitTransaction() throws SQLException {
+
+        String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
+        try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
+            try (Statement stmt = conn.createStatement()) {
+                for (int i = 0; i < 10; i++) {
+                    stmt.execute("show transactions");
+                    try (ResultSet resultSet = stmt.getResultSet()) {
+                        if (resultSet.next()) {
+                            int count = resultSet.getInt(1);
+                            if (count == 0) {
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+        }
     }
 
     public void insertData() throws SQLException {
@@ -104,14 +135,20 @@ public class TestAll {
         SubscribeDemo.main(args);
     }
 
-//    @Test
+    @Test
-//    public void testSubscribeJni() throws SQLException, InterruptedException {
+    public void testSubscribeJni() throws SQLException, InterruptedException {
-//        dropDB("power");
+        dropTopic("topic_meters");
-//        ConsumerLoopFull.main(args);
+        dropDB("power");
-//    }
+        ConsumerLoopFull.main(args);
-//    @Test
+        dropTopic("topic_meters");
-//    public void testSubscribeWs() throws SQLException, InterruptedException {
+        dropDB("power");
-//        dropDB("power");
+    }
-//        WsConsumerLoopFull.main(args);
+    @Test
-//    }
+    public void testSubscribeWs() throws SQLException, InterruptedException {
+        dropTopic("topic_meters");
+        dropDB("power");
+        WsConsumerLoopFull.main(args);
+        dropTopic("topic_meters");
+        dropDB("power");
+    }
 }

docs/zh/01-index.md

@@ -4,9 +4,9 @@ sidebar_label: 文档首页
 slug: /
 ---
 
-TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的<a href="https://www.taosdata.com/" data-internallinksmanager029f6b8e52c="2" title="时序数据库" target="_blank" rel="noopener">时序数据库</a>（<a href="https://www.taosdata.com/time-series-database" data-internallinksmanager029f6b8e52c="9" title="Time Series DataBase" target="_blank" rel="noopener">Time Series Database</a>, <a href="https://www.taosdata.com/tsdb" data-internallinksmanager029f6b8e52c="8" title="TSDB" target="_blank" rel="noopener">TSDB</a>）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept)
+TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的<a href="https://www.taosdata.com/" data-internallinksmanager029f6b8e52c="2" title="时序数据库" target="_blank" rel="noopener">时序数据库</a>（<a href="https://www.taosdata.com/time-series-database" data-internallinksmanager029f6b8e52c="9" title="Time Series DataBase" target="_blank" rel="noopener">Time Series Database</a>, <a href="https://www.taosdata.com/tsdb" data-internallinksmanager029f6b8e52c="8" title="TSDB" target="_blank" rel="noopener">TSDB</a>）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept)。
 
-TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[数据模型](./basic/model)一章。
+TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论你在工作中是什么角色，请您仔细阅读[数据模型](./basic/model)一章。
 
 如果你是开发工程师，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、插入数据、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，你只要复制粘贴示例代码，针对自己的应用稍作改动，就能跑起来。对 REST API、各种编程语言的连接器（Connector）想做更多详细了解的话，请看[连接器](./reference/connector)一章。
 
@@ -16,6 +16,8 @@ TDengine 采用 SQL 作为查询语言，大大降低学习成本、降低迁移
 
 如果你是系统管理员，关心安装、升级、容错灾备、关心数据导入、导出、配置参数，如何监测 TDengine 是否健康运行，如何提升系统运行的性能，请仔细参考[运维指南](./operation)一章。
 
+如果你对数据库内核设计感兴趣，或是开源爱好者，建议仔细阅读[技术内幕](./tdinterna)一章。该章从分布式架构到存储引擎、查询引擎、数据订阅，再到流计算引擎都做了详细阐述。建议对照文档，查看TDengine在GitHub的源代码，对TDengine的设计和编码做深入了解，更欢迎加入开源社区，贡献代码。
+
 最后，作为一个开源软件，欢迎大家的参与。如果发现文档有任何错误、描述不清晰的地方，请在每个页面的最下方，点击“编辑本文档”直接进行修改。
 
 Together, we make a difference!

docs/zh/04-get-started/01-docker.md

@@ -17,7 +17,7 @@ docker pull tdengine/tdengine:latest
 或者指定版本的容器镜像：
 
 ```shell
-docker pull tdengine/tdengine:3.0.1.4
+docker pull tdengine/tdengine:3.3.3.0
 ```
 
 然后只需执行下面的命令：

docs/zh/04-get-started/03-package.md

@@ -14,7 +14,9 @@ TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）
 
 为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。
 
-在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTools 包获得。TDengine 也提供 Windows x64 平台和 macOS x64/m1 平台的安装包。
+在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统，用户可以根据自己的运行环境自行选择。同时我们也提供了 tar.gz 格式安装包，以及 `apt-get` 工具从线上进行安装。
+
+此外，TDengine 也提供 macOS x64/m1 平台的 pkg 安装包。
 
 ## 运行环境要求
 在linux系统中，运行环境最低要求如下:

docs/zh/06-advanced/02-cache.md

@@ -54,7 +54,7 @@ TDengine 利用这些日志文件实现故障前的状态恢复。在写入 WAL
 
 数据库参数 wal_level 和 wal_fsync_period 共同决定了 WAL 的保存行为。。
 - wal_level：此参数控制 WAL 的保存级别。级别 1 表示仅将数据写入 WAL，但不立即执行 fsync 函数；级别 2 则表示在写入 WAL 的同时执行 fsync 函数。默认情况下，wal_level 设为 1。虽然执行 fsync 函数可以提高数据的持久性，但相应地也会降低写入性能。
-- wal_fsync_period：当 wal_level 设置为 2 时，这个参数控制执行 fsync 的频率。设置为 0 表示每次写入后立即执行 fsync，这可以确保数据的安全性，但可能会牺牲一些性能。当设置为大于 0 的数值时，表示 fsync 周期，默认为 3000，范围是[1， 180000]，单位毫秒。
+- wal_fsync_period：当 wal_level 设置为 1 时，这个参数控制执行 fsync 的频率。设置为 0 表示每次写入后立即执行 fsync，这可以确保数据的安全性，但可能会牺牲一些性能。当设置为大于 0 的数值时，表示 fsync 周期，默认为 3000，范围是[1， 180000]，单位毫秒。
 
 ```sql
 CREATE DATABASE POWER WAL_LEVEL 1 WAL_FSYNC_PERIOD 3000;

docs/zh/06-advanced/05-data-in/index.md

@@ -14,18 +14,26 @@ TDengine Enterprise 配备了一个强大的可视化数据管理工具—taosEx
 
 ## 支持的数据源
 
-目前 TDengine 支持的数据源如下：
+目前 TDengine 支持的数据源如下表：
 
-1. Aveva PI System：一个工业数据管理和分析平台，前身为 OSIsoft PI System，它能够实时采集、整合、分析和可视化工业数据，助力企业实现智能化决策和精细化管理
+| 数据源 | 支持版本 | 描述 |
-2. Aveva Historian：一个工业大数据分析软件，前身为 Wonderware Historian，专为工业环境设计，用于存储、管理和分析来自各种工业设备、传感器的实时和历史数据。
+| --- | --- | --- |
-3. OPC DA/UA：OPC 是 Open Platform Communications 的缩写，是一种开放式、标准化的通信协议，用于不同厂商的自动化设备之间进行数据交换。它最初由微软公司开发，旨在解决工业控制领域中不同设备之间互操作性差的问题。OPC 协议最初于 1996 年发布，当时称为 OPC DA （Data Access），主要用于实时数据采集和控制；2006 年，OPC 基金会发布了 OPC UA （Unified Architecture） 标准，它是一种基于服务的面向对象的协议，具有更高的灵活性和可扩展性，已成为 OPC 协议的主流版本。
+| Aveva PI System | PI AF Server Version 2.10.9.593 或以上 | 工业数据管理和分析平台，前身为 OSIsoft PI System，它能够实时采集、整合、分析和可视化工业数据，助力企业实现智能化决策和精细化管理 |
-4. MQTT：Message Queuing Telemetry Transport 的缩写，一种基于发布/订阅模式的轻量级通讯协议，专为低开销、低带宽占用的即时通讯设计，广泛适用于物联网、小型设备、移动应用等领域。
+| Aveva Historian | AVEVA Historian 2020 RS SP1 | 工业大数据分析软件，前身为 Wonderware Historian，专为工业环境设计，用于存储、管理和分析来自各种工业设备、传感器的实时和历史数据 |
-5. Kafka：由 Apache 软件基金会开发的一个开源流处理平台，主要用于处理实时数据，并提供一个统一、高通量、低延迟的消息系统。它具备高速度、可伸缩性、持久性和分布式设计等特点，使得它能够在每秒处理数十万次的读写操作，支持上千个客户端，同时保持数据的可靠性和可用性。
+| OPC DA |  Matrikon OPC version: 1.7.2.7433  | OPC 是 Open Platform Communications 的缩写，是一种开放式、标准化的通信协议，用于不同厂商的自动化设备之间进行数据交换。它最初由微软公司开发，旨在解决工业控制领域中不同设备之间互操作性差的问题；OPC 协议最初于 1996 年发布，当时称为 OPC DA （Data Access），主要用于实时数据采集和控制。 |
-6. OpenTSDB：基于 HBase 的分布式、可伸缩的时序数据库。它主要用于存储、索引和提供从大规模集群（包括网络设备、操作系统、应用程序等）中收集的指标数据，使这些数据更易于访问和图形化展示。
+| OPC UA |  KeepWare KEPServerEx 6.5 | 2006 年，OPC 基金会发布了 OPC UA （Unified Architecture） 标准，它是一种基于服务的面向对象的协议，具有更高的灵活性和可扩展性，已成为 OPC 协议的主流版本 |
-7. CSV：Comma Separated Values 的缩写，是一种以逗号分隔的纯文本文件格式，通常用于电子表格或数据库软件。
+| MQTT | emqx: 3.0.0 到 5.7.1<br/> hivemq: 4.0.0 到 4.31.0<br/> mosquitto: 1.4.4 到 2.0.18 | Message Queuing Telemetry Transport 的缩写，一种基于发布/订阅模式的轻量级通讯协议，专为低开销、低带宽占用的即时通讯设计，广泛适用于物联网、小型设备、移动应用等领域。 |
-8. TDengine 2：泛指运行 TDengine 2.x 版本的 TDengine 实例。
+| Kafka | 2.11 ~ 3.8.0  | 由 Apache 软件基金会开发的一个开源流处理平台，主要用于处理实时数据，并提供一个统一、高通量、低延迟的消息系统。它具备高速度、可伸缩性、持久性和分布式设计等特点，使得它能够在每秒处理数十万次的读写操作，支持上千个客户端，同时保持数据的可靠性和可用性。 |
-9. TDengine 3：泛指运行 TDengine 3.x 版本的 TDengine 实例。
+| InfluxDB | 1.7、1.8、2.0-2.7 | InfluxDB 是一种流行的开源时间序列数据库，它针对处理大量时间序列数据进行了优化。|
-10. MySQL， PostgreSQL， Oracle 等关系型数据库。
+| OpenTSDB | 2.4.1 | 基于 HBase 的分布式、可伸缩的时序数据库。它主要用于存储、索引和提供从大规模集群（包括网络设备、操作系统、应用程序等）中收集的指标数据，使这些数据更易于访问和图形化展示。 |
+| MySQL | 5.6,5.7,8.0+ | MySQL是最流行的关系型数据库管理系统之一，由于其体积小、速度快、总体拥有成本低，尤其是开放源码这一特点，一般中小型和大型网站的开发都选择 MySQL 作为网站数据库。 |
+| Oracle | 11G/12c/19c | Oracle 数据库系统是世界上流行的关系数据库管理系统，系统可移植性好、使用方便、功能强，适用于各类大、中、小微机环境。它是一种高效率的、可靠性好的、适应高吞吐量的数据库方案。 |
+| PostgreSQL | v15.0+  | PostgreSQL 是一个功能非常强大的、源代码开放的客户/服务器关系型数据库管理系统， 有很多在大型商业RDBMS中所具有的特性，包括事务、子选择、触发器、视图、外键引用完整性和复杂锁定功能。|
+| SQL Server | 2012/2022 | Microsoft SQL Server 是一种关系型数据库管理系统，由 Microsoft 公司开发，具有使用方便可伸缩性好与相关软件集成程度高等优点。 |
+| MongoDB | 3.6+ | MongoDB 是一个介于关系型数据库与非关系型数据库之间的产品，被广泛应用于内容管理系统、移动应用与物联网等众多领域。 |
+| CSV | -  | Comma Separated Values 的缩写，是一种以逗号分隔的纯文本文件格式，通常用于电子表格或数据库软件。 |
+| TDengine 2.x | 2.4 或 2.6+ | TDengine 旧版本，已不再维护，推荐升级到 3.0 最新版本。 |
+| TDengine 3.x | 源端版本+ | 使用 TMQ 进行 TDengine 指定从数据库或超级表的订阅。 |
 
 ## 数据提取、过滤和转换
 

docs/zh/06-advanced/06-data-analysis/01-arima.md

@@ -0,0 +1,10 @@
+---
+title: "ARIMA"
+sidebar_label: "ARIMA"
+---
+
+本节讲述如何 ARIMA 算法的使用方法。
+
+## 功能概述
+
+……

docs/zh/06-advanced/06-data-analysis/index.md

@@ -0,0 +1,13 @@
+---
+sidebar_label: 数据分析
+title: 数据分析功能
+---
+
+## 概述
+
+TDengine 提供数据分析功能的扩展组件，通过引入 ANode，TDengine 能够支持时间序列的机器学习分析
+
+下图展示了数据分析的技术架构。
+
+![数据分析功能架构图](./pic/data-analysis.png)
+

docs/zh/08-operation/02-planning.md

@@ -53,7 +53,7 @@ M = (T × S × 3 + (N / 4096) + 100)
 
 与 WebSocket 连接方式相比，RESTful 连接方式在内存占用上更大，除了缓冲区所需的内存以外，还需要考虑每个连接响应结果的内存开销。这种内存开销与响应结果的JSON 数据大小密切相关，特别是在查询数据量很大时，会占用大量内存。
 
-由于 RESTful 连接方式不支持分批获取查询数据，这就导致在查询获取超大结果集时，可能会占用特别大的内存，从而导致内存溢出，因此，在大型项目中，建议打开batchfetch=true 选项，以启用 WebSocket 连接方式，实现流式结果集返回，从而避免内存溢出的风险
+由于 RESTful 连接方式不支持分批获取查询数据，这就导致在查询获取超大结果集时，可能会占用特别大的内存，从而导致内存溢出，因此，在大型项目中，建议使用 WebSocket 连接方式，实现流式结果集返回，从而避免内存溢出的风险
 
 **注意**
 - 建议采用 RESTful/WebSocket 连接方式来访问 TDengine 集群，而不采用taosc 原生连接方式。
@@ -147,7 +147,7 @@ TDengine 的多级存储功能在使用上还具备以下优点。
 下表列出了 TDengine 的一些接口或组件的常用端口，这些端口均可以通过配置文件中的参数进行修改。
 
 |    接口或组件     |    端口    |
-|:---------------------------:|:---------:|
+| :---------------: | :--------: |
 | 原生接口（taosc） |    6030    |
 |   RESTful 接口    |    6041    |
 |  WebSocket 接口   |    6041    |

docs/zh/08-operation/03-deployment.md

@@ -368,6 +368,18 @@ spec:
       labels:
         app: "tdengine"
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: app
+                      operator: In
+                      values:
+                        - tdengine
+                topologyKey: kubernetes.io/hostname
       containers:
         - name: "tdengine"
           image: "tdengine/tdengine:3.2.3.0"

docs/zh/08-operation/12-multi.md

@@ -76,7 +76,7 @@ dataDir /mnt/data6 2 0
 |s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000 (30天），默认值 60 秒 |
 |s3PageCacheSize |s3 page cache 缓存页数目，单位：页。最小值：4；最大值：1024*1024\*1024。 ，默认值 4096|
 |s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 |
-|s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 1，表示开启自动 S3 迁移，可配置为 0。 |
+|s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1。 |
 
 ### 检查配置参数可用性
 
@@ -108,9 +108,37 @@ s3migrate database <db_name>;
 
 | #    | 参数         | 默认值 | 最小值 | 最大值  | 描述                                                         |
 | :--- | :----------- | :----- | :----- | :------ | :----------------------------------------------------------- |
-| 1    | s3_keeplocal | 3650   | 1      | 365000  | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 |
+| 1    | s3_keeplocal | 365    | 1      | 365000  | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 |
-| 2    | s3_chunksize | 262144 | 131072 | 1048576 | 上传对象的大小阈值，与 TSDB_PAGESIZE 参数一样，不可修改，单位为 TSDB 页 |
+| 2    | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 |
-| 3    | s3_compact   | 0      | 0      | 1       | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作。       |
+| 3    | s3_compact   | 1      | 0      | 1       | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作。       |
+
+### 对象存储读写次数估算
+
+对象存储服务的使用成本与存储的数据量及请求次数相关，下面分别介绍数据的上传及下载过程。
+
+#### 数据上传
+
+当 TSDB 时序数据超过 `s3_keeplocal` 参数指定的时间，相关的数据文件会被切分成多个文件块，每个文件块的默认大小是 512M 字节 (`s3_chunkpages * tsdb_pagesize`)。除了最后一个文件块保留在本地文件系统外，其余的文件块会被上传到对象存储服务。
+
+```math
+上传次数 = 数据文件大小 / (s3_chunkpages * tsdb_pagesize) - 1 
+```
+
+在创建数据库时，可以通过 `s3_chunkpages` 参数调整每个文件块的大小，从而控制每个数据文件的上传次数。
+
+其它类型的文件如 head, stt, sma 等，保留在本地文件系统，以加速预计算相关查询。
+
+#### 数据下载
+
+在查询操作中，如果需要访问对象存储中的数据，TSDB 不会下载整个数据文件，而是计算所需数据在文件中的位置，只下载相应的数据到 TSDB 页缓存中，然后将数据返回给查询执行引擎。后续查询首先检查页缓存，查看数据是否已被缓存。如果数据已缓存，则直接使用缓存中的数据，而无需重复从对象存储下载，从而有效降低从对象存储下载数据的次数。
+
+相邻的多个数据页会作为一个数据块从对象存储下载一次，以减少从对象存储下载的次数。每个数据页的大小，在创建数据库时，通过 `tsdb_pagesize` 参数指定，默认 4K 字节。
+
+```math
+下载次数 = 查询需要的数据块数量 - 已缓存的数据块数量
+```
+
+页缓存是内存缓存，节点重启后，再次查询需要重新下载数据。缓存采用 LRU (Least Recently Used) 策略，当缓存空间不足时，最近最少使用的数据将被淘汰。缓存的大小可以通过 `s3PageCacheSize` 参数进行调整，通常来说，缓存越大，下载次数越少。
 
 ## Azure Blob 存储
 本节介绍在 TDengine Enterprise 如何使用微软 Azure Blob 对象存储。本功能是上一小节‘对象存储’功能的扩展，需额外依赖 Flexify 服务提供的 S3 网关。通过适当的参数配置，可以把大部分较冷的时序数据存储到 Azure Blob 服务中。

docs/zh/14-reference/01-components/01-taosd.md

@@ -33,7 +33,7 @@ taosd 命令行参数如下
 |        secondEp        | taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，缺省值：无 |
 |          fqdn          |        启动 taosd 后所监听的服务地址，缺省值：所在服务器上配置的第一个 hostname         |
 |       serverPort       |                         启动 taosd 后所监听的端口，缺省值：6030                         |
-|    numOfRpcSessions    |          允许一个客户端能创建的最大连接数，取值范围 100-100000，缺省值：30000           |
+|    numOfRpcSessions    |          允许一个 dnode 能发起的最大连接数，取值范围 100-100000，缺省值：30000           |
 | timeToGetAvailableConn |      获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值：500000       |
 
 ### 监控相关
@@ -163,7 +163,7 @@ charset 的有效值是 UTF-8。
 
 |      参数名称      |                    参数说明                     |
 | :----------------: | :---------------------------------------------: |
-| numOfCommitThreads | 写入线程的最大数量，取值范围 0-1024，缺省值为 4 |
+| numOfCommitThreads | 落盘线程的最大数量，取值范围 0-1024，缺省值为 4 |
 
 ### 日志相关
 
@@ -225,9 +225,9 @@ lossyColumns     float|double
 | :--------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
 |  enableCoreFile  | crash 时是否生成 core 文件；0: 不生成，1：生成；默认值为 1; 不同的启动方式，生成 core 文件的目录如下：1、systemctl start taosd 启动：生成的 core 在根目录下  <br/> 2、手动启动，就在 taosd 执行目录下。  |
 |       udf        |                                                                            是否启动 UDF 服务；0: 不启动，1：启动；默认值为 0                                                                             |
-| ttlChangeOnWrite |                                                                   ttl 到期时间是否伴随表的修改操作改变; 0: 不改变，1：改变 ；默认值 为                                                                   |
+| ttlChangeOnWrite |                                                                   ttl 到期时间是否伴随表的修改操作改变; 0: 不改变，1：改变；默认值为 0                                                                   |
 |  tmqMaxTopicNum  |                                                                        订阅最多可建立的 topic 数量; 取值范围 1-10000；缺省值为20                                                                         |
-|    maxTsmaNum    |                                                                             集群内可创建的TSMA个数；取值范围：0-3；缺省值: 3                                                                             |
+|    maxTsmaNum    |                                                                             集群内可创建的TSMA个数；取值范围：0-3；缺省值为 3                                                                            |
 
 
 ## taosd 监控指标

docs/zh/14-reference/01-components/07-explorer.md

@@ -8,7 +8,7 @@ taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工
 
 ## 安装
 
-taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。如果按照 GitHub 里步骤自己编译 TDengine 源代码生成的安装包不包含 taosExplorer。
+taosExplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。如果按照 GitHub 里步骤自己编译 TDengine 源代码生成的安装包不包含 taosExplorer。
 
 ## 配置
 

docs/zh/14-reference/02-tools/10-taosbenchmark.md

@@ -364,6 +364,8 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
 
 - **max** : 数据类型的 列/标签 的最大值。生成的值将小于最小值。
 
+- **scalingFactor** : 浮点数精度增强因子，仅当数据类型是float/double时生效，有效值范围为1至1000000的正整数。用于增强生成浮点数的精度，特别是在min或max值较小的情况下。此属性按10的幂次增强小数点后的精度：scalingFactor为10表示增强1位小数精度，100表示增强2位，依此类推。
+
 - **fun** : 此列数据以函数填充，目前只支持 sin 和 cos 两函数，输入参数为时间戳换算成角度值，换算公式： 角度 x = 输入的时间列ts值 % 360。同时支持系数调节，随机波动因子调节，以固定格式的表达式展现，如 fun=“10\*sin(x)+100\*random(5)” , x 表示角度，取值 0 ~ 360度，增长步长与时间列步长一致。10 表示乘的系数，100 表示加或减的系数，5 表示波动幅度在 5% 的随机范围内。目前支持的数据类型为 int, bigint, float, double 四种数据类型。注意：表达式为固定模式，不可前后颠倒。
 
 - **values** : nchar/binary 列/标签的值域，将从值中随机选择。

docs/zh/14-reference/03-taos-sql/03-table.md

@@ -87,7 +87,7 @@ CREATE TABLE [IF NOT EXISTS] USING [db_name.]stb_name (field1_name [, field2_nam
 
 **参数说明**
 
-1. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。CSV 文件中应仅包含 table name 与 tag 值。如需插入数据，请参考数据写入章节。
+1. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。CSV 文件中应仅包含 table name 与 tag 值。如需插入数据，请参考'数据写入'章节。
 2. 为指定的 stb_name 创建子表，该超级表必须已经存在。
 3. field_name 列表顺序与 CSV 文件各列内容顺序一致。列表中不允许出现重复项，且必须包含 `tbname`，可包含零个或多个超级表中已定义的标签列。未包含在列表中的标签值将被设置为 NULL。
 

docs/zh/14-reference/03-taos-sql/05-insert.md

@@ -47,7 +47,7 @@ INSERT INTO
 
 2. VALUES 语法表示了要插入的一行或多行数据。
 
-3. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。
+3. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。如仅需创建子表，请参考'表'章节。
 
 4. `INSERT ... VALUES` 语句和 `INSERT ... FILE` 语句均可以在一条 INSERT 语句中同时向多个表插入数据。
 
@@ -154,12 +154,20 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c
 INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/csvfile_21001.csv'
             d21002 USING meters (groupId) TAGS (2) FILE '/tmp/csvfile_21002.csv';
 ```
-## 超级表语法
+## 向超级表插入数据并自动创建子表
 
 自动建表, 表名通过 tbname 列指定
 ```sql
 INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase)
-                values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) 
+                VALUES ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32)
                 ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
                 ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)
 ```
+## 通过 CSV 文件向超级表插入数据并自动创建子表
+
+根据 csv 文件内容，为 超级表创建子表，并填充相应 column 与 tag
+
+```sql
+INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase)
+                FILE '/tmp/csvfile_21002.csv'
+```

docs/zh/14-reference/03-taos-sql/14-stream.md

@@ -212,7 +212,7 @@ TDengine 对于修改数据提供两种处理方式，由 IGNORE UPDATE 选项
 ```sql
 [field1_name,...]
 ```
-用来指定stb_name的列与subquery输出结果的对应关系。如果stb_name的列与subquery输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果stb_name的列与subquery输出结果的数据类型不匹配，会把subquery输出结果的类型转换成对应的stb_name的列的类型。
+在本页文档顶部的 [field1_name,...] 是用来指定 stb_name 的列与 subquery 输出结果的对应关系的。如果 stb_name 的列与 subquery 输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果 stb_name 的列与 subquery 输出结果的数据类型不匹配，会把 subquery 输出结果的类型转换成对应的 stb_name 的列的类型。
 
 对于已经存在的超级表，检查列的schema信息
 1. 检查列的 schema 信息是否匹配，对于不匹配的，则自动进行类型转换，当前只有数据长度大于 4096byte 时才报错，其余场景都能进行类型转换。

docs/zh/14-reference/03-taos-sql/21-node.md

@@ -27,11 +27,15 @@ SHOW DNODES;
 ## 删除数据节点
 
 ```sql
-DROP DNODE dnode_id
+DROP DNODE dnode_id [force] [unsafe]
 ```
 
 注意删除 dnode 不等于停止相应的进程。实际中推荐先将一个 dnode 删除之后再停止其所对应的进程。
 
+只有在线节点可以被删除。如果要强制删除离线节点，需要执行强制删除操作, 即指定force选项。
+
+当节点上存在单副本，并且节点处于离线，如果要强制删除该节点，需要执行非安全删除，即制定unsafe，并且数据不可再恢复。
+
 ## 修改数据节点配置
 
 ```sql

docs/zh/14-reference/05-connector/50-odbc.mdx

@@ -3,11 +3,14 @@ sidebar_label: ODBC
 title: TDengine ODBC
 ---
 
-TDengine ODBC  是为 TDengine 实现的 ODBC 驱动程序，支持 Windows 系统的应用（如 [PowerBI](https://powerbi.microsoft.com/zh-cn/) 等）通过 ODBC 标准接口访问本地、远程和云服务的 TDengine 数据库的数据表/视图。
+TDengine ODBC 是为 TDengine 实现的 ODBC 驱动程序，支持 Windows 系统的应用（如 [PowerBI](https://powerbi.microsoft.com/zh-cn/) 等）以及用户自定义开发的应用程序，通过 ODBC 标准接口访问本地、远程和云服务的 TDengine 数据库。
 
 TDengine ODBC 提供基于 WebSocket（推荐）和 原生连接两种方式连接 TDengine 数据库，使用时可以为 TDengine 数据源设置不同的连接方式。访问云服务时必须使用 WebSocket 连接方式。
 
-注意：TDengine ODBC 支持 32/64 位 Windows 系统，调用 TDengine ODBC 需要通过相应位数的 ODBC 驱动管理器进行。在 32 位 Windows 系统或者 64 位 Windows 系统的 32 位应用程序中，仅支持使用 WebSocket 连接方式访问 TDengine 数据库。
+TDengine ODBC 提供 64 位 和 32 位 两种驱动程序。但 32 位仅企业版支持，且仅支持 WebSocket 连接方式。  
+**注意：**
+- 驱动管理器：确保使用与应用程序架构匹配的 ODBC 驱动管理器。32 位应用程序需要使用 32 位 ODBC 驱动管理器，64 位应用程序需要使用 64 位 ODBC 驱动管理器。
+- 数据源名称（DSN）：32 位和 64 位 ODBC 驱动管理器都可以看到所有 DSN，用户 DSN 标签页下的 DSN 如果名字相同会共用，因此需要在 DSN 名称上去区分。
 
 想更多了解 TDengine 时序时序数据库的使用，可访问 [TDengine官方文档](https://docs.taosdata.com/intro/)。
 
@@ -24,17 +27,17 @@ TDengine ODBC  提供基于 WebSocket（推荐）和 原生连接两种方式连
 
 ### 数据源连接类型与区别
 
-TDengine ODBC 支持两种连接 TDengine 数据库方式：Websocket 连接与 Native 连接，其区别如下：
+TDengine ODBC 支持两种连接 TDengine 数据库方式：WebSocket 连接与 Native 连接，其区别如下：
 
-1. 访问云服务仅支持使用 Websocket 连接方式。
+1. 访问云服务仅支持使用 WebSocket 连接方式。
 
 2. 32 位应用程序仅支持使用 WebSocket 连接方式。
 
-3. Websocket 连接的兼容性更好，一般不需要随着 TDengine 数据库服务端升级而升级客户端的库。
+3. WebSocket 连接的兼容性更好，一般不需要随着 TDengine 数据库服务端升级而升级客户端的库。
 
 4. Native 连接通常性能更好一点，但是必须与 TDengine 数据库服务端的版本保持一致。
 
-5. 对于一般用户，建议使用 **Websocket** 连接方式，性能与 Native 差别不大，兼容性更好。
+5. 对于一般用户，建议使用 **WebSocket** 连接方式，性能与 Native 差别不大，兼容性更好。
 
 ### WebSocket 连接
 
@@ -46,11 +49,11 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式：Websocket 连接与
 
 4. 点击完成，进入 TDengine ODBC 数据源配置页面，填写如下必要信息 
 
-    ![ODBC websocket connection config](./assets/odbc-ws-config-zh.webp)
+    ![ODBC WebSocket connection config](./assets/odbc-ws-config-zh.webp)
 
     4.1 【DSN】:Data Source Name 必填，为新添加的 ODBC 数据源命名
 
-    4.2【连接类型】 : 必选，选择连接类型，这里选择 【Websocket】
+    4.2【连接类型】 : 必选，选择连接类型，这里选择 【WebSocket】
 
     4.3【URL】必填，ODBC 数据源 URL，示例: `http://localhost:6041`， 云服务的 url 示例: `https://gw.cloud.taosdata.com?token=your_token`
 
@@ -111,7 +114,7 @@ WebSocket 连接方式除此之外还支持 Windows X64系统上运行的 32 位
 
 | taos_odbc 版本 | 主要变化                                                                                             |   TDengine 版本    |
 | :-----------  | :--------------------------------------------------------------------------------------------------  | :----------------  |
-|      v1.1.0   | 1. 支持视图功能；<br/>2. 支持 VARBINARY/GEOMETRY 数据类型；                                             | 3.3.3.0及更高版本 |
+|      v1.1.0   | 1. 支持视图功能；<br/>2. 支持 VARBINARY/GEOMETRY 数据类型；<br/>3. 支持 ODBC 32 位 WebSocket 连接方式（仅企业版支持）；<br/>4. 支持 ODBC 数据源配置对话框设置对工业软件 KingSCADA、Kepware 等的兼容性适配选项（仅企业版支持）； | 3.3.3.0及更高版本 |
 |      v1.0.2   | 支持 CP1252 字符编码；                                                                                 | 3.2.3.0及更高版本 |
 |      v1.0.1   | 1. 支持 DSN 设置 BI 模式，在 BI 模式下 TDengine 数据库不返回系统数据库和超级表子表信息；<br/>2. 重构字符集转换模块，提升读写性能；<br/>3. ODBC 数据源配置对话框中默认修改默认连接方式为“WebSocket”；<br/>4. ODBC 数据源配置对话框增加“测试连接”控件；<br/>5. ODBC 数据源配置支持中文/英文界面； |         -          |
 |    v1.0.0.0   | 发布初始版本，支持与Tdengine数据库交互以读写数据，具体请参考“API 参考”一节                                | 3.2.2.0及更高版本 |

docs/zh/14-reference/07-supported.md

@@ -6,15 +6,28 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表"
 
 ## TDengine 服务端支持的平台列表
 
-|              | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 以上** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** |
+|                       | **版本**        | **X64 64bit** | **ARM64** |
-| ------------ | ---------------------------- | ----------------- | ---------------- | ------------------ | ------------ | ----------------- | ---------------- | --------- |
+| ----------------------|----------------| ------------- | --------- |
-| X64          | ●/E                          | ●/E               | ●                | ●                  | ●/E          | ●/E               | ●/E              | ●         |
+| **CentOS**            | **7.9 以上**    | ●             | ●         |
-| 树莓派 ARM64 |                              |                   | ●                |                    |              |                   |                  |           |
+| **Ubuntu**            | **18 以上**     | ●             | ●         |
-| 华为云 ARM64 |                              |                   |                  | ●                  |              |                   |                  |           |
+| **RedHat**            | **RHEL 7 以上** | ●             | ●         |
-| M1           |                              |                   |                  |                    |              |                   |                  | ●         |
+| **Debian**            | **6.0 以上**    | ●             | ●         |
+| **FreeBSD**           | **12 以上**     | ●             | ●         |
+| **OpenSUSE**          | **全部版本**     | ●             | ●         |
+| **SUSE Linux**        | **11 以上**     | ●             | ●         |
+| **Fedora**            | **21 以上**     | ●             | ●         |
+| **Windows Server**    | **2016 以上**  | ●/E           |           |
+| **Windows**           | **10/11**      | ●/E           |           |
+| **银河麒麟**           | **V10 以上**     | ●/E           | ●/E      |
+| **中标麒麟**           | **V7.0 以上**    | ●/E           | ●/E      |
+| **统信 UOS**          | **V20 以上**     | ●/E           |           |
+| **凝思磐石**           | **V8.0 以上**    | ●/E           |           |
+| **华为欧拉 openEuler** | **V20.03 以上**  | ●/E           |           |
+| **龙蜥 Anolis OS**     | **V8.6 以上**   | ●/E           |           | 
+| **macOS**             | **11.0 以上**   |                | ●         |
 
 注：1) ● 表示经过官方测试验证， ○ 表示非官方测试验证，E 表示仅企业版支持。
-   2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。
+   2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/CentOS Stream/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。
 
 ## TDengine 客户端和连接器支持的平台列表
 
@@ -22,16 +35,16 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表"
 
 对照矩阵如下：
 
-| **CPU**     | **X64 64bit** | **X64 64bit** | **ARM64** | **X64 64bit** | **ARM64** |
+| **CPU**     | **X64 64bit** | **X64 64bit** | **X64 64bit** | **ARM64** | **ARM64** |
-| ----------- | ------------- | ------------- | --------- | ------------- | --------- |
+| ----------- | ------------- | ------------- | ------------- | --------- | --------- |
-| **OS**      | **Linux**     | **Win64**     | **Linux** | **macOS**     | **macOS** |
+| **OS**      | **Linux**     | **Win64**     | **macOS**     | **Linux** | **macOS** |
 | **C/C++**   | ●             | ●             | ●             | ●         | ●         |
-| **JDBC**    | ●             | ●             | ●         | ○             | ○         |
+| **JDBC**    | ●             | ●             | ●             | ●         | ●         |
 | **Python**  | ●             | ●             | ●             | ●         | ●         |
 | **Go**      | ●             | ●             | ●             | ●         | ●         |
-| **NodeJs**  | ●             | ●             | ●         | ○             | ○         |
+| **NodeJs**  | ●             | ●             | ●             | ●         | ●         |
-| **C#**      | ●             | ●             | ○         | ○             | ○         |
+| **C#**      | ●             | ●             | ○             | ●         | ○         |
-| **Rust**    | ●             | ●             | ○         | ●             | ●         |
+| **Rust**    | ●             | ●             | ●             | ○         | ●         |
 | **RESTful** | ●             | ●             | ●             | ●         | ●         |
 
 注：● 表示官方测试验证通过，○ 表示非官方测试验证通过，-- 表示未经验证。

docs/zh/26-tdinternal/01-arch.md

@@ -178,7 +178,7 @@ TDengine 集群可以容纳单个、多个甚至几千个数据节点。应用
 
 TDengine 存储的数据包括采集的时序数据以及库、表相关的元数据、标签数据等，这些数据具体分为三部分：
 
-- 时序数据：TDengine 的核心存储对象，存放于 vnode 里，由 data、head 和 last 三个文件组成，数据量大，查询量取决于应用场景。允许乱序写入，但暂时不支持删除操作，并且仅在 update 参数设置为 1 时允许更新操作。通过采用一个采集点一张表的模型，一个时间段的数据是连续存储，对单张表的写入是简单的追加操作，一次读，可以读到多条记录，这样保证对单个采集点的插入和查询操作，性能达到最优。
+- 时序数据：时序数据是 TDengine 的核心存储对象，它们被存储在 vnode 中。时序数据由 data、head、sma 和 stt 4 类文件组成，这些文件共同构成了时序数据的完整存储结构。由于时序数据的特点是数据量大且查询需求取决于具体应用场景，因此 TDengine 采用了“一个数据采集点一张表”的模型来优化存储和查询性能。在这种模型下，一个时间段内的数据是连续存储的，对单张表的写入是简单的追加操作，一次读取可以获取多条记录。这种设计确保了单个数据采集点的写入和查询操作都能达到最优性能。
 - 数据表元数据：包含标签信息和 Table Schema 信息，存放于 vnode 里的 meta 文件，支持增删改查四个标准操作。数据量很大，有 N 张表，就有 N 条记录，因此采用 LRU 存储，支持标签数据的索引。TDengine 支持多核多线程并发查询。只要计算内存足够，元数据全内存存储，千万级别规模的标签数据过滤结果能毫秒级返回。在内存资源不足的情况下，仍然可以支持数千万张表的快速查询。
 - 数据库元数据：存放于 mnode 里，包含系统节点、用户、DB、STable Schema 等信息，支持增删改查四个标准操作。这部分数据的量不大，可以全内存保存，而且由于客户端有缓存，查询量也不大。因此目前的设计虽是集中式存储管理，但不会构成性能瓶颈。
 

include/common/tmsg.h

@@ -4104,7 +4104,6 @@ void    tDeleteMqMetaRsp(SMqMetaRsp* pRsp);
 #define MQ_DATA_RSP_VERSION 100
 
 typedef struct {
-  struct {
   SMqRspHead   head;
   STqOffsetVal rspOffset;
   STqOffsetVal reqOffset;
@@ -4115,7 +4114,6 @@ typedef struct {
   SArray*      blockData;
   SArray*      blockTbName;
   SArray*      blockSchema;
-  };
 
   union{
     struct{

include/common/ttokendef.h

@@ -122,7 +122,7 @@
 #define TK_STT_TRIGGER          104
 #define TK_TABLE_PREFIX         105
 #define TK_TABLE_SUFFIX         106
-#define TK_S3_CHUNKSIZE                   107
+#define TK_S3_CHUNKPAGES        107
 #define TK_S3_KEEPLOCAL         108
 #define TK_S3_COMPACT           109
 #define TK_KEEP_TIME_OFFSET     110

include/libs/executor/executor.h

@@ -151,8 +151,9 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
  * @param tversion
  * @return
  */
-int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
+int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, int32_t dbNameBuffLen, char* tableName,
-                                    int32_t* tversion, int32_t idx, bool* tbGet);
+                                    int32_t tbaleNameBuffLen, int32_t* sversion, int32_t* tversion, int32_t idx,
+                                    bool* tbGet);
 
 /**
  * The main task execution function, including query on both table and multiple tables,

include/libs/parser/parser.h

@@ -176,8 +176,8 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS
                     STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
                     char* msgBuf, int32_t msgBufLen);
 int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
-int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* fields,
+int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
-                     int numFields, bool needChangeLength, char* errstr, int32_t errstrLen);
+                     int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);
 
 int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
 int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);

include/util/tdef.h

@@ -450,7 +450,7 @@ typedef enum ELogicConditionType {
 
 #define TSDB_MIN_S3_CHUNK_SIZE     (128 * 1024)
 #define TSDB_MAX_S3_CHUNK_SIZE     (1024 * 1024)
-#define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024)
+#define TSDB_DEFAULT_S3_CHUNK_SIZE (128 * 1024)
 #define TSDB_MIN_S3_KEEP_LOCAL     (1 * 1440)  // unit minute
 #define TSDB_MAX_S3_KEEP_LOCAL     (365000 * 1440)
 #define TSDB_DEFAULT_S3_KEEP_LOCAL (365 * 1440)

include/util/tlog.h

@@ -70,7 +70,7 @@ extern int32_t tdbDebugFlag;
 extern int32_t sndDebugFlag;
 extern int32_t simDebugFlag;
 
-extern int32_t tqClientDebug;
+extern int32_t tqClientDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc);
 void    taosCloseLog();

packaging/tools/make_install.bat

@@ -12,9 +12,18 @@ if exist C:\\TDengine\\data\\dnode\\dnodeCfg.json (
 
 rem // stop and delete service
 mshta vbscript:createobject("shell.application").shellexecute("%~s0",":stop_delete","","runas",1)(window.close)
+
+if exist %binary_dir%\\build\\bin\\taosadapter.exe (
     echo This might take a few moment to accomplish deleting service taosd/taosadapter ...
+)
+
+if exist %binary_dir%\\build\\bin\\taoskeeper.exe (
+    echo This might take a few moment to accomplish deleting service taosd/taoskeeper ...
+)
+
 call :check_svc taosd
 call :check_svc taosadapter
+call :check_svc taoskeeper
 
 set source_dir=%2
 set source_dir=%source_dir:/=\\%
@@ -46,6 +55,11 @@ if exist %binary_dir%\\test\\cfg\\taosadapter.toml (
         copy %binary_dir%\\test\\cfg\\taosadapter.toml %target_dir%\\cfg\\taosadapter.toml > nul
     )
 )
+if exist %binary_dir%\\test\\cfg\\taoskeeper.toml (
+    if not exist %target_dir%\\cfg\\taoskeeper.toml (
+        copy %binary_dir%\\test\\cfg\\taoskeeper.toml %target_dir%\\cfg\\taoskeeper.toml > nul
+    )
+)
 copy %source_dir%\\include\\client\\taos.h %target_dir%\\include > nul
 copy %source_dir%\\include\\util\\taoserror.h %target_dir%\\include > nul
 copy %source_dir%\\include\\libs\\function\\taosudf.h %target_dir%\\include > nul
@@ -104,6 +118,9 @@ copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul
 if exist %binary_dir%\\build\\bin\\taosadapter.exe (
     copy %binary_dir%\\build\\bin\\taosadapter.exe %target_dir% > nul
 )
+if exist %binary_dir%\\build\\bin\\taoskeeper.exe (
+    copy %binary_dir%\\build\\bin\\taoskeeper.exe %target_dir% > nul
+)
 
 mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)
 
@@ -116,6 +133,10 @@ if exist %binary_dir%\\build\\bin\\taosadapter.exe (
     echo To start/stop taosAdapter with administrator privileges: %ESC%[92msc start/stop taosadapter %ESC%[0m
 )
 
+if exist %binary_dir%\\build\\bin\\taoskeeper.exe (
+    echo To start/stop taosKeeper with administrator privileges: %ESC%[92msc start/stop taoskeeper %ESC%[0m
+)
+
 goto :eof
 
 :hasAdmin
@@ -123,6 +144,7 @@ goto :eof
 call :stop_delete
 call :check_svc taosd
 call :check_svc taosadapter
+call :check_svc taoskeeper
 
 if exist c:\\windows\\sysnative (
     echo x86
@@ -141,6 +163,7 @@ if exist c:\\windows\\sysnative (
 rem // create services
 sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND
 sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND
+sc create "taoskeeper" binPath= "C:\\TDengine\\taoskeeper.exe" start= DEMAND
 
 set "env=HKLM\System\CurrentControlSet\Control\Session Manager\Environment"
 for /f "tokens=2*" %%I in ('reg query "%env%" /v Path ^| findstr /i "\<Path\>"') do (
@@ -181,6 +204,8 @@ sc stop taosd
 sc delete taosd
 sc stop taosadapter
 sc delete taosadapter
+sc stop taoskeeper
+sc delete taoskeeper
 exit /B 0
 
 :check_svc

packaging/tools/make_install.sh

@@ -129,6 +129,13 @@ function kill_taosadapter() {
   fi
 }
 
+function kill_taoskeeper() {
+  pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}')
+  if [ -n "$pid" ]; then
+    ${csudo}kill -9 $pid || :
+  fi
+}
+
 function kill_taosd() {
   pid=$(ps -ef | grep -w ${serverName} | grep -v "grep" | awk '{print $2}')
   if [ -n "$pid" ]; then
@@ -155,6 +162,7 @@ function install_bin() {
   ${csudo}rm -f ${bin_link_dir}/${clientName} || :
   ${csudo}rm -f ${bin_link_dir}/${serverName} || :
   ${csudo}rm -f ${bin_link_dir}/taosadapter || :
+  ${csudo}rm -f ${bin_link_dir}/taoskeeper || :
   ${csudo}rm -f ${bin_link_dir}/udfd || :
   ${csudo}rm -f ${bin_link_dir}/taosdemo || :
   ${csudo}rm -f ${bin_link_dir}/taosdump || :
@@ -169,6 +177,7 @@ function install_bin() {
     [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
     [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
+    [ -f ${binary_dir}/build/bin/taoskeeper ] && ${csudo}cp -r ${binary_dir}/build/bin/taoskeeper ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || :
     ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || :
@@ -183,6 +192,7 @@ function install_bin() {
     [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
     [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
     [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
+    [ -x ${install_main_dir}/bin/taoskeeper ] && ${csudo}ln -s ${install_main_dir}/bin/taoskeeper ${bin_link_dir}/taoskeeper > /dev/null 2>&1 || :
     [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
     [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
     [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo > /dev/null 2>&1 || :
@@ -197,6 +207,7 @@ function install_bin() {
     [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
     [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
+    [ -f ${binary_dir}/build/bin/taoskeeper ] && ${csudo}cp -r ${binary_dir}/build/bin/taoskeeper ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || :
     [ -f ${binary_dir}/build/bin/*explorer ] && ${csudo}cp -r ${binary_dir}/build/bin/*explorer ${install_main_dir}/bin || :
@@ -208,6 +219,7 @@ function install_bin() {
     [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
     [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
     [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
+    [ -x ${install_main_dir}/bin/taoskeeper ] && ${csudo}ln -s ${install_main_dir}/bin/taoskeeper ${bin_link_dir}/taoskeeper > /dev/null 2>&1 || :
     [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
     [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
     [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
@@ -407,6 +419,29 @@ function install_taosadapter_config() {
   fi
 }
 
+function install_taoskeeper_config() {
+  if [ ! -f "${cfg_install_dir}/taoskeeper.toml" ]; then
+    ${csudo}mkdir -p ${cfg_install_dir} || :
+    [ -f ${binary_dir}/test/cfg/taoskeeper.toml ] &&
+      ${csudo}cp ${binary_dir}/test/cfg/taoskeeper.toml ${cfg_install_dir} &&
+      ${csudo}cp ${binary_dir}/test/cfg/taoskeeper.toml ${cfg_dir} || :
+    [ -f ${cfg_install_dir}/taoskeeper.toml ] &&
+      ${csudo}chmod 644 ${cfg_install_dir}/taoskeeper.toml || :
+    [ -f ${binary_dir}/test/cfg/taoskeeper.toml ] &&
+      ${csudo}cp -f ${binary_dir}/test/cfg/taoskeeper.toml \
+        ${cfg_install_dir}/taoskeeper.toml.${verNumber} || :
+    [ -f ${cfg_install_dir}/taoskeeper.toml ] &&
+      ${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml \
+        ${install_main_dir}/cfg/taoskeeper.toml > /dev/null 2>&1 || :
+  else
+    if [ -f "${binary_dir}/test/cfg/taoskeeper.toml" ]; then
+      ${csudo}cp -f ${binary_dir}/test/cfg/taoskeeper.toml \
+        ${cfg_install_dir}/taoskeeper.toml.${verNumber} || :
+      ${csudo}cp -f ${binary_dir}/test/cfg/taoskeeper.toml ${cfg_dir} || :
+    fi
+  fi
+}
+
 function install_log() {
   ${csudo}rm -rf ${log_dir} || :
   ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
@@ -526,6 +561,15 @@ function install_taosadapter_service() {
   fi
 }
 
+function install_taoskeeper_service() {
+  if ((${service_mod} == 0)); then
+    [ -f ${binary_dir}/test/cfg/taoskeeper.service ] &&
+      ${csudo}cp ${binary_dir}/test/cfg/taoskeeper.service \
+        ${service_config_dir}/ || :
+    ${csudo}systemctl daemon-reload
+  fi
+}
+
 function install_service_on_launchctl() {
   ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
   ${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist
@@ -534,6 +578,10 @@ function install_service_on_launchctl() {
   ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || :
   ${csudo}cp ${script_dir}/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist
   ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || :
+
+  ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist > /dev/null 2>&1 || :
+  ${csudo}cp ${script_dir}/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist
+  ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist > /dev/null 2>&1 || :
 }
 
 function install_service() {
@@ -549,6 +597,7 @@ function install_service() {
     install_service_on_launchctl
   fi
 }
+
 function install_app() {
   if [ "$osType" = "Darwin" ]; then
     ${csudo}rm -rf /Applications/TDengine.app &&
@@ -573,6 +622,7 @@ function update_TDengine() {
     elif ((${service_mod} == 1)); then
       ${csudo}service ${serverName} stop || :
     else
+      kill_taoskeeper
       kill_taosadapter
       kill_taosd
     fi
@@ -591,9 +641,11 @@ function update_TDengine() {
 
   install_service
   install_taosadapter_service
+  install_taoskeeper_service
 
   install_config
   install_taosadapter_config
+  install_taoskeeper_config
 
   echo
   echo -e "\033[44;32;1m${productName} is updated successfully!${NC}"
@@ -602,22 +654,33 @@ function update_TDengine() {
   echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}"
   [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
     echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml"
+  [ -f ${configDir}/taoskeeper.toml ] && [ -f ${installDir}/bin/taoskeeper ] && \
+    echo -e "${GREEN_DARK}To configure Keeper ${NC}: edit ${configDir}/taoskeeper.toml"
   if ((${service_mod} == 0)); then
     echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${csudo}systemctl start ${serverName}${NC}"
     [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
       echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
+    [ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
+      echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}systemctl start taoskeeper ${NC}"
   elif ((${service_mod} == 1)); then
     echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${csudo}service ${serverName} start${NC}"
     [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
       echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}"
+    [ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
+      echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}service taoskeeper start${NC}"
   else
     if [ "$osType" != "Darwin" ]; then
       echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${serverName}${NC}"
       [ -f ${installDir}/bin/taosadapter ] && \
         echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}"
+      [ -f ${installDir}/bin/taoskeeper ] && \
+        echo -e "${GREEN_DARK}To start Keeper ${NC}: taoskeeper &${NC}"
     else
       echo -e "${GREEN_DARK}To start service      ${NC}: sudo launchctl start com.tdengine.taosd${NC}"
+      [ -f ${installDir}/bin/taosadapter ] && \
         echo -e "${GREEN_DARK}To start Adapter      ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}"
+      [ -f ${installDir}/bin/taoskeeper ] && \
+        echo -e "${GREEN_DARK}To start Keeper       ${NC}: sudo launchctl start com.tdengine.taoskeeper${NC}"
     fi
   fi
 
@@ -643,9 +706,11 @@ function install_TDengine() {
 
   install_service
   install_taosadapter_service
+  install_taoskeeper_service
 
   install_config
   install_taosadapter_config
+  install_taoskeeper_config
 
   # Ask if to start the service
   echo
@@ -654,22 +719,33 @@ function install_TDengine() {
   echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}"
   [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
     echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml"
+  [ -f ${configDir}/taoskeeper.toml ] && [ -f ${installDir}/bin/taoskeeper ] && \
+    echo -e "${GREEN_DARK}To configure Keeper ${NC}: edit ${configDir}/taoskeeper.toml"
   if ((${service_mod} == 0)); then
     echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${csudo}systemctl start ${serverName}${NC}"
     [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
       echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
+    [ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
+      echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}systemctl start taoskeeper ${NC}"
   elif ((${service_mod} == 1)); then
     echo -e "${GREEN_DARK}To start ${productName}    ${NC}: ${csudo}service ${serverName} start${NC}"
     [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
       echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}"
+    [ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
+      echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}service taoskeeper start${NC}"
   else
     if [ "$osType" != "Darwin" ]; then
       echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${serverName}${NC}"
       [ -f ${installDir}/bin/taosadapter ] && \
         echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}"
+      [ -f ${installDir}/bin/taoskeeper ] && \
+        echo -e "${GREEN_DARK}To start Keeper ${NC}: taoskeeper &${NC}"
     else
       echo -e "${GREEN_DARK}To start service      ${NC}: sudo launchctl start com.tdengine.taosd${NC}"
+      [ -f ${installDir}/bin/taosadapter ] && \
         echo -e "${GREEN_DARK}To start Adapter      ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}"
+      [ -f ${installDir}/bin/taoskeeper ] && \
+        echo -e "${GREEN_DARK}To start Keeper       ${NC}: sudo launchctl start com.tdengine.taoskeeper${NC}"
     fi
   fi
 

source/client/src/clientRawBlockWrite.c

@@ -53,9 +53,7 @@
 #define TMQ_META_VERSION "1.0"
 
 static int32_t  tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
-
 static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
-
 static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t,
                                  SColCmprWrapper* pColCmprRow, cJSON** pJson) {
   int32_t code = TSDB_CODE_SUCCESS;
@@ -890,9 +888,6 @@ end:
 }
 
 static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVCreateStbReq req = {0};
   SDecoder       coder;
   SMCreateStbReq pReq = {0};
@@ -1003,9 +998,6 @@ end:
 }
 
 static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVDropStbReq req = {0};
   SDecoder     coder = {0};
   SMDropStbReq pReq = {0};
@@ -1115,9 +1107,6 @@ static void destroyCreateTbReqBatch(void* data) {
 }
 
 static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVCreateTbBatchReq req = {0};
   SDecoder           coder = {0};
   int32_t            code = TSDB_CODE_SUCCESS;
@@ -1304,9 +1293,6 @@ static void destroyDropTbReqBatch(void* data) {
 }
 
 static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVDropTbBatchReq req = {0};
   SDecoder         coder = {0};
   int32_t          code = TSDB_CODE_SUCCESS;
@@ -1419,9 +1405,6 @@ end:
 }
 
 static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SDeleteRes req = {0};
   SDecoder   coder = {0};
   char       sql[256] = {0};
@@ -1457,9 +1440,6 @@ end:
 }
 
 static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
-  if (taos == NULL || meta == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
   SVAlterTbReq   req = {0};
   SDecoder       dcoder = {0};
   int32_t        code = TSDB_CODE_SUCCESS;
@@ -1622,7 +1602,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
   RAW_NULL_CHECK(pVgHash);
   RAW_RETURN_CHECK(
       taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
-  RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0));
+  RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0, false));
   RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
 
   launchQueryImpl(pRequest, pQuery, true, NULL);
@@ -1682,7 +1662,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
   RAW_NULL_CHECK(pVgHash);
   RAW_RETURN_CHECK(
       taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
-  RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0));
+  RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0, false));
   RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
 
   launchQueryImpl(pRequest, pQuery, true, NULL);
@@ -1708,116 +1688,6 @@ static void* getRawDataFromRes(void* pRetrieve) {
   return rawData;
 }
 
-static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
-  if (taos == NULL || data == NULL) {
-    SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
-    return TSDB_CODE_INVALID_PARA;
-  }
-  int32_t     code = TSDB_CODE_SUCCESS;
-  SHashObj*   pVgHash = NULL;
-  SQuery*     pQuery = NULL;
-  SMqRspObj   rspObj = {0};
-  SDecoder    decoder = {0};
-  STableMeta* pTableMeta = NULL;
-
-  SRequestObj* pRequest = NULL;
-  RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
-
-  uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
-  pRequest->syncQuery = true;
-  rspObj.resIter = -1;
-  rspObj.resType = RES_TYPE__TMQ;
-
-  int8_t dataVersion = *(int8_t*)data;
-  if (dataVersion >= MQ_DATA_RSP_VERSION) {
-    data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
-    dataLen -= sizeof(int8_t) + sizeof(int32_t);
-  }
-  tDecoderInit(&decoder, data, dataLen);
-  code = tDecodeMqDataRsp(&decoder, &rspObj.dataRsp);
-  if (code != 0) {
-    SET_ERROR_MSG("decode mq data rsp failed");
-    code = TSDB_CODE_INVALID_MSG;
-    goto end;
-  }
-
-  if (!pRequest->pDb) {
-    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
-    goto end;
-  }
-
-  struct SCatalog* pCatalog = NULL;
-  RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
-
-  SRequestConnInfo conn = {0};
-  conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter;
-  conn.requestId = pRequest->requestId;
-  conn.requestObjRefId = pRequest->self;
-  conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
-
-  RAW_RETURN_CHECK(smlInitHandle(&pQuery));
-  pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
-  RAW_NULL_CHECK(pVgHash);
-  while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
-    void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
-    RAW_NULL_CHECK(pRetrieve);
-    if (!rspObj.dataRsp.withSchema) {
-      goto end;
-    }
-
-    const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
-    RAW_NULL_CHECK(tbName);
-
-    SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
-    tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
-    tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
-
-    RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
-
-    SVgroupInfo vg = {0};
-    RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg));
-
-    void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
-    if (hData == NULL) {
-      RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
-    }
-
-    SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
-    RAW_NULL_CHECK(pSW);
-    TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
-    RAW_NULL_CHECK(fields);
-    for (int i = 0; i < pSW->nCols; i++) {
-      fields[i].type = pSW->pSchema[i].type;
-      fields[i].bytes = pSW->pSchema[i].bytes;
-      tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
-    }
-    void* rawData = getRawDataFromRes(pRetrieve);
-    char  err[ERR_MSG_LEN] = {0};
-    code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN);
-    taosMemoryFree(fields);
-    taosMemoryFreeClear(pTableMeta);
-    if (code != TSDB_CODE_SUCCESS) {
-      SET_ERROR_MSG("table:%s, err:%s", tbName, err);
-      goto end;
-    }
-  }
-
-  RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
-
-  launchQueryImpl(pRequest, pQuery, true, NULL);
-  code = pRequest->code;
-
-end:
-  uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
-  tDeleteMqDataRsp(&rspObj.dataRsp);
-  tDecoderClear(&decoder);
-  qDestroyQuery(pQuery);
-  destroyRequest(pRequest);
-  taosHashCleanup(pVgHash);
-  taosMemoryFreeClear(pTableMeta);
-  return code;
-}
-
 static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
   // find schema data info
   int32_t       code = 0;
@@ -1855,152 +1725,368 @@ end:
   return code;
 }
 
-static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
+typedef enum {
-  if (taos == NULL || data == NULL) {
+  WRITE_RAW_INIT_START = 0,
-    SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
+  WRITE_RAW_INIT_OK,
-    return TSDB_CODE_INVALID_PARA;
+  WRITE_RAW_INIT_FAIL,
+} WRITE_RAW_INIT_STATUS;
+
+static SHashObj* writeRawCache = NULL;
+static int8_t    initFlag = 0;
+static int8_t    initedFlag = WRITE_RAW_INIT_START;
+
+typedef struct {
+  SHashObj* pVgHash;
+  SHashObj* pNameHash;
+  SHashObj* pMetaHash;
+} rawCacheInfo;
+
+typedef struct {
+  SVgroupInfo vgInfo;
+  int64_t     uid;
+  int64_t     suid;
+} tbInfo;
+
+static void tmqFreeMeta(void* data) {
+  STableMeta* pTableMeta = *(STableMeta**)data;
+  taosMemoryFree(pTableMeta);
 }
-  int32_t        code = TSDB_CODE_SUCCESS;
-  SHashObj*      pVgHash = NULL;
-  SQuery*        pQuery = NULL;
-  SMqRspObj      rspObj = {0};
-  SDecoder       decoder = {0};
-  STableMeta*    pTableMeta = NULL;
-  SHashObj*      pCreateTbHash = NULL;
 
-  SRequestObj* pRequest = NULL;
+static void freeRawCache(void* data) {
-  RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
+  rawCacheInfo* pRawCache = (rawCacheInfo*)data;
+  taosHashCleanup(pRawCache->pMetaHash);
+  taosHashCleanup(pRawCache->pNameHash);
+  taosHashCleanup(pRawCache->pVgHash);
+}
 
-  uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
+static int32_t initRawCacheHash() {
-  pRequest->syncQuery = true;
+  if (writeRawCache == NULL) {
-  rspObj.resIter = -1;
+    writeRawCache = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
-  rspObj.resType = RES_TYPE__TMQ_METADATA;
+    if (writeRawCache == NULL) {
+      return terrno;
+    }
+    taosHashSetFreeFp(writeRawCache, freeRawCache);
+  }
+  return 0;
+}
 
+static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW) {
+  char* p = (char*)rawData;
+  // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
+  // column length |
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(uint64_t);
+  int8_t* fields = p;
+
+  if (pSW->nCols != pTableMeta->tableInfo.numOfColumns) {
+    return true;
+  }
+  for (int i = 0; i < pSW->nCols; i++) {
+    int j = 0;
+    for (; j < pTableMeta->tableInfo.numOfColumns; j++) {
+      SSchema* pColSchema = &pTableMeta->schema[j];
+      char*    fieldName = pSW->pSchema[i].name;
+
+      if (strcmp(pColSchema->name, fieldName) == 0) {
+        if (*fields != pColSchema->type || *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
+          return true;
+        }
+        break;
+      }
+    }
+    fields += sizeof(int8_t) + sizeof(int32_t);
+
+    if (j == pTableMeta->tableInfo.numOfColumns) return true;
+  }
+  return false;
+}
+
+static int32_t getRawCache(SHashObj** pVgHash, SHashObj** pNameHash, SHashObj** pMetaHash, void* key) {
+  int32_t code = 0;
+  void*   cacheInfo = taosHashGet(writeRawCache, &key, POINTER_BYTES);
+  if (cacheInfo == NULL) {
+    *pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(*pVgHash);
+    *pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(*pNameHash);
+    *pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+    RAW_NULL_CHECK(*pMetaHash);
+    taosHashSetFreeFp(*pMetaHash, tmqFreeMeta);
+    rawCacheInfo info = {*pVgHash, *pNameHash, *pMetaHash};
+    RAW_RETURN_CHECK(taosHashPut(writeRawCache, &key, POINTER_BYTES, &info, sizeof(rawCacheInfo)));
+  } else {
+    rawCacheInfo* info = (rawCacheInfo*)cacheInfo;
+    *pVgHash = info->pVgHash;
+    *pNameHash = info->pNameHash;
+    *pMetaHash = info->pMetaHash;
+  }
+
+  return 0;
+end:
+  taosHashCleanup(*pMetaHash);
+  taosHashCleanup(*pNameHash);
+  taosHashCleanup(*pVgHash);
+  return code;
+}
+
+static int32_t buildRawRequest(TAOS* taos, SRequestObj** pRequest, SCatalog** pCatalog, SRequestConnInfo* conn) {
+  int32_t code = 0;
+  RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, pRequest));
+  (*pRequest)->syncQuery = true;
+  if (!(*pRequest)->pDb) {
+    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
+    goto end;
+  }
+
+  RAW_RETURN_CHECK(catalogGetHandle((*pRequest)->pTscObj->pAppInfo->clusterId, pCatalog));
+  conn->pTrans = (*pRequest)->pTscObj->pAppInfo->pTransporter;
+  conn->requestId = (*pRequest)->requestId;
+  conn->requestObjRefId = (*pRequest)->self;
+  conn->mgmtEps = getEpSet_s(&(*pRequest)->pTscObj->pAppInfo->mgmtEp);
+
+end:
+  return code;
+}
+
+typedef int32_t _raw_decode_func_(SDecoder* pDecoder, SMqDataRsp* pRsp);
+static int32_t  decodeRawData(SDecoder* decoder, void* data, int32_t dataLen, _raw_decode_func_ func,
+                              SMqRspObj* rspObj) {
    int8_t dataVersion = *(int8_t*)data;
    if (dataVersion >= MQ_DATA_RSP_VERSION) {
      data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
      dataLen -= sizeof(int8_t) + sizeof(int32_t);
   }
 
-  tDecoderInit(&decoder, data, dataLen);
+   rspObj->resIter = -1;
-  code = tDecodeSTaosxRsp(&decoder, &rspObj.dataRsp);
+   tDecoderInit(decoder, data, dataLen);
+   int32_t code = func(decoder, &rspObj->dataRsp);
    if (code != 0) {
      SET_ERROR_MSG("decode mq taosx data rsp failed");
-    code = TSDB_CODE_INVALID_MSG;
+  }
+   return code;
+}
+
+static int32_t processCacheMeta(SHashObj* pVgHash, SHashObj* pNameHash, SHashObj* pMetaHash,
+                                SVCreateTbReq* pCreateReqDst, SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName,
+                                STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry) {
+  int32_t     code = 0;
+  STableMeta* pTableMeta = NULL;
+  tbInfo*     tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
+  if (tmpInfo == NULL || retry > 0) {
+    tbInfo info = {0};
+
+    RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, conn, pName, &info.vgInfo));
+    if (pCreateReqDst && tmpInfo == NULL) {  // change stable name to get meta
+      tstrncpy(pName->tname, pCreateReqDst->ctb.stbName, TSDB_TABLE_NAME_LEN);
+    }
+    RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
+    info.uid = pTableMeta->uid;
+    if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
+      info.suid = pTableMeta->suid;
+    } else {
+      info.suid = pTableMeta->uid;
+    }
+    code = taosHashPut(pMetaHash, &info.suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
+    if (code != 0) {
+      taosMemoryFree(pTableMeta);
+      goto end;
+    }
+    if (pCreateReqDst) {
+      pTableMeta->vgId = info.vgInfo.vgId;
+      pTableMeta->uid = pCreateReqDst->uid;
+      pCreateReqDst->ctb.suid = pTableMeta->suid;
+    }
+
+    RAW_RETURN_CHECK(taosHashPut(pNameHash, pName->tname, strlen(pName->tname), &info, sizeof(tbInfo)));
+    tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
+    RAW_RETURN_CHECK(
+        taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo)));
+  }
+
+  if (pTableMeta == NULL || retry > 0) {
+    STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, &tmpInfo->suid, LONG_BYTES);
+    if (pTableMetaTmp == NULL || retry > 0 || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
+      RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
+      code = taosHashPut(pMetaHash, &tmpInfo->suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
+      if (code != 0) {
+        taosMemoryFree(pTableMeta);
         goto end;
       }
 
-  if (!pRequest->pDb) {
+    } else {
-    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
+      pTableMeta = *pTableMetaTmp;
-    goto end;
+      pTableMeta->uid = tmpInfo->uid;
+      pTableMeta->vgId = tmpInfo->vgInfo.vgId;
+    }
+  }
+  *pMeta = pTableMeta;
+
+end:
+  return code;
 }
 
-  struct SCatalog* pCatalog = NULL;
+static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
-  RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
+  int32_t   code = TSDB_CODE_SUCCESS;
+  SQuery*   pQuery = NULL;
+  SMqRspObj rspObj = {0};
+  SDecoder  decoder = {0};
 
+  SRequestObj*     pRequest = NULL;
+  SCatalog*        pCatalog = NULL;
   SRequestConnInfo conn = {0};
-  conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter;
+  RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
-  conn.requestId = pRequest->requestId;
+  uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
-  conn.requestObjRefId = pRequest->self;
+  RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeMqDataRsp, &rspObj));
-  conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
 
+  SHashObj* pVgHash = NULL;
+  SHashObj* pNameHash = NULL;
+  SHashObj* pMetaHash = NULL;
+  RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
+  int retry = 0;
+  while (1) {
     RAW_RETURN_CHECK(smlInitHandle(&pQuery));
-  pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
-  RAW_NULL_CHECK(pVgHash);
-  pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
-  RAW_NULL_CHECK(pCreateTbHash);
-  RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
-
     uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
     while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
-    void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
-    RAW_NULL_CHECK(pRetrieve);
       if (!rspObj.dataRsp.withSchema) {
         goto end;
       }
 
       const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
-    if (!tbName) {
+      RAW_NULL_CHECK(tbName);
-      SET_ERROR_MSG("block tbname is null");
+      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
-      code = terrno;
+      RAW_NULL_CHECK(pSW);
+      void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
+      RAW_NULL_CHECK(pRetrieve);
+      void* rawData = getRawDataFromRes(pRetrieve);
+      RAW_NULL_CHECK(rawData);
+
+      uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
+      SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
+      tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
+      tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
+
+      STableMeta* pTableMeta = NULL;
+      RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn, &pName, &pTableMeta, pSW,
+                                        rawData, retry));
+      char err[ERR_MSG_LEN] = {0};
+      code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
+      if (code != TSDB_CODE_SUCCESS) {
+        SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
+        goto end;
+      }
+    }
+    RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
+    launchQueryImpl(pRequest, pQuery, true, NULL);
+    code = pRequest->code;
+
+    if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) {
+      uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
+      qDestroyQuery(pQuery);
+      pQuery = NULL;
+      rspObj.resIter = -1;
+      continue;
+    }
+    break;
+  }
+
+end:
+  uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
+  tDeleteMqDataRsp(&rspObj.dataRsp);
+  tDecoderClear(&decoder);
+  qDestroyQuery(pQuery);
+  destroyRequest(pRequest);
+  return code;
+}
+
+static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
+  int32_t   code = TSDB_CODE_SUCCESS;
+  SQuery*   pQuery = NULL;
+  SMqRspObj rspObj = {0};
+  SDecoder  decoder = {0};
+  SHashObj* pCreateTbHash = NULL;
+
+  SRequestObj*     pRequest = NULL;
+  SCatalog*        pCatalog = NULL;
+  SRequestConnInfo conn = {0};
+
+  RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
+  uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
+  RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeSTaosxRsp, &rspObj));
+
+  pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  RAW_NULL_CHECK(pCreateTbHash);
+  RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
+
+  SHashObj* pVgHash = NULL;
+  SHashObj* pNameHash = NULL;
+  SHashObj* pMetaHash = NULL;
+  RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
+  int retry = 0;
+  while (1) {
+    RAW_RETURN_CHECK(smlInitHandle(&pQuery));
+    uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
+    while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
+      if (!rspObj.dataRsp.withSchema) {
         goto end;
       }
 
-    uDebug(LOG_ID_TAG " write raw metadata block tbname:%s", LOG_ID_VALUE, tbName);
+      const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
+      RAW_NULL_CHECK(tbName);
+      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
+      RAW_NULL_CHECK(pSW);
+      void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
+      RAW_NULL_CHECK(pRetrieve);
+      void* rawData = getRawDataFromRes(pRetrieve);
+      RAW_NULL_CHECK(rawData);
+
+      uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
       SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
       tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
       tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
 
       // find schema data info
-    SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
+      SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, pName.tname, strlen(pName.tname));
-    SVgroupInfo    vg = {0};
+      STableMeta*    pTableMeta = NULL;
-    RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg));
+      RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, pCreateReqDst, pCatalog, &conn, &pName,
-    if (pCreateReqDst) {  // change stable name to get meta
+                                        &pTableMeta, pSW, rawData, retry));
-      tstrncpy(pName.tname, pCreateReqDst->ctb.stbName, TSDB_TABLE_NAME_LEN);
-    }
-    RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
-
-    if (pCreateReqDst) {
-      pTableMeta->vgId = vg.vgId;
-      pTableMeta->uid = pCreateReqDst->uid;
-      pCreateReqDst->ctb.suid = pTableMeta->suid;
-    }
-    void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
-    if (hData == NULL) {
-      RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
-    }
-
-    SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
-    RAW_NULL_CHECK(pSW);
-    TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
-    if (fields == NULL) {
-      SET_ERROR_MSG("calloc fields failed");
-      code = terrno;
-      goto end;
-    }
-    for (int i = 0; i < pSW->nCols; i++) {
-      fields[i].type = pSW->pSchema[i].type;
-      fields[i].bytes = pSW->pSchema[i].bytes;
-      tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
-    }
-    void*          rawData = getRawDataFromRes(pRetrieve);
       char err[ERR_MSG_LEN] = {0};
-    SVCreateTbReq* pCreateReqTmp = NULL;
+      code =
-    if (pCreateReqDst) {
+          rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
-      RAW_RETURN_CHECK(cloneSVreateTbReq(pCreateReqDst, &pCreateReqTmp));
-    }
-    code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN);
-    if (pCreateReqTmp != NULL) {
-      tdDestroySVCreateTbReq(pCreateReqTmp);
-      taosMemoryFree(pCreateReqTmp);
-    }
-    taosMemoryFree(fields);
-    taosMemoryFreeClear(pTableMeta);
       if (code != TSDB_CODE_SUCCESS) {
-      SET_ERROR_MSG("table:%s, err:%s", tbName, err);
+        SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
         goto end;
       }
     }
-
     RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
-
     launchQueryImpl(pRequest, pQuery, true, NULL);
     code = pRequest->code;
 
+    if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) {
+      uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
+      qDestroyQuery(pQuery);
+      pQuery = NULL;
+      rspObj.resIter = -1;
+      continue;
+    }
+    break;
+  }
+
 end:
   uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
+  tDeleteSTaosxRsp(&rspObj.dataRsp);
   void* pIter = taosHashIterate(pCreateTbHash, NULL);
   while (pIter) {
     tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
     pIter = taosHashIterate(pCreateTbHash, pIter);
   }
   taosHashCleanup(pCreateTbHash);
-  tDeleteSTaosxRsp(&rspObj.dataRsp);
   tDecoderClear(&decoder);
   qDestroyQuery(pQuery);
   destroyRequest(pRequest);
-  taosHashCleanup(pVgHash);
-  taosMemoryFreeClear(pTableMeta);
   return code;
 }
 
@@ -2199,7 +2285,31 @@ void tmq_free_raw(tmq_raw_data raw) {
   (void)memset(terrMsg, 0, ERR_MSG_LEN);
 }
 
+static int32_t writeRawInit() {
+  while (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_START) {
+    int8_t old = atomic_val_compare_exchange_8(&initFlag, 0, 1);
+    if (old == 0) {
+      int32_t code = initRawCacheHash();
+      if (code != 0) {
+        uError("tmq writeRawImpl init error:%d", code);
+        atomic_store_8(&initedFlag, WRITE_RAW_INIT_FAIL);
+        return code;
+      }
+      atomic_store_8(&initedFlag, WRITE_RAW_INIT_OK);
+    }
+  }
+
+  if (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_FAIL) {
+    return TSDB_CODE_INTERNAL_ERROR;
+  }
+  return 0;
+}
+
 static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) {
+  if (writeRawInit() != 0) {
+    return TSDB_CODE_INTERNAL_ERROR;
+  }
+
   if (type == TDMT_VND_CREATE_STB) {
     return taosCreateStb(taos, buf, len);
   } else if (type == TDMT_VND_ALTER_STB) {
@@ -2214,10 +2324,10 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
     return taosDropTable(taos, buf, len);
   } else if (type == TDMT_VND_DELETE) {
     return taosDeleteData(taos, buf, len);
-  } else if (type == RES_TYPE__TMQ) {
-    return tmqWriteRawDataImpl(taos, buf, len);
   } else if (type == RES_TYPE__TMQ_METADATA) {
     return tmqWriteRawMetaDataImpl(taos, buf, len);
+  } else if (type == RES_TYPE__TMQ) {
+    return tmqWriteRawDataImpl(taos, buf, len);
   } else if (type == RES_TYPE__TMQ_BATCH_META) {
     return tmqWriteBatchMetaDataImpl(taos, buf, len);
   }
@@ -2225,7 +2335,8 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
 }
 
 int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {
-  if (!taos) {
+  if (taos == NULL || raw.raw == NULL || raw.raw_len <= 0) {
+    SET_ERROR_MSG("taos:%p or data:%p is NULL or raw_len <= 0", taos, raw.raw);
     return TSDB_CODE_INVALID_PARA;
   }
 

source/client/src/clientTmq.c

@@ -24,12 +24,9 @@
 #include "tref.h"
 #include "ttimer.h"
 
-#define tqFatalC(...) do { if (cDebugFlag & DEBUG_FATAL || tqClientDebug) { taosPrintLog("TQ  FATAL ", DEBUG_FATAL, tqDebugFlag, __VA_ARGS__); }}     while(0)
+#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ  ERROR ", DEBUG_ERROR, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }}     while(0)
-#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebug) { taosPrintLog("TQ  ERROR ", DEBUG_ERROR, tqDebugFlag, __VA_ARGS__); }}     while(0)
+#define tqInfoC(...)  do { if (cDebugFlag & DEBUG_INFO || tqClientDebugFlag & DEBUG_INFO)  { taosPrintLog("TQ  ", DEBUG_INFO, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }}            while(0)
-#define tqWarnC(...)  do { if (cDebugFlag & DEBUG_WARN || tqClientDebug)  { taosPrintLog("TQ  WARN ", DEBUG_WARN, tqDebugFlag, __VA_ARGS__); }}       while(0)
+#define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ  ", DEBUG_DEBUG, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }} while(0)
-#define tqInfoC(...)  do { if (cDebugFlag & DEBUG_INFO || tqClientDebug)  { taosPrintLog("TQ  ", DEBUG_INFO, tqDebugFlag, __VA_ARGS__); }}            while(0)
-#define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebug) { taosPrintLog("TQ  ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
-#define tqTraceC(...) do { if (cDebugFlag & DEBUG_TRACE || tqClientDebug) { taosPrintLog("TQ  ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
 
 #define EMPTY_BLOCK_POLL_IDLE_DURATION 10
 #define DEFAULT_AUTO_COMMIT_INTERVAL   5000
@@ -831,8 +828,8 @@ static int32_t innerCommitAll(tmq_t* tmq, SMqCommitCbParamSet* pParamSet){
       }
 
       code = innerCommit(tmq, pTopic->topicName, &pVg->offsetInfo.endOffset, pVg, pParamSet);
-      if (code != 0){
+      if (code != 0 && code != TSDB_CODE_TMQ_SAME_COMMITTED_VALUE){
-        tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d",
+        tqErrorC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d",
                  tmq->consumerId, pTopic->topicName, pVg->vgId, tstrerror(code), pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups);
       }
     }
@@ -857,7 +854,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
     return;
   }
   code = innerCommitAll(tmq, pParamSet);
-  if (code != 0){
+  if (code != 0 && code != TSDB_CODE_TMQ_SAME_COMMITTED_VALUE){
     tqErrorC("consumer:0x%" PRIx64 " innerCommitAll failed, code:%s", tmq->consumerId, tstrerror(code));
   }
 
@@ -957,7 +954,8 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
     }
   }
 
-  tqClientDebug = rsp.debugFlag;
+  tqClientDebugFlag = rsp.debugFlag;
+
   tDestroySMqHbRsp(&rsp);
 
 END:
@@ -978,6 +976,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
   req.consumerId = tmq->consumerId;
   req.epoch = tmq->epoch;
   req.pollFlag = atomic_load_8(&tmq->pollFlag);
+  tqDebugC("consumer:0x%" PRIx64 " send heartbeat, pollFlag:%d", tmq->consumerId, req.pollFlag);
   req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
   if (req.topics == NULL) {
     goto END;
@@ -1063,7 +1062,7 @@ END:
   tDestroySMqHbReq(&req);
   if (tmrId != NULL) {
     bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer);
-    tqDebugC("reset timer fo tmq hb:%d", ret);
+    tqDebugC("consumer:0x%" PRIx64 " reset timer for tmq heartbeat:%d, pollFlag:%d", tmq->consumerId, ret, tmq->pollFlag);
   }
   int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId);
   if (ret != 0){
@@ -1269,7 +1268,9 @@ static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   if (code != TSDB_CODE_SUCCESS) {
+    if (code != TSDB_CODE_MND_CONSUMER_NOT_READY){
       tqErrorC("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code));
+    }
     goto END;
   }
 
@@ -1422,7 +1423,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
       tqDebugC("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
       bool ret = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer,
                               &pTmq->epTimer);
-      tqDebugC("reset timer fo tmq ask ep:%d", ret);
+      tqDebugC("reset timer for tmq ask ep:%d", ret);
     } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
       tmq_commit_cb* pCallbackFn = (pTmq->commitCb != NULL) ? pTmq->commitCb : defaultCommitCbFn;
       asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
@@ -1430,7 +1431,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
                pTmq->autoCommitInterval / 1000.0);
       bool ret = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer,
                               &pTmq->commitTimer);
-      tqDebugC("reset timer fo commit:%d", ret);
+      tqDebugC("reset timer for commit:%d", ret);
     } else {
       tqErrorC("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
     }

source/common/src/systable.c

@@ -118,7 +118,7 @@ static const SSysDbTableSchema userDBSchema[] = {
     {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
     {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
-    {.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
+    {.name = "s3_chunkpages", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
     {.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},

source/common/src/tglobal.c

@@ -548,7 +548,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
-  TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebug", tqClientDebug, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
+  TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebugFlag", tqClientDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
@@ -2000,7 +2000,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
         {"tdbDebugFlag", &tdbDebugFlag},   {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag},
         {"smaDebugFlag", &smaDebugFlag},   {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag},
         {"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag},   {"sndDebugFlag", &sndDebugFlag},
-        {"tqClientDebug", &tqClientDebug},
+        {"tqClientDebugFlag", &tqClientDebugFlag},
     };
 
     static OptionNameAndVar options[] = {{"audit", &tsEnableAudit},

source/common/src/tmsg.c

@@ -10983,6 +10983,7 @@ _exit:
 int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
   TAOS_CHECK_RETURN(tEncodeMqDataRspCommon(pEncoder, pRsp));
   TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->sleepTime));
+
   return 0;
 }
 
@@ -11094,6 +11095,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
       TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, createTableReq, createTableLen));
     }
   }
+
 _exit:
   return code;
 }

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -36,12 +36,13 @@ typedef struct SVnodeMgmt {
   SSingleWorker         mgmtWorker;
   SSingleWorker         mgmtMultiWorker;
   SHashObj             *hash;
+  SHashObj             *closedHash;
   TdThreadRwlock        lock;
   SVnodesStat           state;
   STfs                 *pTfs;
   TdThread              thread;
   bool                  stop;
-  TdThreadMutex         createLock;
+  TdThreadMutex         fileLock;
 } SVnodeMgmt;
 
 typedef struct {
@@ -94,7 +95,7 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId);
 SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict);
 void       vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode);
 int32_t    vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl);
-void       vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal);
+void       vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal, bool keepClosed);
 
 // vmHandle.c
 SArray *vmGetMsgHandles();
@@ -111,6 +112,7 @@ int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
 int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
 int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt);
 int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
+int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
 
 // vmWorker.c
 int32_t vmStartWorker(SVnodeMgmt *pMgmt);

source/dnode/mgmt/mgmt_vnode/src/vmFile.c

@@ -19,6 +19,54 @@
 
 #define MAX_CONTENT_LEN 2 * 1024 * 1024
 
+int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
+  (void)taosThreadRwlockRdlock(&pMgmt->lock);
+
+  int32_t num = 0;
+  int32_t size = taosHashGetSize(pMgmt->hash);
+  int32_t closedSize = taosHashGetSize(pMgmt->closedHash);
+  size += closedSize;
+  SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *));
+  if (pVnodes == NULL) {
+    (void)taosThreadRwlockUnlock(&pMgmt->lock);
+    return terrno;
+  }
+
+  void *pIter = taosHashIterate(pMgmt->hash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    SVnodeObj  *pVnode = *ppVnode;
+    if (pVnode && num < size) {
+      int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+      // dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount);
+      pVnodes[num++] = (*ppVnode);
+      pIter = taosHashIterate(pMgmt->hash, pIter);
+    } else {
+      taosHashCancelIterate(pMgmt->hash, pIter);
+    }
+  }
+
+  pIter = taosHashIterate(pMgmt->closedHash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    SVnodeObj  *pVnode = *ppVnode;
+    if (pVnode && num < size) {
+      int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
+      // dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount);
+      pVnodes[num++] = (*ppVnode);
+      pIter = taosHashIterate(pMgmt->closedHash, pIter);
+    } else {
+      taosHashCancelIterate(pMgmt->closedHash, pIter);
+    }
+  }
+
+  (void)taosThreadRwlockUnlock(&pMgmt->lock);
+  *numOfVnodes = num;
+  *ppVnodes = pVnodes;
+
+  return 0;
+}
+
 int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
   (void)taosThreadRwlockRdlock(&pMgmt->lock);
 
@@ -203,6 +251,8 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
   SVnodeObj **ppVnodes = NULL;
   char        file[PATH_MAX] = {0};
   char        realfile[PATH_MAX] = {0};
+  int32_t     lino = 0;
+  int32_t     ret = -1;
 
   int32_t nBytes = snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
   if (nBytes <= 0 || nBytes >= sizeof(file)) {
@@ -215,8 +265,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
   }
 
   int32_t numOfVnodes = 0;
-  code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
+  TAOS_CHECK_GOTO(vmGetAllVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes), &lino, _OVER);
-  if (code) goto _OVER;
 
   // terrno = TSDB_CODE_OUT_OF_MEMORY;
   pJson = tjsonCreateObject();
@@ -224,39 +273,56 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
     code = terrno;
     goto _OVER;
   }
-  if ((code = vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes)) != 0) goto _OVER;
+  TAOS_CHECK_GOTO(vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes), &lino, _OVER);
 
   buffer = tjsonToString(pJson);
   if (buffer == NULL) {
     code = TSDB_CODE_INVALID_JSON_FORMAT;
+    lino = __LINE__;
+    goto _OVER;
+  }
+
+  code = taosThreadMutexLock(&pMgmt->fileLock);
+  if (code != 0) {
+    lino = __LINE__;
     goto _OVER;
   }
 
   pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
   if (pFile == NULL) {
     code = terrno;
-    goto _OVER;
+    lino = __LINE__;
+    goto _OVER1;
   }
 
   int32_t len = strlen(buffer);
   if (taosWriteFile(pFile, buffer, len) <= 0) {
     code = terrno;
-    goto _OVER;
+    lino = __LINE__;
+    goto _OVER1;
   }
   if (taosFsyncFile(pFile) < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
-    goto _OVER;
+    lino = __LINE__;
+    goto _OVER1;
   }
 
   code = taosCloseFile(&pFile);
   if (code != 0) {
     code = TAOS_SYSTEM_ERROR(errno);
-    goto _OVER;
+    lino = __LINE__;
+    goto _OVER1;
   }
-  TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER);
+  TAOS_CHECK_GOTO(taosRenameFile(file, realfile), &lino, _OVER1);
 
   dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes);
 
+_OVER1:
+  ret = taosThreadMutexUnlock(&pMgmt->fileLock);
+  if (ret != 0) {
+    dError("failed to unlock since %s", tstrerror(ret));
+  }
+
 _OVER:
   if (pJson != NULL) tjsonDelete(pJson);
   if (buffer != NULL) taosMemoryFree(buffer);
@@ -272,7 +338,8 @@ _OVER:
   }
 
   if (code != 0) {
-    dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, tstrerror(code), numOfVnodes);
+    dError("failed to write vnodes file:%s at line:%d since %s, vnodes:%d", realfile, lino, tstrerror(code),
+           numOfVnodes);
   }
   return code;
 }

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -415,27 +415,30 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     goto _OVER;
   }
 
-  code = taosThreadMutexLock(&pMgmt->createLock);
-  if (code != 0) {
-    dError("vgId:%d, failed to lock since %s", req.vgId, tstrerror(code));
-    goto _OVER;
-  }
   code = vmWriteVnodeListToFile(pMgmt);
   if (code != 0) {
     code = terrno != 0 ? terrno : code;
-    int32_t ret = taosThreadMutexUnlock(&pMgmt->createLock);
-    if (ret != 0) {
-      dError("vgId:%d, failed to unlock since %s", req.vgId, tstrerror(ret));
-    }
     goto _OVER;
   }
-  int32_t ret = taosThreadMutexUnlock(&pMgmt->createLock);
-  if (ret != 0) {
-    dError("vgId:%d, failed to unlock since %s", req.vgId, tstrerror(ret));
-  }
 
 _OVER:
   if (code != 0) {
+    int32_t r = 0;
+    r = taosThreadRwlockWrlock(&pMgmt->lock);
+    if (r != 0) {
+      dError("vgId:%d, failed to lock since %s", req.vgId, tstrerror(r));
+    }
+    if (r == 0) {
+      dInfo("vgId:%d, remove from hash", req.vgId);
+      r = taosHashRemove(pMgmt->hash, &req.vgId, sizeof(int32_t));
+      if (r != 0) {
+        dError("vgId:%d, failed to remove vnode since %s", req.vgId, tstrerror(r));
+      }
+    }
+    r = taosThreadRwlockUnlock(&pMgmt->lock);
+    if (r != 0) {
+      dError("vgId:%d, failed to unlock since %s", req.vgId, tstrerror(r));
+    }
     vnodeClose(pImpl);
     vnodeDestroy(0, path, pMgmt->pTfs, 0);
   } else {
@@ -535,7 +538,7 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
 
   bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
-  vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
+  vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal, true);
 
   int32_t diskPrimary = wrapperCfg.diskPrimary;
   char    path[TSDB_FILENAME_LEN] = {0};
@@ -683,7 +686,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   dInfo("vgId:%d, close vnode", srcVgId);
-  vmCloseVnode(pMgmt, pVnode, true);
+  vmCloseVnode(pMgmt, pVnode, true, false);
 
   int32_t diskPrimary = wrapperCfg.diskPrimary;
   char    srcPath[TSDB_FILENAME_LEN] = {0};
@@ -792,7 +795,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
 
   bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
-  vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
+  vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal, true);
 
   int32_t diskPrimary = wrapperCfg.diskPrimary;
   char    path[TSDB_FILENAME_LEN] = {0};
@@ -860,7 +863,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     return code;
   }
 
-  vmCloseVnode(pMgmt, pVnode, false);
+  vmCloseVnode(pMgmt, pVnode, false, false);
   if (vmWriteVnodeListToFile(pMgmt) != 0) {
     dError("vgId:%d, failed to write vnode list since %s", vgId, terrstr());
   }

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -166,16 +166,34 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
   (void)taosThreadRwlockWrlock(&pMgmt->lock);
   SVnodeObj *pOld = NULL;
   int32_t    r = taosHashGetDup(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld);
+  if (r != 0) {
+    dError("vgId:%d, failed to get vnode from hash", pVnode->vgId);
+  }
   if (pOld) {
     vmFreeVnodeObj(&pOld);
   }
   int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
+
+  pOld = NULL;
+  r = taosHashGetDup(pMgmt->closedHash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld);
+  if (r != 0) {
+    dError("vgId:%d, failed to get vnode from closedHash", pVnode->vgId);
+  }
+  if (pOld) {
+    vmFreeVnodeObj(&pOld);
+  }
+
+  dInfo("vgId:%d, remove from closedHash", pVnode->vgId);
+  r = taosHashRemove(pMgmt->closedHash, &pVnode->vgId, sizeof(int32_t));
+  if (r != 0) {
+    dError("vgId:%d, failed to remove vnode from hash", pVnode->vgId);
+  }
   (void)taosThreadRwlockUnlock(&pMgmt->lock);
 
   return code;
 }
 
-void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) {
+void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal, bool keepClosed) {
   char path[TSDB_FILENAME_LEN] = {0};
   bool atExit = true;
 
@@ -185,7 +203,40 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
 
   (void)taosThreadRwlockWrlock(&pMgmt->lock);
   int32_t r = taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
+  if (r != 0) {
+    dError("vgId:%d, failed to remove vnode from hash", pVnode->vgId);
+  }
+  if (keepClosed) {
+    SVnodeObj *pClosedVnode = taosMemoryCalloc(1, sizeof(SVnodeObj));
+    (void)memset(pClosedVnode, 0, sizeof(SVnodeObj));
+    if (pVnode == NULL) {
+      dError("vgId:%d, failed to alloc vnode since %s", pVnode->vgId, terrstr());
       (void)taosThreadRwlockUnlock(&pMgmt->lock);
+      return;
+    }
+
+    pClosedVnode->vgId = pVnode->vgId;
+    pClosedVnode->dropped = pVnode->dropped;
+    pClosedVnode->vgVersion = pVnode->vgVersion;
+    pClosedVnode->diskPrimary = pVnode->diskPrimary;
+    pClosedVnode->toVgId = pVnode->toVgId;
+
+    SVnodeObj *pOld = NULL;
+    r = taosHashGetDup(pMgmt->closedHash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld);
+    if (r != 0) {
+      dError("vgId:%d, failed to get vnode from closedHash", pVnode->vgId);
+    }
+    if (pOld) {
+      vmFreeVnodeObj(&pOld);
+    }
+    dInfo("vgId:%d, put vnode to closedHash", pVnode->vgId);
+    r = taosHashPut(pMgmt->closedHash, &pVnode->vgId, sizeof(int32_t), &pClosedVnode, sizeof(SVnodeObj *));
+    if (r != 0) {
+      dError("vgId:%d, failed to put vnode to closedHash", pVnode->vgId);
+    }
+  }
+  (void)taosThreadRwlockUnlock(&pMgmt->lock);
+
   vmReleaseVnode(pMgmt, pVnode);
 
   if (pVnode->failed) {
@@ -362,9 +413,15 @@ static void *vmOpenVnodeInThread(void *param) {
 static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
   pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
   if (pMgmt->hash == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
     dError("failed to init vnode hash since %s", terrstr());
-    return -1;
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  pMgmt->closedHash =
+      taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  if (pMgmt->hash == NULL) {
+    dError("failed to init vnode closed hash since %s", terrstr());
+    return TSDB_CODE_OUT_OF_MEMORY;
   }
 
   SWrapperCfg *pCfgs = NULL;
@@ -459,7 +516,7 @@ static void *vmCloseVnodeInThread(void *param) {
              pMgmt->state.openVnodes, pMgmt->state.totalVnodes);
     tmsgReportStartup("vnode-close", stepDesc);
 
-    vmCloseVnode(pMgmt, pVnode, false);
+    vmCloseVnode(pMgmt, pVnode, false, false);
   }
 
   dInfo("thread:%d, numOfVnodes:%d is closed", pThread->threadIndex, pThread->vnodeNum);
@@ -537,6 +594,18 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
     pMgmt->hash = NULL;
   }
 
+  void *pIter = taosHashIterate(pMgmt->closedHash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    vmFreeVnodeObj(ppVnode);
+    pIter = taosHashIterate(pMgmt->closedHash, pIter);
+  }
+
+  if (pMgmt->closedHash != NULL) {
+    taosHashCleanup(pMgmt->closedHash);
+    pMgmt->closedHash = NULL;
+  }
+
   dInfo("total vnodes:%d are all closed", numOfVnodes);
 }
 
@@ -545,7 +614,7 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
   vmStopWorker(pMgmt);
   vnodeCleanup();
   (void)taosThreadRwlockDestroy(&pMgmt->lock);
-  (void)taosThreadMutexDestroy(&pMgmt->createLock);
+  (void)taosThreadMutexDestroy(&pMgmt->fileLock);
   taosMemoryFree(pMgmt);
 }
 
@@ -637,7 +706,7 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
     goto _OVER;
   }
 
-  code = taosThreadMutexInit(&pMgmt->createLock, NULL);
+  code = taosThreadMutexInit(&pMgmt->fileLock, NULL);
   if (code != 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _OVER;

source/dnode/mnode/impl/src/mndArbGroup.c

@@ -15,13 +15,10 @@
 
 #define _DEFAULT_SOURCE
 #include "mndArbGroup.h"
-#include "audit.h"
 #include "mndDb.h"
 #include "mndDnode.h"
-#include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndTrans.h"
-#include "mndUser.h"
 #include "mndVgroup.h"
 
 #define ARBGROUP_VER_NUMBER   1
@@ -245,11 +242,11 @@ static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *p
   }
 
   for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
-    (void)memcpy(pOld->members[i].state.token, pNew->members[i].state.token, TSDB_ARB_TOKEN_SIZE);
+    tstrncpy(pOld->members[i].state.token, pNew->members[i].state.token, TSDB_ARB_TOKEN_SIZE);
   }
   pOld->isSync = pNew->isSync;
   pOld->assignedLeader.dnodeId = pNew->assignedLeader.dnodeId;
-  (void)memcpy(pOld->assignedLeader.token, pNew->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
+  tstrncpy(pOld->assignedLeader.token, pNew->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
   pOld->assignedLeader.acked = pNew->assignedLeader.acked;
   pOld->version++;
 
@@ -834,12 +831,12 @@ static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
     newGroup.dbUid = pUpdateGroup->dbUid;
     for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
       newGroup.members[i].info.dnodeId = pUpdateGroup->members[i].dnodeId;
-      (void)memcpy(newGroup.members[i].state.token, pUpdateGroup->members[i].token, TSDB_ARB_TOKEN_SIZE);
+      tstrncpy(newGroup.members[i].state.token, pUpdateGroup->members[i].token, TSDB_ARB_TOKEN_SIZE);
     }
 
     newGroup.isSync = pUpdateGroup->isSync;
     newGroup.assignedLeader.dnodeId = pUpdateGroup->assignedLeader.dnodeId;
-    (void)memcpy(newGroup.assignedLeader.token, pUpdateGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
+    tstrncpy(newGroup.assignedLeader.token, pUpdateGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
     newGroup.assignedLeader.acked = pUpdateGroup->assignedLeader.acked;
     newGroup.version = pUpdateGroup->version;
 
@@ -897,7 +894,7 @@ static void mndArbGroupSetAssignedLeader(SArbGroup *pGroup, int32_t index) {
   SArbGroupMember *pMember = &pGroup->members[index];
 
   pGroup->assignedLeader.dnodeId = pMember->info.dnodeId;
-  (void)strncpy(pGroup->assignedLeader.token, pMember->state.token, TSDB_ARB_TOKEN_SIZE);
+  tstrncpy(pGroup->assignedLeader.token, pMember->state.token, TSDB_ARB_TOKEN_SIZE);
   pGroup->assignedLeader.acked = false;
 }
 
@@ -979,7 +976,7 @@ bool mndUpdateArbGroupByHeartBeat(SArbGroup *pGroup, SVArbHbRspMember *pRspMembe
 
   // update token
   mndArbGroupDupObj(pGroup, pNewGroup);
-  (void)memcpy(pNewGroup->members[index].state.token, pRspMember->memberToken, TSDB_ARB_TOKEN_SIZE);
+  tstrncpy(pNewGroup->members[index].state.token, pRspMember->memberToken, TSDB_ARB_TOKEN_SIZE);
   pNewGroup->isSync = false;
 
   bool resetAssigned = false;

source/dnode/mnode/impl/src/mndConsumer.c

@@ -239,12 +239,13 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
   MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer));
   MND_TMQ_RETURN_CHECK(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user));
   atomic_store_32(&pConsumer->hbStatus, 0);
+  mDebug("consumer:0x%" PRIx64 " receive hb pollFlag:%d %d", consumerId, req.pollFlag, pConsumer->pollStatus);
   if (req.pollFlag == 1){
     atomic_store_32(&pConsumer->pollStatus, 0);
   }
 
   storeOffsetRows(pMnode, &req, pConsumer);
-  rsp.debugFlag = tqClientDebug;
+  rsp.debugFlag = tqClientDebugFlag;
   code = buildMqHbRsp(pMsg, &rsp);
 
 END:

source/dnode/mnode/sdb/src/sdbFile.c

@@ -400,8 +400,8 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
   pSdb->commitTerm = pSdb->applyTerm;
   pSdb->commitConfig = pSdb->applyConfig;
   memcpy(pSdb->tableVer, tableVer, sizeof(tableVer));
-  mInfo("read sdb file:%s success, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64, file, pSdb->commitIndex,
+  mInfo("vgId:1, trans:0, read sdb file:%s success, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64, file,
-        pSdb->commitTerm, pSdb->commitConfig);
+        pSdb->commitIndex, pSdb->commitTerm, pSdb->commitConfig);
 
 _OVER:
   if ((ret = taosCloseFile(&pFile)) != 0) {
@@ -573,7 +573,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb, int32_t skip_type) {
     pSdb->commitIndex = pSdb->applyIndex;
     pSdb->commitTerm = pSdb->applyTerm;
     pSdb->commitConfig = pSdb->applyConfig;
-    mInfo("write sdb file success, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s",
+    mInfo("vgId:1, trans:0, write sdb file success, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64
+          " file:%s",
           pSdb->commitIndex, pSdb->commitTerm, pSdb->commitConfig, curfile);
   }
 
@@ -610,8 +611,8 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) {
   if (code != 0) {
     mError("failed to write sdb file since %s", tstrerror(code));
   } else {
-    mInfo("write sdb file success, apply index:%" PRId64 " term:%" PRId64 " config:%" PRId64, pSdb->applyIndex,
+    mInfo("vgId:1, trans:0, write sdb file success, apply index:%" PRId64 ", term:%" PRId64 ", config:%" PRId64,
-          pSdb->applyTerm, pSdb->applyConfig);
+          pSdb->applyIndex, pSdb->applyTerm, pSdb->applyConfig);
   }
   (void)taosThreadMutexUnlock(&pSdb->filelock);
   return code;

source/dnode/vnode/inc/vnode.h

@@ -243,7 +243,7 @@ int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, con
 int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
 bool    tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
 int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
-int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
+int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, int64_t *createTime);
 int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished);
 
 // sma

source/dnode/vnode/src/inc/vnodeInt.h

@@ -163,7 +163,7 @@ int32_t         metaDropTables(SMeta* pMeta, SArray* tbUids);
 int             metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount);
 int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
 int             metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs);
-SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
+SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime);
 int32_t         metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
 int32_t         metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);

source/dnode/vnode/src/meta/metaQuery.c

@@ -371,7 +371,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) {
   return 0;
 }
 
-SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
+SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime) {
   void           *pData = NULL;
   int             nData = 0;
   int64_t         version;
@@ -407,6 +407,9 @@ _query:
     }
   } else if (me.type == TSDB_CHILD_TABLE) {
     uid = me.ctbEntry.suid;
+    if (createTime != NULL){
+      *createTime = me.ctbEntry.btime;
+    }
     tDecoderClear(&dc);
     goto _query;
   } else {
@@ -617,7 +620,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
   STSchema       *pTSchema = NULL;
   SSchemaWrapper *pSW = NULL;
 
-  pSW = metaGetTableSchema(pMeta, uid, sver, lock);
+  pSW = metaGetTableSchema(pMeta, uid, sver, lock, NULL);
   if (!pSW) return NULL;
 
   pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version);

source/dnode/vnode/src/meta/metaSnapshot.c

@@ -552,7 +552,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
 
 void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) {
   bool            ret = false;
-  SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1);
+  SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1, NULL);
   if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
     ret = true;
   }

source/dnode/vnode/src/tq/tqRead.c

@@ -263,7 +263,7 @@ bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; }
 
 void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
   bool            ret = false;
-  SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
+  SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1, NULL);
   if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
     ret = true;
   }
@@ -669,7 +669,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
       (pReader->cachedSchemaVer != sversion)) {
     tDeleteSchemaWrapper(pReader->pSchemaWrapper);
 
-    pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
+    pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, NULL);
     if (pReader->pSchemaWrapper == NULL) {
       tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64
              "version %d, possibly dropped table",
@@ -961,10 +961,8 @@ END:
   return code;
 }
 
-int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) {
+int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, int64_t *createTime) {
-  tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
+  tqTrace("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
-  SSDataBlock* block = NULL;
-
   SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
   if (pSubmitTbData == NULL) {
     return terrno;
@@ -980,7 +978,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
   pReader->lastBlkUid = uid;
 
   tDeleteSchemaWrapper(pReader->pSchemaWrapper);
-  pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
+  pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, createTime);
   if (pReader->pSchemaWrapper == NULL) {
     tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
            pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer);

source/dnode/vnode/src/tq/tqScan.c

@@ -210,13 +210,6 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
 
     if (pDataBlock != NULL && pDataBlock->info.rows > 0) {
       if (pRsp->withTbName) {
-        if (pOffset->type == TMQ_OFFSET__LOG) {
-          int64_t uid = pExec->pTqReader->lastBlkUid;
-          if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) {
-            tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId);
-            continue;
-          }
-        } else {
         char* tbName = taosStrdup(qExtractTbnameFromTask(task));
         if (tbName == NULL) {
           tqError("vgId:%d, failed to add tbname to rsp msg, null", pTq->pVnode->config.vgId);
@@ -227,21 +220,13 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
           continue;
         }
       }
-      }
       if (pRsp->withSchema) {
-        if (pOffset->type == TMQ_OFFSET__LOG) {
-          if (tqAddBlockSchemaToRsp(pExec, pRsp) != 0){
-            tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId);
-            continue;
-          }
-        } else {
         SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
         if(taosArrayPush(pRsp->blockSchema, &pSW) == NULL){
           tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId);
           continue;
         }
       }
-      }
 
       if (tqAddBlockDataToRsp(pDataBlock, pRsp, taosArrayGetSize(pDataBlock->pDataBlock),
                           pTq->pVnode->config.tsdbCfg.precision) != 0) {
@@ -249,12 +234,9 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
         continue;
       }
       pRsp->blockNum++;
-      if (pOffset->type == TMQ_OFFSET__LOG) {
-        continue;
-      } else {
       rowCnt += pDataBlock->info.rows;
       if (rowCnt <= tmqRowSize) continue;
-      }
+
     }
 
     // get meta
@@ -296,6 +278,54 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
   return code;
 }
 
+static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){
+  int32_t code = 0;
+  void*   createReq = NULL;
+  if (pRsp->createTableNum == 0) {
+    pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
+    if (pRsp->createTableLen == NULL) {
+      code = terrno;
+      goto END;
+    }
+    pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
+    if (pRsp->createTableReq == NULL) {
+      code = terrno;
+      goto END;
+    }
+  }
+
+  uint32_t len = 0;
+  tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code);
+  if (TSDB_CODE_SUCCESS != code) {
+    goto END;
+  }
+  createReq = taosMemoryCalloc(1, len);
+  if (createReq == NULL){
+    code = terrno;
+    goto END;
+  }
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, createReq, len);
+  code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq);
+  tEncoderClear(&encoder);
+  if (code < 0) {
+    goto END;
+  }
+  if (taosArrayPush(pRsp->createTableLen, &len) == NULL){
+    code = terrno;
+    goto END;
+  }
+  if (taosArrayPush(pRsp->createTableReq, &createReq) == NULL){
+    code = terrno;
+    goto END;
+  }
+  pRsp->createTableNum++;
+
+  return 0;
+END:
+  taosMemoryFree(createReq);
+  return code;
+}
 
 static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){
   int32_t code = 0;
@@ -315,7 +345,8 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int
   }
 
   SSubmitTbData* pSubmitTbDataRet = NULL;
-  code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet);
+  int64_t createTime = INT64_MAX;
+  code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet, &createTime);
   if (code != 0) {
     tqError("vgId:%d, failed to retrieve block", pTq->pVnode->config.vgId);
     goto END;
@@ -333,46 +364,13 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int
     }
   }
   if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) {
-    if (pRsp->createTableNum == 0) {
+    if (pSubmitTbDataRet->ctimeMs - createTime <= 1000) {  // judge if table is already created to avoid sending crateTbReq
-      pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
+      code = buildCreateTbInfo(pRsp, pSubmitTbDataRet->pCreateTbReq);
-      if (pRsp->createTableLen == NULL) {
+      if (code != 0){
-        code = terrno;
+        tqError("vgId:%d, failed to build create table info", pTq->pVnode->config.vgId);
-        goto END;
-      }
-      pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
-      if (pRsp->createTableReq == NULL) {
-        code = terrno;
         goto END;
       }
     }
-
-    uint32_t len = 0;
-    tEncodeSize(tEncodeSVCreateTbReq, pSubmitTbDataRet->pCreateTbReq, len, code);
-    if (TSDB_CODE_SUCCESS != code) {
-      goto END;
-    }
-    void*    createReq = taosMemoryCalloc(1, len);
-    if (createReq == NULL){
-      code = terrno;
-      goto END;
-    }
-    SEncoder encoder = {0};
-    tEncoderInit(&encoder, createReq, len);
-    code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq);
-    tEncoderClear(&encoder);
-    if (code < 0) {
-      taosMemoryFree(createReq);
-      goto END;
-    }
-    if (taosArrayPush(pRsp->createTableLen, &len) == NULL){
-      taosMemoryFree(createReq);
-      goto END;
-    }
-    if (taosArrayPush(pRsp->createTableReq, &createReq) == NULL){
-      taosMemoryFree(createReq);
-      goto END;
-    }
-    pRsp->createTableNum++;
   }
   if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL) {
     goto END;

source/dnode/vnode/src/tq/tqUtil.c

@@ -51,7 +51,8 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
   pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
   pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
 
-  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
+  if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL ||
+      pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
     if (pRsp->blockData != NULL) {
       taosArrayDestroy(pRsp->blockData);
       pRsp->blockData = NULL;
@@ -71,6 +72,7 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
       taosArrayDestroy(pRsp->blockSchema);
       pRsp->blockSchema = NULL;
     }
+
     return terrno;
   }
 

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -723,11 +723,9 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid,
   rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
   {
     SLastCol *pLastCol = NULL;
+    if (values_list[0] != NULL) {
       code = tsdbCacheDeserialize(values_list[0], values_list_sizes[0], &pLastCol);
-    if (code == TSDB_CODE_INVALID_PARA) {
+      if (code != TSDB_CODE_SUCCESS) {
-      tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
-                tstrerror(code));
-    } else if (code != TSDB_CODE_SUCCESS) {
         tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
                   tstrerror(code));
         goto _exit;
@@ -736,13 +734,12 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid,
         rocksdb_writebatch_delete(wb, keys_list[0], klen);
       }
       taosMemoryFreeClear(pLastCol);
+    }
 
     pLastCol = NULL;
+    if (values_list[1] != NULL) {
       code = tsdbCacheDeserialize(values_list[1], values_list_sizes[1], &pLastCol);
-    if (code == TSDB_CODE_INVALID_PARA) {
+      if (code != TSDB_CODE_SUCCESS) {
-      tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
-                tstrerror(code));
-    } else if (code != TSDB_CODE_SUCCESS) {
         tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
                   tstrerror(code));
         goto _exit;
@@ -751,6 +748,7 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid,
         rocksdb_writebatch_delete(wb, keys_list[1], klen);
       }
       taosMemoryFreeClear(pLastCol);
+    }
 
     rocksdb_free(values_list[0]);
     rocksdb_free(values_list[1]);
@@ -1218,15 +1216,14 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray
       SColVal        *pColVal = &updCtx->colVal;
 
       SLastCol *pLastCol = NULL;
+      if (values_list[i] != NULL) {
         code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol);
-      if (code == TSDB_CODE_INVALID_PARA) {
+        if (code != TSDB_CODE_SUCCESS) {
-        tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
-                  tstrerror(code));
-      } else if (code != TSDB_CODE_SUCCESS) {
           tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
                     tstrerror(code));
           goto _exit;
         }
+      }
       /*
       if (code) {
         tsdbError("tsdb/cache: vgId:%d, deserialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code));
@@ -1692,15 +1689,14 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA
       continue;
     }
 
+    if (values_list[i] != NULL) {
       code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol);
-    if (code == TSDB_CODE_INVALID_PARA) {
+      if (code != TSDB_CODE_SUCCESS) {
-      tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
-                tstrerror(code));
-    } else if (code != TSDB_CODE_SUCCESS) {
         tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
                   tstrerror(code));
         goto _exit;
       }
+    }
     SLastCol *pToFree = pLastCol;
     SIdxKey  *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[j];
     if (pLastCol && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) {
@@ -1959,15 +1955,14 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
   rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
   for (int i = 0; i < numKeys; ++i) {
     SLastCol *pLastCol = NULL;
+    if (values_list[i] != NULL) {
       code = tsdbCacheDeserialize(values_list[i], values_list_sizes[i], &pLastCol);
-    if (code == TSDB_CODE_INVALID_PARA) {
+      if (code != TSDB_CODE_SUCCESS) {
-      tsdbTrace("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
-                tstrerror(code));
-    } else if (code != TSDB_CODE_SUCCESS) {
         tsdbError("vgId:%d, %s deserialize failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__,
                   tstrerror(code));
         goto _exit;
       }
+    }
     SIdxKey  *idxKey = taosArrayGet(remainCols, i);
     SLastKey *pLastKey = &idxKey->key;
     if (NULL != pLastCol && (pLastCol->rowKey.ts <= eKey && pLastCol->rowKey.ts >= sKey)) {

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -702,7 +702,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
 }
 
 int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
-  SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0);
+  SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0, NULL);
   if (pSW) {
     *num = pSW->nCols;
     tDeleteSchemaWrapper(pSW);

source/libs/command/src/command.c

@@ -410,27 +410,27 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName,
   int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pCfg->daysToKeep2);
 
   if (IS_SYS_DBNAME(dbName)) {
-    len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName);
-  } else {
     len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
+                     "CREATE DATABASE `%s`", dbName);
+  } else {
+    len +=
+        tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
                   "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %s "
                   "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %s,%s,%s PAGES %d PAGESIZE %d "
                   "PRECISION '%s' REPLICA %d "
                   "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d "
                   "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64
-                   " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d",
+                  " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKPAGES %d S3_KEEPLOCAL %dm S3_COMPACT %d",
-                   dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression,
+                  dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, durationStr,
-                   durationStr,
+                  pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, keep0Str, keep1Str, keep2Str,
-                   pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger,
+                  pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
-                   keep0Str, keep1Str, keep2Str,
+                  1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod,
-                   pCfg->pages, pCfg->pageSize, prec,
+                  pCfg->walRetentionSize, pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm),
-                   pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, 1 == pCfg->numOfStables, hashPrefix,
+                  pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact);
-                   pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize,
-                   pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), pCfg->s3ChunkSize,
-                   pCfg->s3KeepLocal, pCfg->s3Compact);
 
     if (pRetentions) {
-      len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, " RETENTIONS %s", pRetentions);
+      len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
+                       " RETENTIONS %s", pRetentions);
     }
   }
 
@@ -528,12 +528,12 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
                            columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pCfg->pSchemaExt[i].compress)));
     }
     if (!(pSchema->flags & COL_IS_KEY)) {
-      *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s",
+      *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
-                       ((i > 0) ? ", " : ""), pSchema->name, type);
+                        "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
     } else {
       char* pk = "PRIMARY KEY";
-      *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s %s",
+      *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
-                       ((i > 0) ? ", " : ""), pSchema->name, type, pk);
+                        "%s`%s` %s %s", ((i > 0) ? ", " : ""), pSchema->name, type, pk);
     }
   }
 }
@@ -545,14 +545,15 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
     snprintf(type, sizeof(type), "%s", tDataTypes[pSchema->type].name);
     if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type ||
         TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
-      snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
+      snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)",
+               (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
     } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
       snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)",
                (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
     }
 
-    *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s",
+    *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
-                     ((i > 0) ? ", " : ""), pSchema->name, type);
+                      "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
   }
 }
 
@@ -624,7 +625,8 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
         code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, pTagVal->pData, pTagVal->nData, &tlen);
         TAOS_CHECK_ERRNO(code);
       } else {
-        code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen);
+        code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, &pTagVal->i64, tDataTypes[type].bytes,
+                               &tlen);
         TAOS_CHECK_ERRNO(code);
       }
       *len += tlen;

source/libs/executor/src/anomalywindowoperator.c

@@ -86,7 +86,7 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p
 
   pOperator->exprSupp.hasWindowOrGroup = true;
   pInfo->tsSlotId = ((SColumnNode*)pAnomalyNode->window.pTspk)->slotId;
-  strncpy(pInfo->anomalyOpt, pAnomalyNode->anomalyOpt, sizeof(pInfo->anomalyOpt));
+  tstrncpy(pInfo->anomalyOpt, pAnomalyNode->anomalyOpt, sizeof(pInfo->anomalyOpt));
 
   if (pAnomalyNode->window.pExprs != NULL) {
     int32_t    numOfScalarExpr = 0;

source/libs/executor/src/exchangeoperator.c

@@ -320,7 +320,7 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const
   if (!pInfo->pTaskId) {
     return terrno;
   }
-  strncpy(pInfo->pTaskId, id, len);
+  tstrncpy(pInfo->pTaskId, id, len);
   for (int32_t i = 0; i < numOfSources; ++i) {
     SSourceDataInfo dataInfo = {0};
     dataInfo.status = EX_SOURCE_DATA_NOT_READY;

source/libs/executor/src/executor.c

@@ -545,8 +545,9 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
   return code;
 }
 
-int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
+int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, int32_t dbNameBuffLen, char* tableName,
-                                    int32_t* tversion, int32_t idx, bool* tbGet) {
+                                    int32_t tbaleNameBuffLen, int32_t* sversion, int32_t* tversion, int32_t idx,
+                                    bool* tbGet) {
   *tbGet = false;
 
   if (tinfo == NULL || dbName == NULL || tableName == NULL) {
@@ -567,12 +568,12 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
   *sversion = pSchemaInfo->sw->version;
   *tversion = pSchemaInfo->tversion;
   if (pSchemaInfo->dbname) {
-    strcpy(dbName, pSchemaInfo->dbname);
+    tstrncpy(dbName, pSchemaInfo->dbname, dbNameBuffLen);
   } else {
     dbName[0] = 0;
   }
   if (pSchemaInfo->tablename) {
-    strcpy(tableName, pSchemaInfo->tablename);
+    tstrncpy(tableName, pSchemaInfo->tablename, tbaleNameBuffLen);
   } else {
     tableName[0] = 0;
   }
@@ -1494,6 +1495,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
 
       cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
       tstrncpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName, TSDB_TABLE_NAME_LEN);
+//      pTaskInfo->streamInfo.suid = mtInfo.suid == 0 ? mtInfo.uid : mtInfo.suid;
       tDeleteSchemaWrapper(pTaskInfo->streamInfo.schema);
       pTaskInfo->streamInfo.schema = mtInfo.schema;
 

source/libs/executor/src/scanoperator.c

@@ -6345,7 +6345,7 @@ int32_t fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, ch
     QUERY_CHECK_NULL(colInfoData, code, lino, _end, terrno);
     if (strlen(stbName) != 0) {
       char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
-      strncpy(varDataVal(varStbName), stbName, TSDB_TABLE_NAME_LEN);
+      tstrncpy(varDataVal(varStbName), stbName, TSDB_TABLE_NAME_LEN);
       varDataSetLen(varStbName, strlen(stbName));
       code = colDataSetVal(colInfoData, 0, varStbName, false);
       QUERY_CHECK_CODE(code, lino, _end);

source/libs/executor/src/sysscanoperator.c

@@ -425,7 +425,7 @@ static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) {
       SValueNode* pValue = (SValueNode*)node->pRight;
       if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR) {
         char* value = nodesGetValueFromNode(pValue);
-        strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN);
+        tstrncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN);
         return true;
       }
     }
@@ -914,41 +914,41 @@ _end:
   }
 }
 
-int32_t convertTagDataToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) {
+int32_t convertTagDataToStr(char* str, int32_t strBuffLen, int type, void* buf, int32_t bufSize, int32_t* len) {
   int32_t n = 0;
 
   switch (type) {
     case TSDB_DATA_TYPE_NULL:
-      n = sprintf(str, "null");
+      n = tsnprintf(str, strBuffLen, "null");
       break;
 
     case TSDB_DATA_TYPE_BOOL:
-      n = sprintf(str, (*(int8_t*)buf) ? "true" : "false");
+      n = tsnprintf(str, strBuffLen, (*(int8_t*)buf) ? "true" : "false");
       break;
 
     case TSDB_DATA_TYPE_TINYINT:
-      n = sprintf(str, "%d", *(int8_t*)buf);
+      n = tsnprintf(str, strBuffLen, "%d", *(int8_t*)buf);
       break;
 
     case TSDB_DATA_TYPE_SMALLINT:
-      n = sprintf(str, "%d", *(int16_t*)buf);
+      n = tsnprintf(str, strBuffLen, "%d", *(int16_t*)buf);
       break;
 
     case TSDB_DATA_TYPE_INT:
-      n = sprintf(str, "%d", *(int32_t*)buf);
+      n = tsnprintf(str, strBuffLen, "%d", *(int32_t*)buf);
       break;
 
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_TIMESTAMP:
-      n = sprintf(str, "%" PRId64, *(int64_t*)buf);
+      n = tsnprintf(str, strBuffLen, "%" PRId64, *(int64_t*)buf);
       break;
 
     case TSDB_DATA_TYPE_FLOAT:
-      n = sprintf(str, "%.5f", GET_FLOAT_VAL(buf));
+      n = tsnprintf(str, strBuffLen, "%.5f", GET_FLOAT_VAL(buf));
       break;
 
     case TSDB_DATA_TYPE_DOUBLE:
-      n = sprintf(str, "%.9f", GET_DOUBLE_VAL(buf));
+      n = tsnprintf(str, strBuffLen, "%.9f", GET_DOUBLE_VAL(buf));
       break;
 
     case TSDB_DATA_TYPE_BINARY:
@@ -973,19 +973,19 @@ int32_t convertTagDataToStr(char* str, int type, void* buf, int32_t bufSize, int
       n = length;
       break;
     case TSDB_DATA_TYPE_UTINYINT:
-      n = sprintf(str, "%u", *(uint8_t*)buf);
+      n = tsnprintf(str, strBuffLen, "%u", *(uint8_t*)buf);
       break;
 
     case TSDB_DATA_TYPE_USMALLINT:
-      n = sprintf(str, "%u", *(uint16_t*)buf);
+      n = tsnprintf(str, strBuffLen, "%u", *(uint16_t*)buf);
       break;
 
     case TSDB_DATA_TYPE_UINT:
-      n = sprintf(str, "%u", *(uint32_t*)buf);
+      n = tsnprintf(str, strBuffLen, "%u", *(uint32_t*)buf);
       break;
 
     case TSDB_DATA_TYPE_UBIGINT:
-      n = sprintf(str, "%" PRIu64, *(uint64_t*)buf);
+      n = tsnprintf(str, strBuffLen, "%" PRIu64, *(uint64_t*)buf);
       break;
 
     default:
@@ -1065,14 +1065,21 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo,
     int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type;
     pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4);
     QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno);
+    int32_t tagStrBufflen = 32;
     char tagTypeStr[VARSTR_HEADER_SIZE + 32];
-    int  tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name);
+    int  tagTypeLen = tsnprintf(varDataVal(tagTypeStr), tagStrBufflen, "%s", tDataTypes[tagType].name);
+    tagStrBufflen -= tagTypeLen;
+    if (tagStrBufflen <= 0) {
+      code = TSDB_CODE_INVALID_PARA;
+      QUERY_CHECK_CODE(code, lino, _end);
+    }
+
     if (tagType == TSDB_DATA_TYPE_NCHAR) {
-      tagTypeLen += sprintf(
+      tagTypeLen += tsnprintf(
-          varDataVal(tagTypeStr) + tagTypeLen, "(%d)",
+          varDataVal(tagTypeStr) + tagTypeLen, tagStrBufflen, "(%d)",
           (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
     } else if (IS_VAR_DATA_TYPE(tagType)) {
-      tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)",
+      tagTypeLen += tsnprintf(varDataVal(tagTypeStr) + tagTypeLen,  tagStrBufflen, "(%d)",
                             (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE));
     }
     varDataSetLen(tagTypeStr, tagTypeLen);
@@ -1127,7 +1134,7 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo,
         QUERY_CHECK_NULL(tagVarChar, code, lino, _end, terrno);
         int32_t len = -1;
         if (tagLen > 0)
-          convertTagDataToStr(varDataVal(tagVarChar), tagType, tagData, tagLen, &len);
+          convertTagDataToStr(varDataVal(tagVarChar), bufSize + 1 - VARSTR_HEADER_SIZE, tagType, tagData, tagLen, &len);
         else
           len = 0;
         varDataSetLen(tagVarChar, len);
@@ -1197,13 +1204,19 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo,
     int8_t colType = schemaRow->pSchema[i].type;
     pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4);
     QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno);
+    int32_t colStrBufflen = 32;
     char colTypeStr[VARSTR_HEADER_SIZE + 32];
-    int  colTypeLen = sprintf(varDataVal(colTypeStr), "%s", tDataTypes[colType].name);
+    int  colTypeLen = tsnprintf(varDataVal(colTypeStr), colStrBufflen, "%s", tDataTypes[colType].name);
+    colStrBufflen -= colTypeLen;
+    if (colStrBufflen <= 0) {
+      code = TSDB_CODE_INVALID_PARA;
+      QUERY_CHECK_CODE(code, lino, _end);
+    }
     if (colType == TSDB_DATA_TYPE_VARCHAR) {
-      colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)",
+      colTypeLen += tsnprintf(varDataVal(colTypeStr) + colTypeLen, colStrBufflen, "(%d)",
                             (int32_t)(schemaRow->pSchema[i].bytes - VARSTR_HEADER_SIZE));
     } else if (colType == TSDB_DATA_TYPE_NCHAR) {
-      colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)",
+      colTypeLen += tsnprintf(varDataVal(colTypeStr) + colTypeLen, colStrBufflen, "(%d)",
                             (int32_t)((schemaRow->pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
     }
     varDataSetLen(colTypeStr, colTypeLen);
@@ -2019,8 +2032,7 @@ static EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
 
       SValueNode* node = (SValueNode*)pNode;
       char*       dbName = nodesGetValueFromNode(node);
-      strncpy(pContext, varDataVal(dbName), varDataLen(dbName));
+      tstrncpy((char*)pContext, varDataVal(dbName), TSDB_DB_NAME_LEN);
-      *((char*)pContext + varDataLen(dbName)) = 0;
       return DEAL_RES_END;  // stop walk
     }
     default:
@@ -2056,11 +2068,11 @@ static int32_t doSysTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes)
       getDBNameFromCondition(pInfo->pCondition, dbName);
       if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, TSDB_TABLE_FNAME_LEN) != 0 &&
           strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) {
-        sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
+        TAOS_UNUSED(tsnprintf(pInfo->req.db, sizeof(pInfo->req.db), "%d.%s", pInfo->accountId, dbName));
       }
     } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) {
       getDBNameFromCondition(pInfo->pCondition, dbName);
-      if (dbName[0]) sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
+      if (dbName[0]) TAOS_UNUSED(tsnprintf(pInfo->req.db, sizeof(pInfo->req.db), "%d.%s", pInfo->accountId, dbName));
       (void)sysTableIsCondOnOneTable(pInfo->pCondition, pInfo->req.filterTb);
     }
 

source/libs/executor/test/executorTests.cpp

@@ -115,7 +115,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
     int32_t code = colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(&v), false);
     ASSERT(code == 0);
 
-    //    sprintf(buf, "this is %d row", i);
+    //    tsnprintf(buf, "this is %d row", i);
     //    STR_TO_VARSTR(b1, buf);
     //
     //    SColumnInfoData* pColInfo2 = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 1));
@@ -179,7 +179,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
     code = colDataSetVal(pColInfo1, i, reinterpret_cast<const char*>(&v), false);
     ASSERT(code == 0);
 
-    //    sprintf(buf, "this is %d row", i);
+    //    tsnprintf(buf, "this is %d row", i);
     //    STR_TO_VARSTR(b1, buf);
     //
     //    SColumnInfoData* pColInfo2 = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 1));

source/libs/executor/test/lhashTests.cpp

@@ -26,7 +26,7 @@
 
 TEST(testCase, linear_hash_Tests) {
   taosSeedRand(taosGetTimestampSec());
-  strcpy(tsTempDir, "/tmp/");
+  tstrncpy((char*)tsTempDir, "/tmp/", sizeof(tsTempDir));
 
   _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT);
 

source/libs/monitor/src/monFramework.c

@@ -183,7 +183,7 @@ void monGenClusterInfoTable(SMonInfo *pMonitor){
     }
 
     if (taosHashRemove(tsMonitor.metrics, metric_names[i], strlen(metric_names[i])) != 0) {
-      uError("failed to remove metric %s", metric_names[i]);
+      uTrace("failed to remove metric %s", metric_names[i]);
     }
   } 
 
@@ -652,7 +652,7 @@ void monGenMnodeRoleTable(SMonInfo *pMonitor){
     }
 
     if (taosHashRemove(tsMonitor.metrics, mnodes_role_gauges[i], strlen(mnodes_role_gauges[i])) != 0) {
-      uError("failed to remove metric %s", mnodes_role_gauges[i]);
+      uTrace("failed to remove metric %s", mnodes_role_gauges[i]);
     }
   }
 
@@ -725,7 +725,7 @@ void monGenVnodeRoleTable(SMonInfo *pMonitor){
     }
 
     if (taosHashRemove(tsMonitor.metrics, vnodes_role_gauges[i], strlen(vnodes_role_gauges[i])) != 0) {
-      uError("failed to remove metric %s", vnodes_role_gauges[i]);
+      uTrace("failed to remove metric %s", vnodes_role_gauges[i]);
     }
   }
 

source/libs/monitor/test/monTest.cpp

@@ -26,7 +26,10 @@ class MonitorTest : public ::testing::Test {
     monInit(&cfg);
   }
 
-  static void TearDownTestSuite() { monCleanup(); }
+  static void TearDownTestSuite() {
+    monCleanup();
+    taosMsleep(100);
+  }
 
  public:
   void SetUp() override {}

source/libs/parser/inc/parAst.h

@@ -64,7 +64,7 @@ typedef enum EDatabaseOptionType {
   DB_OPTION_STT_TRIGGER,
   DB_OPTION_TABLE_PREFIX,
   DB_OPTION_TABLE_SUFFIX,
-  DB_OPTION_S3_CHUNKSIZE,
+  DB_OPTION_S3_CHUNKPAGES,
   DB_OPTION_S3_KEEPLOCAL,
   DB_OPTION_S3_COMPACT,
   DB_OPTION_KEEP_TIME_OFFSET,

source/libs/parser/inc/sql.y

@@ -280,7 +280,7 @@ db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C).
 db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C).                        { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
 db_options(A) ::= db_options(B) TABLE_PREFIX signed(C).                           { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
 db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C).                           { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
-db_options(A) ::= db_options(B) S3_CHUNKSIZE NK_INTEGER(C).                       { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_CHUNKSIZE, &C); }
+db_options(A) ::= db_options(B) S3_CHUNKPAGES NK_INTEGER(C).                      { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_CHUNKPAGES, &C); }
 db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_INTEGER(C).                       { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
 db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C).                      { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
 db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C).                         { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); }

source/libs/parser/src/parAstCreater.c

@@ -1959,7 +1959,7 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
       nodesDestroyNode((SNode*)pNode);
       break;
     }
-    case DB_OPTION_S3_CHUNKSIZE:
+    case DB_OPTION_S3_CHUNKPAGES:
       pDbOptions->s3ChunkSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_S3_KEEPLOCAL: {
@@ -1974,14 +1974,13 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
     case DB_OPTION_S3_COMPACT:
       pDbOptions->s3Compact = taosStr2Int8(((SToken*)pVal)->z, NULL, 10);
       break;
-    case DB_OPTION_KEEP_TIME_OFFSET: {
+    case DB_OPTION_KEEP_TIME_OFFSET: 
       pDbOptions->keepTimeOffset = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
     case DB_OPTION_ENCRYPT_ALGORITHM:
       COPY_STRING_FORM_STR_TOKEN(pDbOptions->encryptAlgorithmStr, (SToken*)pVal);
       pDbOptions->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
       break;
-    }
     default:
       break;
   }

source/libs/parser/src/parInsertUtil.c

@@ -886,17 +886,32 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) {
   return false;
 }
 
-int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* tFields,
+int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* tFields,
-                     int numFields, bool needChangeLength, char* errstr, int32_t errstrLen) {
+                     int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw) {
+  int ret = 0;
   if(data == NULL) {
     uError("rawBlockBindData, data is NULL");
     return TSDB_CODE_APP_ERROR;
   }
   void* tmp =
       taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid));
+  SVCreateTbReq *pCreateReqTmp = NULL;
+  if (tmp == NULL && pCreateTb != NULL){
+    ret = cloneSVreateTbReq(pCreateTb, &pCreateReqTmp);
+    if (ret != TSDB_CODE_SUCCESS){
+      uError("cloneSVreateTbReq error");
+      goto end;
+    }
+  }
+
   STableDataCxt* pTableCxt = NULL;
-  int            ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
+  ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
-                                          sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false);
+                                          sizeof(pTableMeta->uid), pTableMeta, &pCreateReqTmp, &pTableCxt, true, false);
+  if (pCreateReqTmp != NULL) {
+    tdDestroySVCreateTbReq(pCreateReqTmp);
+    taosMemoryFree(pCreateReqTmp);
+  }
+
   if (ret != TSDB_CODE_SUCCESS) {
     uError("insGetTableDataCxt error");
     goto end;
@@ -948,12 +963,17 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
     ret = TSDB_CODE_INVALID_PARA;
     goto end;
   }
-  if (tFields != NULL && numFields > boundInfo->numOfBound) {
+//  if (tFields != NULL && numFields > boundInfo->numOfBound) {
-    if (errstr != NULL)
+//    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
-      snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
+//    ret = TSDB_CODE_INVALID_PARA;
+//    goto end;
+//  }
+  if (tFields == NULL && numOfCols != boundInfo->numOfBound) {
+    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d not equal to num of bound cols:%d", numOfCols, boundInfo->numOfBound);
     ret = TSDB_CODE_INVALID_PARA;
     goto end;
   }
+
   if (tFields == NULL) {
     for (int j = 0; j < boundInfo->numOfBound; j++) {
       SSchema*  pColSchema = &pSchema[j];
@@ -991,7 +1011,13 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
     for (int i = 0; i < numFields; i++) {
       for (int j = 0; j < boundInfo->numOfBound; j++) {
         SSchema* pColSchema = &pSchema[j];
-        if (strcmp(pColSchema->name, tFields[i].name) == 0) {
+        char* fieldName = NULL;
+        if (raw) {
+          fieldName = ((SSchemaWrapper*)tFields)->pSchema[i].name;
+        } else {
+          fieldName = ((TAOS_FIELD*)tFields)[i].name;
+        }
+        if (strcmp(pColSchema->name, fieldName) == 0) {
           if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
             if (errstr != NULL)
               snprintf(errstr, errstrLen,
@@ -1011,6 +1037,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
             pStart += numOfRows * sizeof(int32_t);
           } else {
             pStart += BitmapLen(numOfRows);
+//            for(int k = 0; k < numOfRows; k++) {
+//              if(!colDataIsNull_f(offset, k) && pColSchema->type == TSDB_DATA_TYPE_INT){
+//                printf("colName:%s,val:%d", fieldName, *(int32_t*)(pStart + k * sizeof(int32_t)));
+//              }
+//            }
           }
           char* pData = pStart;
 

source/libs/parser/src/parTokenizer.c

@@ -340,7 +340,7 @@ static SKeyword keywordTable[] = {
     {"_FROWTS",              TK_FROWTS},
     {"ALIVE",                TK_ALIVE},
     {"VARBINARY",            TK_VARBINARY},
-    {"S3_CHUNKSIZE",         TK_S3_CHUNKSIZE},
+    {"S3_CHUNKPAGES",        TK_S3_CHUNKPAGES},
     {"S3_KEEPLOCAL",         TK_S3_KEEPLOCAL},
     {"S3_COMPACT",           TK_S3_COMPACT},
     {"S3MIGRATE",            TK_S3MIGRATE},
@@ -834,9 +834,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC
 
 bool taosIsKeyWordToken(const char* z, int32_t len) { return (tKeywordCode((char*)z, len) != TK_NK_ID); }
 
-int32_t taosInitKeywordsTable() {
+int32_t taosInitKeywordsTable() { return doInitKeywordsTable(); }
-  return doInitKeywordsTable();
-}
 
 void taosCleanupKeywordsTable() {
   void* m = keywordHashTable;

source/libs/parser/src/parTranslater.c

@@ -1019,8 +1019,7 @@ static uint8_t getPrecisionFromCurrStmt(SNode* pCurrStmt, uint8_t defaultVal) {
   if (isDeleteStmt(pCurrStmt)) {
     return ((SDeleteStmt*)pCurrStmt)->precision;
   }
-  if (pCurrStmt && nodeType(pCurrStmt) == QUERY_NODE_CREATE_TSMA_STMT)
+  if (pCurrStmt && nodeType(pCurrStmt) == QUERY_NODE_CREATE_TSMA_STMT) return ((SCreateTSMAStmt*)pCurrStmt)->precision;
-    return ((SCreateTSMAStmt*)pCurrStmt)->precision;
   return defaultVal;
 }
 
@@ -1194,8 +1193,7 @@ int32_t buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum,
       break;
     }
   }
-  if(TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) *ppOut = pPartitionList;
-    *ppOut = pPartitionList;
 
   return code;
 }
@@ -1229,7 +1227,8 @@ static int32_t isTimeLineAlignedQuery(SNode* pStmt, bool* pRes) {
       }
     }
   }
-  if (TSDB_CODE_SUCCESS == code && QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
+  if (TSDB_CODE_SUCCESS == code &&
+      QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
     SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
     if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
       SNodeList* pPartitionList = NULL;
@@ -1397,12 +1396,16 @@ static void setColumnPrimTs(STranslateContext* pCxt, SColumnNode* pCol, const ST
   }
 }
 
-static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList, bool skipProjRef) {
+static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, bool igTags, SNodeList* pList,
+                                    bool skipProjRef) {
   int32_t code = 0;
   if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
     const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta;
     int32_t           nums = pMeta->tableInfo.numOfColumns +
-                   (igTags ? 0 : ((TSDB_SUPER_TABLE == pMeta->tableType || ((SRealTableNode*)pTable)->stbRewrite) ? pMeta->tableInfo.numOfTags : 0));
+                   (igTags ? 0
+                           : ((TSDB_SUPER_TABLE == pMeta->tableType || ((SRealTableNode*)pTable)->stbRewrite)
+                                  ? pMeta->tableInfo.numOfTags
+                                  : 0));
     for (int32_t i = 0; i < nums; ++i) {
       if (invisibleColumn(pCxt->pParseCxt->enableSysInfo, pMeta->tableType, pMeta->schema[i].flags)) {
         pCxt->pParseCxt->hasInvisibleCol = true;
@@ -1433,7 +1436,8 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p
         code = setColumnInfoByExpr(pTempTable, (SExprNode*)pNode, (SColumnNode**)&pCell->pNode);
       }
       if (TSDB_CODE_SUCCESS == code) {
-        if (!skipProjRef) pCol->projRefIdx = ((SExprNode*)pNode)->projIdx; // only set proj ref when select * from (select ...)
+        if (!skipProjRef)
+          pCol->projRefIdx = ((SExprNode*)pNode)->projIdx;  // only set proj ref when select * from (select ...)
       } else {
         break;
       }
@@ -1603,25 +1607,6 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
     }
   }
   if (*pFound) {
-    if (QUERY_NODE_FUNCTION == nodeType(pFoundNode) && (SQL_CLAUSE_GROUP_BY == pCxt->currClause || SQL_CLAUSE_PARTITION_BY == pCxt->currClause)) {
-      pCxt->errCode = getFuncInfo(pCxt, (SFunctionNode*)pFoundNode);
-      if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-        if (fmIsVectorFunc(((SFunctionNode*)pFoundNode)->funcId)) {
-          pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, (*pCol)->colName);
-          return DEAL_RES_ERROR;
-        } else if (fmIsPseudoColumnFunc(((SFunctionNode*)pFoundNode)->funcId)) {
-          if ('\0' != (*pCol)->tableAlias[0]) {
-            return translateColumnWithPrefix(pCxt, pCol);
-          } else {
-            return translateColumnWithoutPrefix(pCxt, pCol);
-          }
-        } else {
-          /* Do nothing and replace old node with found node. */
-        }
-      } else {
-        return DEAL_RES_ERROR;
-      }
-    }
     SNode*  pNew = NULL;
     int32_t code = nodesCloneNode(pFoundNode, &pNew);
     if (NULL == pNew) {
@@ -1630,14 +1615,6 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p
     }
     nodesDestroyNode(*(SNode**)pCol);
     *(SNode**)pCol = (SNode*)pNew;
-    if (QUERY_NODE_COLUMN == nodeType(pFoundNode)) {
-      pCxt->errCode = TSDB_CODE_SUCCESS;
-      if ('\0' != (*pCol)->tableAlias[0]) {
-        return translateColumnWithPrefix(pCxt, pCol);
-      } else {
-        return translateColumnWithoutPrefix(pCxt, pCol);
-      }
-    }
   }
   return DEAL_RES_CONTINUE;
 }
@@ -1744,8 +1721,7 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt
               ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) {
             SNode* pTbnameNode = NULL;
             code = biMakeTbnameProjectAstNode(pFunc->functionName, NULL, &pTbnameNode);
-            if (TSDB_CODE_SUCCESS == code)
+            if (TSDB_CODE_SUCCESS == code) code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode);
-              code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode);
           }
         }
         if (TSDB_CODE_SUCCESS == code && LIST_LENGTH(pTbnameNodeList) > 0) {
@@ -1761,8 +1737,7 @@ static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt
             ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) {
           SNode* pTbnameNode = NULL;
           code = biMakeTbnameProjectAstNode(pFunc->functionName, pTableAlias, &pTbnameNode);
-          if (TSDB_CODE_SUCCESS == code)
+          if (TSDB_CODE_SUCCESS == code) code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode);
-            code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode);
         }
         if (TSDB_CODE_SUCCESS == code && LIST_LENGTH(pTbnameNodeList) > 0) {
           nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList);
@@ -1794,8 +1769,7 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
             ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) {
           SNode* pTbnameNode = NULL;
           code = biMakeTbnameProjectAstNode(NULL, NULL, &pTbnameNode);
-          if (TSDB_CODE_SUCCESS == code)
+          if (TSDB_CODE_SUCCESS == code) code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode);
-            code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode);
         }
       }
       if (TSDB_CODE_SUCCESS == code && LIST_LENGTH(pTbnameNodeList) > 0) {
@@ -1810,8 +1784,7 @@ int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
           ((SRealTableNode*)pTable)->pMeta != NULL && ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) {
         SNode* pTbnameNode = NULL;
         code = biMakeTbnameProjectAstNode(NULL, pTableAlias, &pTbnameNode);
-        if (TSDB_CODE_SUCCESS ==code)
+        if (TSDB_CODE_SUCCESS == code) code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode);
-          code = nodesListStrictAppend(pTbnameNodeList, pTbnameNode);
       }
       if (TSDB_CODE_SUCCESS == code && LIST_LENGTH(pTbnameNodeList) > 0) {
         nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList);
@@ -1877,9 +1850,40 @@ int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* p
 }
 
 static bool clauseSupportAlias(ESqlClause clause) {
-  return SQL_CLAUSE_GROUP_BY == clause ||
+  return SQL_CLAUSE_GROUP_BY == clause || SQL_CLAUSE_PARTITION_BY == clause || SQL_CLAUSE_ORDER_BY == clause;
-         SQL_CLAUSE_PARTITION_BY == clause ||
+}
-         SQL_CLAUSE_ORDER_BY == clause;
+
+static EDealRes translateColumnInGroupByClause(STranslateContext* pCxt, SColumnNode** pCol, bool *translateAsAlias) {
+  *translateAsAlias = false;
+  // count(*)/first(*)/last(*) and so on
+  if (0 == strcmp((*pCol)->colName, "*")) {
+    return DEAL_RES_CONTINUE;
+  }
+
+  if (pCxt->pParseCxt->biMode) {
+    SNode** ppNode = (SNode**)pCol;
+    bool ret;
+    pCxt->errCode = biRewriteToTbnameFunc(pCxt, ppNode, &ret);
+    if (TSDB_CODE_SUCCESS != pCxt->errCode) return DEAL_RES_ERROR;
+    if (ret) {
+      return translateFunction(pCxt, (SFunctionNode**)ppNode);
+    }
+  }
+
+  EDealRes res = DEAL_RES_CONTINUE;
+  if ('\0' != (*pCol)->tableAlias[0]) {
+    res = translateColumnWithPrefix(pCxt, pCol);
+  } else {
+    bool found = false;
+    res = translateColumnWithoutPrefix(pCxt, pCol);
+    if (!(*pCol)->node.asParam &&
+        res != DEAL_RES_CONTINUE &&
+        res != DEAL_RES_END && pCxt->errCode != TSDB_CODE_PAR_AMBIGUOUS_COLUMN) {
+      res = translateColumnUseAlias(pCxt, pCol, &found);
+      *translateAsAlias = true;
+    }
+  }
+  return res;
 }
 
 static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
@@ -1908,8 +1912,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
     res = translateColumnWithPrefix(pCxt, pCol);
   } else {
     bool found = false;
-    if ((pCxt->currClause == SQL_CLAUSE_ORDER_BY) &&
+    if ((pCxt->currClause == SQL_CLAUSE_ORDER_BY) && !(*pCol)->node.asParam) {
-        !(*pCol)->node.asParam) {
       res = translateColumnUseAlias(pCxt, pCol, &found);
     }
     if (DEAL_RES_ERROR != res && !found) {
@@ -1919,9 +1922,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) {
         res = translateColumnWithoutPrefix(pCxt, pCol);
       }
     }
-    if (clauseSupportAlias(pCxt->currClause) &&
+    if (clauseSupportAlias(pCxt->currClause) && !(*pCol)->node.asParam && res != DEAL_RES_CONTINUE &&
-        !(*pCol)->node.asParam &&
-        res != DEAL_RES_CONTINUE &&
         res != DEAL_RES_END) {
       res = translateColumnUseAlias(pCxt, pCol, &found);
     }
@@ -2750,7 +2751,8 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p
   }
 
   if (NULL != pSelect->pWindow) {
-    if (QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow) ||  QUERY_NODE_COUNT_WINDOW == nodeType(pSelect->pWindow)) {
+    if (QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow) ||
+        QUERY_NODE_COUNT_WINDOW == nodeType(pSelect->pWindow)) {
       return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
                                      "%s function is not supported in count/event window", pFunc->functionName);
     }
@@ -3222,8 +3224,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode** pFunc
 
   pCxt->errCode = getFuncInfo(pCxt, *pFunc);
   if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    if ((SQL_CLAUSE_GROUP_BY == pCxt->currClause ||
+    if ((SQL_CLAUSE_GROUP_BY == pCxt->currClause || SQL_CLAUSE_PARTITION_BY == pCxt->currClause) &&
-         SQL_CLAUSE_PARTITION_BY == pCxt->currClause) &&
         fmIsVectorFunc((*pFunc)->funcId)) {
       pCxt->errCode = TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION;
     }
@@ -4278,8 +4279,8 @@ static int32_t setTableTsmas(STranslateContext* pCxt, SName* pName, SRealTableNo
         SVgroupInfo     vgInfo = {0};
         bool            exists = false;
         toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, "", &tsmaTargetTbName);
-        int32_t len = tsnprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName, pTsma->name,
+        int32_t len = tsnprintf(buf, TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN, "%s.%s_%s", pTsma->dbFName,
-                               pRealTable->table.tableName);
+                                pTsma->name, pRealTable->table.tableName);
         len = taosCreateMD5Hash(buf, len);
         strncpy(tsmaTargetTbName.tname, buf, MD5_OUTPUT_LEN);
         code = collectUseTable(&tsmaTargetTbName, pCxt->pTargetTables);
@@ -4957,8 +4958,7 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinPare
         }
         code = translateAudit(pCxt, pRealTable, &name);
 #endif
-        if (TSDB_CODE_SUCCESS == code)
+        if (TSDB_CODE_SUCCESS == code) code = setTableVgroupList(pCxt, &name, pRealTable);
-          code = setTableVgroupList(pCxt, &name, pRealTable);
         if (TSDB_CODE_SUCCESS == code) {
           code = setTableIndex(pCxt, &name, pRealTable);
         }
@@ -5350,8 +5350,8 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
 }
 
 static EDealRes needFillImpl(SNode* pNode, void* pContext) {
-  if ((isAggFunc(pNode) || isInterpFunc(pNode)) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType
+  if ((isAggFunc(pNode) || isInterpFunc(pNode)) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType &&
-  && FUNCTION_TYPE_GROUP_CONST_VALUE != ((SFunctionNode*)pNode)->funcType) {
+      FUNCTION_TYPE_GROUP_CONST_VALUE != ((SFunctionNode*)pNode)->funcType) {
     *(bool*)pContext = true;
     return DEAL_RES_END;
   }
@@ -5460,9 +5460,7 @@ static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSe
   } else {
     SNode*  pNode;
     int32_t projIdx = 1;
-    FOREACH(pNode, pSelect->pProjectionList) {
+    FOREACH(pNode, pSelect->pProjectionList) { ((SExprNode*)pNode)->projIdx = projIdx++; }
-      ((SExprNode*)pNode)->projIdx = projIdx++;
-    }
     return TSDB_CODE_SUCCESS;
   }
 }
@@ -5472,12 +5470,13 @@ typedef struct SReplaceGroupByAliasCxt {
   SNodeList*         pProjectionList;
 } SReplaceGroupByAliasCxt;
 
-static EDealRes replaceGroupByAliasImpl(SNode** pNode, void* pContext) {
+static EDealRes translateGroupPartitionByImpl(SNode** pNode, void* pContext) {
   SReplaceGroupByAliasCxt* pCxt = pContext;
   SNodeList*               pProjectionList = pCxt->pProjectionList;
   SNode*                   pProject = NULL;
-  if (QUERY_NODE_VALUE == nodeType(*pNode)) {
+  int32_t                  code = TSDB_CODE_SUCCESS;
   STranslateContext*       pTransCxt = pCxt->pTranslateCxt;
+  if (QUERY_NODE_VALUE == nodeType(*pNode)) {
     SValueNode* pVal = (SValueNode*) *pNode;
     if (DEAL_RES_ERROR == translateValue(pTransCxt, pVal)) {
       return DEAL_RES_CONTINUE;
@@ -5488,43 +5487,59 @@ static EDealRes replaceGroupByAliasImpl(SNode** pNode, void* pContext) {
     int32_t pos = getPositionValue(pVal);
     if (0 < pos && pos <= LIST_LENGTH(pProjectionList)) {
       SNode* pNew = NULL;
-      int32_t code = nodesCloneNode(nodesListGetNode(pProjectionList, pos - 1), (SNode**)&pNew);
+      code = nodesCloneNode(nodesListGetNode(pProjectionList, pos - 1), (SNode**)&pNew);
       if (TSDB_CODE_SUCCESS != code) {
         pCxt->pTranslateCxt->errCode = code;
         return DEAL_RES_ERROR;
       }
       nodesDestroyNode(*pNode);
       *pNode = pNew;
-      return DEAL_RES_CONTINUE;
-    } else {
-      return DEAL_RES_CONTINUE;
     }
+    code = translateExpr(pTransCxt, pNode);
+    if (TSDB_CODE_SUCCESS != code) {
+      pTransCxt->errCode = code;
+      return DEAL_RES_ERROR;
+    }
+    return DEAL_RES_CONTINUE;
   } else if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
-    STranslateContext* pTransCxt = pCxt->pTranslateCxt;
+    bool     asAlias = false;
-    return translateColumn(pTransCxt, (SColumnNode**)pNode);
+    EDealRes res = translateColumnInGroupByClause(pTransCxt, (SColumnNode**)pNode, &asAlias);
+    if (DEAL_RES_ERROR == res) {
+      return DEAL_RES_ERROR;
     }
-
+    pTransCxt->errCode = TSDB_CODE_SUCCESS;
+    if (nodeType(*pNode) == QUERY_NODE_COLUMN && !asAlias) {
       return DEAL_RES_CONTINUE;
     }
+    code = translateExpr(pTransCxt, pNode);
+    if (TSDB_CODE_SUCCESS != code) {
+      pTransCxt->errCode = code;
+      return DEAL_RES_ERROR;
+    }
+    return DEAL_RES_CONTINUE;
+  }
+  return doTranslateExpr(pNode, pTransCxt);
+}
 
-static int32_t replaceGroupByAlias(STranslateContext* pCxt, SSelectStmt* pSelect) {
+static int32_t translateGroupByList(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (NULL == pSelect->pGroupByList) {
     return TSDB_CODE_SUCCESS;
   }
   SReplaceGroupByAliasCxt cxt = {
       .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList};
-  nodesRewriteExprsPostOrder(pSelect->pGroupByList, replaceGroupByAliasImpl, &cxt);
+  nodesRewriteExprsPostOrder(pSelect->pGroupByList, translateGroupPartitionByImpl, &cxt);
 
   return pCxt->errCode;
 }
 
-static int32_t replacePartitionByAlias(STranslateContext* pCxt, SSelectStmt* pSelect) {
+static int32_t translatePartitionByList(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (NULL == pSelect->pPartitionByList) {
     return TSDB_CODE_SUCCESS;
   }
+
   SReplaceGroupByAliasCxt cxt = {
       .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList};
-  nodesRewriteExprsPostOrder(pSelect->pPartitionByList, replaceGroupByAliasImpl, &cxt);
+  nodesRewriteExprsPostOrder(pSelect->pPartitionByList, translateGroupPartitionByImpl, &cxt);
 
   return pCxt->errCode;
 }
@@ -5588,11 +5603,8 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
       NODES_DESTORY_LIST(pSelect->pGroupByList);
       return TSDB_CODE_SUCCESS;
     }
-    code = replaceGroupByAlias(pCxt, pSelect);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
     pSelect->timeLineResMode = TIME_LINE_NONE;
-    code = translateExprList(pCxt, pSelect->pGroupByList);
+    code = translateGroupByList(pCxt, pSelect);
   }
   return code;
 }
@@ -6287,10 +6299,7 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec
           (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) {
       pSelect->timeLineResMode = TIME_LINE_MULTI;
     }
-    code = replacePartitionByAlias(pCxt, pSelect);
+    code = translatePartitionByList(pCxt, pSelect);
-    if (TSDB_CODE_SUCCESS == code) {
-      code = translateExprList(pCxt, pSelect->pPartitionByList);
-    }
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = translateExprList(pCxt, pSelect->pTags);
@@ -6400,7 +6409,8 @@ static int32_t isOperatorTbnameInCond(STranslateContext* pCxt, SOperatorNode* pO
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo, bool* pRet) {
+static int32_t findEqCondTbNameInOperatorNode(STranslateContext* pCxt, SNode* pWhere, SEqCondTbNameTableInfo* pInfo,
+                                              bool* pRet) {
   int32_t code = TSDB_CODE_SUCCESS;
   char*   pTableAlias = NULL;
   bool    eqTbnameCond = false, tbnameInCond = false;
@@ -6714,7 +6724,8 @@ static int32_t setEqualTbnameTableVgroups(STranslateContext* pCxt, SSelectStmt*
     }
   }
 
-  qDebug("before ctbname optimize, code:%d, aTableNum:%d, nTbls:%d, stableQuery:%d", code, aTableNum, nTbls, stableQuery);
+  qDebug("before ctbname optimize, code:%d, aTableNum:%d, nTbls:%d, stableQuery:%d", code, aTableNum, nTbls,
+         stableQuery);
 
   if (TSDB_CODE_SUCCESS == code && 1 == aTableNum && 1 == nTbls && stableQuery && NULL == pInfo->pRealTable->pTsmas) {
     code = replaceToChildTableQuery(pCxt, pInfo);
@@ -6808,8 +6819,8 @@ static EDealRes collectTableAlias(SNode* pNode, void* pContext) {
     *(SSHashObj**)pContext = pHash;
   }
 
-  if (TSDB_CODE_SUCCESS != tSimpleHashPut(*(SSHashObj**)pContext, pCol->tableAlias, strlen(pCol->tableAlias), pCol->tableAlias,
+  if (TSDB_CODE_SUCCESS != tSimpleHashPut(*(SSHashObj**)pContext, pCol->tableAlias, strlen(pCol->tableAlias),
-                 sizeof(pCol->tableAlias))) {
+                                          pCol->tableAlias, sizeof(pCol->tableAlias))) {
     return DEAL_RES_ERROR;
   }
 
@@ -7075,8 +7086,7 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = resetSelectFuncNumWithoutDup(pSelect);
-    if (TSDB_CODE_SUCCESS == code)
+    if (TSDB_CODE_SUCCESS == code) code = checkAggColCoexist(pCxt, pSelect);
-      code = checkAggColCoexist(pCxt, pSelect);
   }
   /*
     if (TSDB_CODE_SUCCESS == code) {
@@ -8005,8 +8015,8 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
     code = checkOptionsDependency(pCxt, pDbName, pOptions);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code =
+    code = checkDbRangeOption(pCxt, "s3_chunkpages", pOptions->s3ChunkSize, TSDB_MIN_S3_CHUNK_SIZE,
-        checkDbRangeOption(pCxt, "s3_chunksize", pOptions->s3ChunkSize, TSDB_MIN_S3_CHUNK_SIZE, TSDB_MAX_S3_CHUNK_SIZE);
+                              TSDB_MAX_S3_CHUNK_SIZE);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = checkDbRangeOption(pCxt, "s3_compact", pOptions->s3Compact, TSDB_MIN_S3_COMPACT, TSDB_MAX_S3_COMPACT);
@@ -9024,7 +9034,8 @@ static int32_t createRollupFuncs(SCreateTableStmt* pStmt, SNodeList** ppList) {
   }
   *ppList = pFuncs;
 
-  return code;;
+  return code;
+  ;
 }
 
 static int32_t createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precision, STableMeta** ppTbMeta) {
@@ -9153,8 +9164,7 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm
   // columnDefNodeToField(pStmt->pCols, &pReq->pColumns, true);
   // columnDefNodeToField(pStmt->pTags, &pReq->pTags, true);
   code = columnDefNodeToField(pStmt->pCols, &pReq->pColumns, true);
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = tagDefNodeToField(pStmt->pTags, &pReq->pTags, true);
-    code = tagDefNodeToField(pStmt->pTags, &pReq->pTags, true);
   if (TSDB_CODE_SUCCESS == code) {
     pReq->numOfColumns = LIST_LENGTH(pStmt->pCols);
     pReq->numOfTags = LIST_LENGTH(pStmt->pTags);
@@ -9174,8 +9184,7 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm
     toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName);
     code = tNameExtractFullName(&tableName, pReq->name);
   }
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = collectUseTable(&tableName, pCxt->pTables);
-    code = collectUseTable(&tableName, pCxt->pTables);
   if (TSDB_CODE_SUCCESS == code) {
     code = collectUseTable(&tableName, pCxt->pTargetTables);
   }
@@ -9532,7 +9541,8 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p
     code = tNameExtractFullName(&name, usedbReq.db);
   }
   if (TSDB_CODE_SUCCESS == code)
-    code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs);
+    code =
+        getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs);
   if (TSDB_CODE_SUCCESS == code) {
     code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq);
   }
@@ -10112,8 +10122,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS
   } else if ('\0' != pStmt->subDbName[0]) {
     pReq->subType = TOPIC_SUB_TYPE__DB;
     code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->subDbName, strlen(pStmt->subDbName));
-    if (TSDB_CODE_SUCCESS == code)
+    if (TSDB_CODE_SUCCESS == code) (void)tNameGetFullDbName(&name, pReq->subDbName);
-      (void)tNameGetFullDbName(&name, pReq->subDbName);
   } else {
     pReq->subType = TOPIC_SUB_TYPE__COLUMN;
     char* dbName = ((SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable))->table.dbName;
@@ -10625,8 +10634,7 @@ static int32_t addNullTagsForExistTable(STranslateContext* pCxt, STableMeta* pMe
   for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < numOfTags; ++i) {
     SNode* pNull = NULL;
     code = createNullValue(&pNull);
-    if (TSDB_CODE_SUCCESS == code)
+    if (TSDB_CODE_SUCCESS == code) code = nodesListMakeStrictAppend(&pSelect->pTags, pNull);
-      code = nodesListMakeStrictAppend(&pSelect->pTags, pNull);
   }
   return code;
 }
@@ -10686,8 +10694,7 @@ static int32_t addNullTagsForCreateTable(STranslateContext* pCxt, SCreateStreamS
   for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < LIST_LENGTH(pStmt->pTags); ++i) {
     SNode* pNull = NULL;
     code = createNullValue(&pNull);
-    if (TSDB_CODE_SUCCESS == code)
+    if (TSDB_CODE_SUCCESS == code) code = nodesListMakeStrictAppend(&((SSelectStmt*)pStmt->pQuery)->pTags, pNull);
-      code = nodesListMakeStrictAppend(&((SSelectStmt*)pStmt->pQuery)->pTags, pNull);
   }
   return code;
 }
@@ -10874,7 +10881,8 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
     SValueNode* pVal = (SValueNode*)pStmt->pOptions->pDelay;
     int64_t     minDelay = 0;
     char*       str = "5s";
-    if (DEAL_RES_ERROR != translateValue(pCxt, pVal) && TSDB_CODE_SUCCESS ==
+    if (DEAL_RES_ERROR != translateValue(pCxt, pVal) &&
+        TSDB_CODE_SUCCESS ==
             parseNatualDuration(str, strlen(str), &minDelay, &pVal->unit, pVal->node.resType.precision, false)) {
       if (pVal->datum.i < minDelay) {
         return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
@@ -11170,8 +11178,7 @@ static int32_t adjustOrderOfTags(STranslateContext* pCxt, SNodeList* pTags, cons
       }
       SNode* pNull = NULL;
       code = createNullValue(&pNull);
-      if (TSDB_CODE_SUCCESS == code)
+      if (TSDB_CODE_SUCCESS == code) code = nodesListStrictAppend(pNewTagExprs, pNull);
-        code = nodesListStrictAppend(pNewTagExprs, pNull);
     }
   }
 
@@ -12077,8 +12084,7 @@ static int32_t translateShowCreateDatabase(STranslateContext* pCxt, SShowCreateD
   SName   name;
   int32_t code = tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
   (void)tNameGetFullDbName(&name, pStmt->dbFName);
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) return getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pCfg);
-    return getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pCfg);
   return code;
 }
 
@@ -12218,8 +12224,7 @@ static int32_t buildTSMAAst(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMC
   info.pDbName = pStmt->dbName;
   info.pTableName = tbName;
   code = nodesCloneList(pStmt->pOptions->pFuncs, &info.pFuncs);
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = nodesCloneNode(pStmt->pOptions->pInterval, &info.pInterval);
-    code = nodesCloneNode(pStmt->pOptions->pInterval, &info.pInterval);
 
   SFunctionNode* pTbnameFunc = NULL;
   if (TSDB_CODE_SUCCESS == code) {
@@ -12246,8 +12251,7 @@ static int32_t buildTSMAAst(STranslateContext* pCxt, SCreateTSMAStmt* pStmt, SMC
       if (TSDB_CODE_SUCCESS == code) {
         SNode* pNew = NULL;
         code = nodesCloneNode(pTagCol, &pNew);
-        if (TSDB_CODE_SUCCESS == code)
+        if (TSDB_CODE_SUCCESS == code) code = nodesListMakeStrictAppend(&info.pTags, pNew);
-          code = nodesListMakeStrictAppend(&info.pTags, pNew);
       }
     }
 
@@ -12405,8 +12409,7 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm
   } else {
     if (pReq->intervalUnit == TIME_UNIT_MONTH && (pReq->interval < 1 || pReq->interval > 12))
       return TSDB_CODE_TSMA_INVALID_INTERVAL;
-    if (pReq->intervalUnit == TIME_UNIT_YEAR && (pReq->interval != 1))
+    if (pReq->intervalUnit == TIME_UNIT_YEAR && (pReq->interval != 1)) return TSDB_CODE_TSMA_INVALID_INTERVAL;
-      return TSDB_CODE_TSMA_INVALID_INTERVAL;
   }
 
   STableMeta*     pTableMeta = NULL;
@@ -13337,11 +13340,9 @@ static int32_t addShowUserDatabasesCond(SSelectStmt* pSelect) {
   }
   nodesDestroyNode((SNode*)pValNode2);
   nodesDestroyNode((SNode*)pValNode1);
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = createLogicCondNode(&pNameCond1, &pNameCond2, &pNameCond, LOGIC_COND_TYPE_AND);
-    code = createLogicCondNode(&pNameCond1, &pNameCond2, &pNameCond, LOGIC_COND_TYPE_AND);
 
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = insertCondIntoSelectStmt(pSelect, &pNameCond);
-    code = insertCondIntoSelectStmt(pSelect, &pNameCond);
 
   if (TSDB_CODE_SUCCESS != code) {
     nodesDestroyNode(pNameCond1);
@@ -13373,8 +13374,7 @@ static int32_t addShowSystemDatabasesCond(SSelectStmt* pSelect) {
     code = createLogicCondNode(&pNameCond1, &pNameCond2, &pNameCond, LOGIC_COND_TYPE_OR);
   }
 
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = insertCondIntoSelectStmt(pSelect, &pNameCond);
-    code = insertCondIntoSelectStmt(pSelect, &pNameCond);
 
   if (TSDB_CODE_SUCCESS != code) {
     nodesDestroyNode(pNameCond1);
@@ -13390,8 +13390,7 @@ static int32_t addShowNormalTablesCond(SSelectStmt* pSelect) {
   SValueNode* pValNode1 = NULL;
   code = nodesMakeValueNodeFromString("NORMAL_TABLE", &pValNode1);
 
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = createOperatorNode(OP_TYPE_EQUAL, "type", (SNode*)pValNode1, &pTypeCond);
-    code = createOperatorNode(OP_TYPE_EQUAL, "type", (SNode*)pValNode1, &pTypeCond);
 
   nodesDestroyNode((SNode*)pValNode1);
 
@@ -13406,8 +13405,7 @@ static int32_t addShowChildTablesCond(SSelectStmt* pSelect) {
   SValueNode* pValNode1 = NULL;
   code = nodesMakeValueNodeFromString("CHILD_TABLE", &pValNode1);
 
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = createOperatorNode(OP_TYPE_EQUAL, "type", (SNode*)pValNode1, &pTypeCond);
-    code = createOperatorNode(OP_TYPE_EQUAL, "type", (SNode*)pValNode1, &pTypeCond);
 
   nodesDestroyNode((SNode*)pValNode1);
 
@@ -13506,7 +13504,8 @@ static int32_t checkShowTags(STranslateContext* pCxt, const SShowStmt* pShow) {
   int32_t     code = 0;
   SName       name = {0};
   STableMeta* pTableMeta = NULL;
-  toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, &name);
+  toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal,
+         &name);
   code = getTargetMeta(pCxt, &name, &pTableMeta, true);
   if (TSDB_CODE_SUCCESS != code) {
     code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
@@ -13670,8 +13669,7 @@ static int32_t rewriteShowTableDist(STranslateContext* pCxt, SQuery* pQuery) {
     NODES_DESTORY_LIST(pStmt->pProjectionList);
     SFunctionNode* pFuncNew = NULL;
     code = createBlockDistFunc(&pFuncNew);
-    if (TSDB_CODE_SUCCESS == code)
+    if (TSDB_CODE_SUCCESS == code) code = nodesListMakeStrictAppend(&pStmt->pProjectionList, (SNode*)pFuncNew);
-      code = nodesListMakeStrictAppend(&pStmt->pProjectionList, (SNode*)pFuncNew);
   }
   if (TSDB_CODE_SUCCESS == code) {
     pCxt->showRewrite = true;
@@ -15623,12 +15621,9 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
   SNode* pCond3 = NULL;
   SNode* pCond4 = NULL;
   code = createOperatorNode(OP_TYPE_EQUAL, "v1_status", (SNode*)pValNode, &pCond1);
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = createOperatorNode(OP_TYPE_EQUAL, "v2_status", (SNode*)pValNode, &pCond2);
-    code = createOperatorNode(OP_TYPE_EQUAL, "v2_status", (SNode*)pValNode, &pCond2);
+  if (TSDB_CODE_SUCCESS == code) code = createOperatorNode(OP_TYPE_EQUAL, "v3_status", (SNode*)pValNode, &pCond3);
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = createOperatorNode(OP_TYPE_EQUAL, "v4_status", (SNode*)pValNode, &pCond4);
-    code = createOperatorNode(OP_TYPE_EQUAL, "v3_status", (SNode*)pValNode, &pCond3);
-  if (TSDB_CODE_SUCCESS == code)
-    code = createOperatorNode(OP_TYPE_EQUAL, "v4_status", (SNode*)pValNode, &pCond4);
   nodesDestroyNode((SNode*)pValNode);
   if (TSDB_CODE_SUCCESS != code) {
     nodesDestroyNode(pCond1);
@@ -15643,10 +15638,8 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
   SNode* pTemp2 = NULL;
   SNode* pFullCond = NULL;
   code = createLogicCondNode(&pCond1, &pCond2, &pTemp1, LOGIC_COND_TYPE_OR);
-  if (TSDB_CODE_SUCCESS == code)
+  if (TSDB_CODE_SUCCESS == code) code = createLogicCondNode(&pTemp1, &pCond3, &pTemp2, LOGIC_COND_TYPE_OR);
-    code = createLogicCondNode(&pTemp1, &pCond3, &pTemp2, LOGIC_COND_TYPE_OR);
+  if (TSDB_CODE_SUCCESS == code) code = createLogicCondNode(&pTemp2, &pCond4, &pFullCond, LOGIC_COND_TYPE_OR);
-  if (TSDB_CODE_SUCCESS == code)
-    code = createLogicCondNode(&pTemp2, &pCond4, &pFullCond, LOGIC_COND_TYPE_OR);
   if (TSDB_CODE_SUCCESS != code) {
     nodesDestroyNode(pCond1);
     nodesDestroyNode(pCond2);
@@ -15969,8 +15962,8 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
   }
   // pSubSelect, pWhenThenlist need to free
 
-  // case when leader_col = count_col and leader_col > 0 then 1 when leader_col < count_col and leader_col > 0 then 2 else
+  // case when leader_col = count_col and leader_col > 0 then 1 when leader_col < count_col and leader_col > 0 then 2
-  // 0 end as status
+  // else 0 end as status
   pElse = NULL;
   code = nodesMakeValueNodeFromInt32(0, &pElse);
   if (TSDB_CODE_SUCCESS != code) {

source/libs/qworker/src/qwUtil.c

@@ -533,7 +533,7 @@ int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) {
 
   while (true) {
     tbGet = false;
-    code = qGetQueryTableSchemaVersion(pTaskInfo, dbFName, tbName, &tbInfo.sversion, &tbInfo.tversion, i, &tbGet);
+    code = qGetQueryTableSchemaVersion(pTaskInfo, dbFName, TSDB_DB_FNAME_LEN, tbName, TSDB_TABLE_NAME_LEN, &tbInfo.sversion, &tbInfo.tversion, i, &tbGet);
     if (TSDB_CODE_SUCCESS != code || !tbGet) {
       break;
     }

source/libs/scalar/src/filter.c

@@ -4133,7 +4133,7 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode)
       }
       case TSDB_DATA_TYPE_BOOL: {
         datum->kind = FLT_SCL_DATUM_KIND_INT64;
-        datum->i = (valNode->datum.b) ? 0 : 1;
+        datum->i = (valNode->datum.b) ? 1 : 0;
         break;
       }
       case TSDB_DATA_TYPE_TINYINT:
@@ -4541,6 +4541,7 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) {
 
   if (info->scalarMode) {
     SArray *colRanges = info->sclCtx.fltSclRange;
+    SOperatorNode *optNode = (SOperatorNode *) pNode;
     if (taosArrayGetSize(colRanges) == 1) {
       SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0);
       if (NULL == colRange) {
@@ -4560,7 +4561,8 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) {
         FLT_ERR_JRET(fltSclGetTimeStampDatum(endPt, &end));
         win->skey = start.i;
         win->ekey = end.i;
-        *isStrict = true;
+        if(optNode->opType == OP_TYPE_IN) *isStrict = false;
+        else *isStrict = true;
         goto _return;
       } else if (taosArrayGetSize(points) == 0) {
         *win = TSWINDOW_DESC_INITIALIZER;
@@ -5023,6 +5025,34 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) {
       }
       break;
     }
+    case OP_TYPE_IN: {
+        SNodeListNode *listNode = (SNodeListNode *)oper->valNode;
+        SListCell *cell = listNode->pNodeList->pHead;
+        SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MAX, .type = oper->colNode->node.resType};
+        SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MIN, .type = oper->colNode->node.resType};
+        for (int32_t i = 0; i < listNode->pNodeList->length; ++i) {
+          SValueNode *valueNode = (SValueNode *)cell->pNode;
+          SFltSclDatum valDatum;
+          FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, valueNode));
+          if(valueNode->node.resType.type == TSDB_DATA_TYPE_FLOAT || valueNode->node.resType.type == TSDB_DATA_TYPE_DOUBLE) {
+            minDatum.i = TMIN(minDatum.i, valDatum.d);
+            maxDatum.i = TMAX(maxDatum.i, valDatum.d);
+          } else {
+            minDatum.i = TMIN(minDatum.i, valDatum.i);
+            maxDatum.i = TMAX(maxDatum.i, valDatum.i);
+          }
+          cell = cell->pNext;
+        }
+        SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum};
+        SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum};
+        if (NULL == taosArrayPush(points, &startPt)) {
+          FLT_ERR_RET(terrno);
+        }
+        if (NULL == taosArrayPush(points, &endPt)) {
+          FLT_ERR_RET(terrno);
+        }
+        break;
+    }
     default: {
       qError("not supported operator type : %d when build range points", oper->type);
       break;
@@ -5075,11 +5105,13 @@ static bool fltSclIsCollectableNode(SNode *pNode) {
 
   if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL ||
         pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL ||
-        pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) {
+        pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL ||
+        pOper->opType == OP_TYPE_IN)) {
     return false;
   }
 
-  if (!(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE)) {
+  if (!((nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE) ||
+        (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_NODE_LIST))) {
     return false;
   }
   return true;

source/libs/stream/src/streamBackendRocksdb.c

@@ -3364,7 +3364,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) {
   return 0;
 }
 void streamStateCurNext_rocksdb(SStreamStateCur* pCur) {
-  if (pCur) {
+  if (pCur && pCur->iter && rocksdb_iter_valid(pCur->iter)) {
     rocksdb_iter_next(pCur->iter);
   }
 }

source/libs/stream/src/streamState.c

@@ -120,7 +120,7 @@ SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, i
   SStreamTask* pStreamTask = pTask;
   pState->streamId = streamId;
   pState->taskId = taskId;
-  sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-0x%x", pState->streamId, pState->taskId);
+  TAOS_UNUSED(tsnprintf(pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr), "0x%" PRIx64 "-0x%x", pState->streamId, pState->taskId));
 
   code = streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr);
   QUERY_CHECK_CODE(code, lino, _end);

source/libs/stream/src/tstreamFileState.c

@@ -777,7 +777,7 @@ _end:
 
 int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId) {
   char keyBuf[128] = {0};
-  sprintf(keyBuf, "%s:%" PRId64 "", TASK_KEY, checkpointId);
+  TAOS_UNUSED(tsnprintf(keyBuf, sizeof(keyBuf), "%s:%" PRId64 "", TASK_KEY, checkpointId));
   return streamDefaultDel_rocksdb(pFileState->pFileStore, keyBuf);
 }
 
@@ -799,14 +799,14 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
     }
     memcpy(buf, val, len);
     buf[len] = 0;
-    maxCheckPointId = atol((char*)buf);
+    maxCheckPointId = taosStr2Int64((char*)buf, NULL, 10);
     taosMemoryFree(val);
   }
   for (int64_t i = maxCheckPointId; i > 0; i--) {
     char    buf[128] = {0};
     void*   val = 0;
     int32_t len = 0;
-    sprintf(buf, "%s:%" PRId64 "", TASK_KEY, i);
+    TAOS_UNUSED(tsnprintf(buf, sizeof(buf), "%s:%" PRId64 "", TASK_KEY, i));
     code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len);
     if (code != 0) {
       return TSDB_CODE_FAILED;
@@ -816,7 +816,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
     taosMemoryFree(val);
 
     TSKEY ts;
-    ts = atol((char*)buf);
+    ts = taosStr2Int64((char*)buf, NULL, 10);
     if (ts < mark) {
       // statekey winkey.ts < mark
       int32_t tmpRes = forceRemoveCheckpoint(pFileState, i);

source/libs/sync/src/syncAppendEntries.c

@@ -95,6 +95,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
   bool               accepted = false;
   SSyncRaftEntry*    pEntry = NULL;
   bool               resetElect = false;
+  const STraceId*    trace = &pRpcMsg->info.traceId;
+  char               tbuf[40] = {0};
 
   // if already drop replica, do not process
   if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
@@ -150,7 +152,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
     goto _IGNORE;
   }
 
-  sTrace("vgId:%d, recv append entries msg. index:%" PRId64 ", term:%" PRId64 ", preLogIndex:%" PRId64
+  sGTrace("vgId:%d, recv append entries msg. index:%" PRId64 ", term:%" PRId64 ", preLogIndex:%" PRId64
           ", prevLogTerm:%" PRId64 " commitIndex:%" PRId64 " entryterm:%" PRId64,
           pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex,
           pEntry->term);
@@ -179,6 +181,11 @@ _SEND_RESPONSE:
     sTrace("vgId:%d, update commit return index %" PRId64 "", ths->vgId, returnIndex);
   }
 
+  TRACE_SET_MSGID(&(rpcRsp.info.traceId), tGenIdPI64());
+  trace = &(rpcRsp.info.traceId);
+  sGTrace("vgId:%d, send append reply matchIndex:%" PRId64 " term:%" PRId64 " lastSendIndex:%" PRId64
+          " to dest: 0x%016" PRIx64,
+          ths->vgId, pReply->matchIndex, pReply->term, pReply->lastSendIndex, pReply->destId.addr);
   // ack, i.e. send response
   TAOS_CHECK_RETURN(syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp));
 

source/libs/sync/src/syncAppendEntriesReply.c

@@ -43,6 +43,8 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
   int32_t                 code = 0;
   SyncAppendEntriesReply* pMsg = (SyncAppendEntriesReply*)pRpcMsg->pCont;
   int32_t ret = 0;
+  const STraceId*         trace = &pRpcMsg->info.traceId;
+  char                    tbuf[40] = {0};
 
   // if already drop replica, do not process
   if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
@@ -63,7 +65,7 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
       return TSDB_CODE_SYN_WRONG_TERM;
     }
 
-    sTrace("vgId:%d, received append entries reply. srcId:0x%016" PRIx64 ",  term:%" PRId64 ", matchIndex:%" PRId64 "",
+    sGTrace("vgId:%d, received append entries reply. srcId:0x%016" PRIx64 ",  term:%" PRId64 ", matchIndex:%" PRId64 "",
             pMsg->vgId, pMsg->srcId.addr, pMsg->term, pMsg->matchIndex);
 
     if (pMsg->success) {

source/libs/sync/src/syncPipeline.c

@@ -1026,6 +1026,14 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn
   int32_t         code = 0;
   if (pMgr->restored != false) return TSDB_CODE_SYN_INTERNAL_ERROR;
 
+  sTrace("vgId:%d, begin to recover sync log repl. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 ", %" PRId64
+         ", %" PRId64 ") restore:%d, buffer: [%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64
+         "), msg: {lastSendIndex:%" PRId64 ", matchIndex:%" PRId64 ", fsmState:%d, success:%d, lastMatchTerm:%" PRId64
+         "}",
+         pNode->vgId, DID(&destId), destId.addr, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pMgr->restored,
+         pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex, pMsg->lastSendIndex, pMsg->matchIndex,
+         pMsg->fsmState, pMsg->success, pMsg->lastMatchTerm);
+
   if (pMgr->endIndex == 0) {
     if (pMgr->startIndex != 0) return TSDB_CODE_SYN_INTERNAL_ERROR;
     if (pMgr->matchIndex != 0) return TSDB_CODE_SYN_INTERNAL_ERROR;
@@ -1171,6 +1179,11 @@ int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex inde
   int64_t nowMs = taosGetMonoTimestampMs();
   int32_t code = 0;
 
+  sTrace("vgId:%d, begin to probe peer:%" PRIx64 " with msg of index:%" PRId64 ". repl-mgr:[%" PRId64 ", %" PRId64
+         ", %" PRId64 "), restored:%d",
+         pNode->vgId, pNode->replicasId[pMgr->peerId].addr, index, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
+         pMgr->restored);
+
   if (pMgr->endIndex > pMgr->startIndex &&
       nowMs < pMgr->states[pMgr->startIndex % pMgr->size].timeMs + retryMaxWaitMs) {
     return 0;
@@ -1206,6 +1219,10 @@ int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex inde
 int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
   if (!pMgr->restored) return TSDB_CODE_SYN_INTERNAL_ERROR;
 
+  sTrace("vgId:%d, begin to attempt replicate log entries from end to match. repl-mgr:[%" PRId64 ", %" PRId64
+         ", %" PRId64 "), restore:%d",
+         pNode->vgId, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pMgr->restored);
+
   SRaftId*  pDestId = &pNode->replicasId[pMgr->peerId];
   int32_t   batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff));
   int32_t   code = 0;
@@ -1527,10 +1544,11 @@ int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex ind
     goto _err;
   }
 
-  TAOS_CHECK_GOTO(syncNodeSendAppendEntries(pNode, pDestId, &msgOut), &lino, _err);
+  TRACE_SET_MSGID(&(msgOut.info.traceId), tGenIdPI64());
-
+  STraceId* trace = &(msgOut.info.traceId);
-  sTrace("vgId:%d, replicate one msg index:%" PRId64 " term:%" PRId64 " prevterm:%" PRId64 " to dest: 0x%016" PRIx64,
+  sGTrace("vgId:%d, replicate one msg index:%" PRId64 " term:%" PRId64 " prevterm:%" PRId64 " to dest: 0x%016" PRIx64,
           pNode->vgId, pEntry->index, pEntry->term, prevLogTerm, pDestId->addr);
+  TAOS_CHECK_GOTO(syncNodeSendAppendEntries(pNode, pDestId, &msgOut), &lino, _err);
 
   if (!inBuf) {
     syncEntryDestroy(pEntry);

source/libs/sync/src/syncUtil.c

@@ -152,7 +152,7 @@ static void syncLogReplStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLe
   for (int32_t i = 0; i < pSyncNode->replicaNum; i++) {
     SSyncLogReplMgr* pMgr = pSyncNode->logReplMgrs[i];
     if (pMgr == NULL) break;
-    len += tsnprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 " %" PRId64 ", %" PRId64 "]", i, pMgr->restored,
+    len += tsnprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 ", %" PRId64 ", %" PRId64 "]", i, pMgr->restored,
                      pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex);
     if (i + 1 < pSyncNode->replicaNum) {
       len += tsnprintf(buf + len, bufLen - len, "%s", ", ");

source/libs/transport/inc/transComm.h

@@ -283,12 +283,12 @@ bool    transAsyncPoolIsEmpty(SAsyncPool* pool);
     if (id > 0) {                                                                        \
       SExHandle* exh2 = transAcquireExHandle(idMgt, id);                                 \
       if (exh2 == NULL || exh1 != exh2 || (exh2 != NULL && exh2->refId != id)) {         \
-        tError("handle not match, exh1:%p, exh2:%p, refId:%"PRId64"", exh1, exh2, id);                      \
+        tDebug("handle not match, exh1:%p, exh2:%p, refId:%" PRId64 "", exh1, exh2, id); \
         code = TSDB_CODE_INVALID_MSG;                                                    \
         goto _return1;                                                                   \
       }                                                                                  \
     } else {                                                                             \
-      tError("invalid handle to release");                                                                  \
+      tDebug("invalid handle to release");                                               \
       goto _return2;                                                                     \
     }                                                                                    \
   } while (0)

source/libs/transport/src/transCli.c

@@ -3090,7 +3090,7 @@ int32_t transReleaseCliHandle(void* handle) {
 
 static int32_t transInitMsg(void* pInstRef, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx, SCliReq** pCliMsg) {
   int32_t code = 0;
-  TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64());
+  if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64());
 
   SCliReq* pCliReq = NULL;
   SReqCtx* pCtx = taosMemoryCalloc(1, sizeof(SReqCtx));

source/util/src/tlog.c

@@ -23,6 +23,8 @@
 
 #define LOG_MAX_LINE_SIZE              (10024)
 #define LOG_MAX_LINE_BUFFER_SIZE       (LOG_MAX_LINE_SIZE + 3)
+#define LOG_MAX_STACK_LINE_SIZE        (512)
+#define LOG_MAX_STACK_LINE_BUFFER_SIZE (LOG_MAX_STACK_LINE_SIZE + 3)
 #define LOG_MAX_LINE_DUMP_SIZE         (1024 * 1024)
 #define LOG_MAX_LINE_DUMP_BUFFER_SIZE  (LOG_MAX_LINE_DUMP_SIZE + 128)
 
@@ -126,7 +128,7 @@ int32_t idxDebugFlag = 131;
 int32_t sndDebugFlag = 131;
 int32_t simDebugFlag = 131;
 
-int32_t tqClientDebug = 0;
+int32_t tqClientDebugFlag = 131;
 
 int64_t dbgEmptyW = 0;
 int64_t dbgWN = 0;
@@ -669,16 +671,40 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b
   }
 }
 
-void taosPrintLog(const char *flags, int32_t level, int32_t dflag, const char *format, ...) {
+/*
-  if (!(dflag & DEBUG_FILE) && !(dflag & DEBUG_SCREEN)) return;
+  use taosPrintLogImpl_useStackBuffer to avoid stack overflow
 
-  char    buffer[LOG_MAX_LINE_BUFFER_SIZE];
+*/
+static int8_t taosPrintLogImplUseStackBuffer(const char *flags, int32_t level, int32_t dflag, const char *format,
+                                             va_list args) {
+  char    buffer[LOG_MAX_STACK_LINE_BUFFER_SIZE];
   int32_t len = taosBuildLogHead(buffer, flags);
 
-  va_list argpointer;
+  int32_t writeLen = len + vsnprintf(buffer + len, LOG_MAX_STACK_LINE_BUFFER_SIZE - len - 1, format, args);
-  va_start(argpointer, format);
+  if (writeLen > LOG_MAX_STACK_LINE_SIZE) {
-  int32_t writeLen = len + vsnprintf(buffer + len, LOG_MAX_LINE_BUFFER_SIZE - len, format, argpointer);
+    return 1;
-  va_end(argpointer);
+  }
+
+  buffer[writeLen++] = '\n';
+  buffer[writeLen] = 0;
+
+  taosPrintLogImp(level, dflag, buffer, writeLen);
+
+  if (tsLogFp && level <= DEBUG_INFO) {
+    buffer[writeLen - 1] = 0;
+    (*tsLogFp)(taosGetTimestampMs(), level, buffer + len);
+  }
+  return 0;
+}
+static int8_t taosPrintLogImplUseHeapBuffer(const char *flags, int32_t level, int32_t dflag, const char *format,
+                                            va_list args) {
+  char *buffer = taosMemoryCalloc(1, LOG_MAX_LINE_BUFFER_SIZE + 1);
+  if (buffer == NULL) {
+    return 1;
+  }
+  int32_t len = taosBuildLogHead(buffer, flags);
+
+  int32_t writeLen = len + vsnprintf(buffer + len, LOG_MAX_LINE_BUFFER_SIZE - len - 1, format, args);
 
   if (writeLen > LOG_MAX_LINE_SIZE) writeLen = LOG_MAX_LINE_SIZE;
   buffer[writeLen++] = '\n';
@@ -690,6 +716,22 @@ void taosPrintLog(const char *flags, int32_t level, int32_t dflag, const char *f
     buffer[writeLen - 1] = 0;
     (*tsLogFp)(taosGetTimestampMs(), level, buffer + len);
   }
+  taosMemoryFree(buffer);
+  return 0;
+}
+void taosPrintLog(const char *flags, int32_t level, int32_t dflag, const char *format, ...) {
+  if (!(dflag & DEBUG_FILE) && !(dflag & DEBUG_SCREEN)) return;
+
+  va_list argpointer, argpointer_copy;
+  va_start(argpointer, format);
+  va_copy(argpointer_copy, argpointer);
+
+  if (taosPrintLogImplUseStackBuffer(flags, level, dflag, format, argpointer) == 0) {
+  } else {
+    TAOS_UNUSED(taosPrintLogImplUseHeapBuffer(flags, level, dflag, format, argpointer_copy));
+  }
+  va_end(argpointer_copy);
+  va_end(argpointer);
 }
 
 void taosPrintLongString(const char *flags, int32_t level, int32_t dflag, const char *format, ...) {

source/util/test/CMakeLists.txt

@@ -126,6 +126,13 @@ add_test(
     COMMAND regexTest
 )
 
+add_executable(logTest "log.cpp")
+    target_link_libraries(logTest os util common gtest_main)
+    add_test(
+        NAME logTest
+        COMMAND logTest 
+)
+
 add_executable(decompressTest "decompressTest.cpp")
 target_link_libraries(decompressTest os util common gtest_main)
 add_test(

source/util/test/log.cpp

@@ -0,0 +1,46 @@
+#include <gtest/gtest.h>
+#include <stdlib.h>
+#include <time.h>
+#include <random>
+#include <tlog.h> 
+#include <iostream> 
+
+using namespace std;
+
+
+TEST(log, check_log_refactor) {
+  const char   *logDir = "/tmp";
+  const char   *defaultLogFileNamePrefix = "taoslog";
+  const int32_t maxLogFileNum = 10000;
+  tsAsyncLog = 0;
+  // idxDebugFlag = 143;
+  strcpy(tsLogDir, (char *)logDir);
+  taosInitLog(tsLogDir, 10, false);
+  tsAsyncLog = 0;
+  uDebugFlag = 143;
+
+  std::string str;
+  str.push_back('a');
+  
+  for (int i = 0;  i < 10000; i += 2) {	 
+    str.push_back('a');
+    uError("write to file %s", str.c_str());
+  }
+  str.clear();
+  for (int i = 0;  i < 10000; i += 2) {	 
+    str.push_back('a');
+    uDebug("write to file %s", str.c_str());
+  }
+
+  for (int i = 0;  i < 10000; i += 2) {	 
+    str.push_back('a');
+    uInfo("write to file %s", str.c_str());
+  }
+  str.clear();
+
+  for (int i = 0;  i < 10000; i += 2) {	 
+    str.push_back('a');
+    uTrace("write to file %s", str.c_str());
+  }
+  taosCloseLog();
+}

tests/army/frame/common.py

@@ -803,11 +803,14 @@ class TDCom:
         else:
             tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}")
 
-    def killProcessor(self, processorName):
+    def kill_signal_process(self,  signal=15, processor_name: str = "taosd"):   
         if (platform.system().lower() == 'windows'):
-            os.system("TASKKILL /F /IM %s.exe"%processorName)
+            os.system(f"TASKKILL /F /IM {processor_name}.exe")
         else:
-            os.system("unset LD_PRELOAD; pkill %s " % processorName)
+            command = f"unset LD_PRELOAD; sudo pkill -f -{signal} '{processor_name}'"
+            tdLog.debug(f"command: {command}")
+            os.system(command)
+
 
     def gen_tag_col_str(self, gen_type, data_type, count):
         """

tests/army/storage/s3/s3Basic.json

@@ -20,7 +20,7 @@
                 "replica": 1,
                 "duration":"10d",
                 "s3_keeplocal":"30d", 
-                "s3_chunksize":"131072",
+                "s3_chunkpages":"131072",
                 "tsdb_pagesize":"1",
                 "s3_compact":"1",
                 "wal_retention_size":"1",

tests/army/storage/s3/s3Basic.py

@@ -168,13 +168,13 @@ class TDTestCase(TBase):
         if keepLocal is not None:
             kw1 = f"s3_keeplocal {keepLocal}"
         if chunkSize is not None:
-            kw2 = f"s3_chunksize {chunkSize}"
+            kw2 = f"s3_chunkpages {chunkSize}"
         if compact is not None:
             kw3 = f"s3_compact {compact}"    
 
         sql = f" create database db1 vgroups 1 duration 1h {kw1} {kw2} {kw3}"
         tdSql.execute(sql, show=True)
-        #sql = f"select name,s3_keeplocal,s3_chunksize,s3_compact from information_schema.ins_databases where name='db1';"
+        #sql = f"select name,s3_keeplocal,s3_chunkpages,s3_compact from information_schema.ins_databases where name='db1';"
         sql = f"select * from information_schema.ins_databases where name='db1';"
         tdSql.query(sql)
         # 29 30 31 -> chunksize keeplocal compact
@@ -194,9 +194,9 @@ class TDTestCase(TBase):
             f"create database db2 s3_keeplocal -1",
             f"create database db2 s3_keeplocal 0",
             f"create database db2 s3_keeplocal 365001",
-            f"create database db2 s3_chunksize -1",
+            f"create database db2 s3_chunkpages -1",
-            f"create database db2 s3_chunksize 0",
+            f"create database db2 s3_chunkpages 0",
-            f"create database db2 s3_chunksize 900000000",
+            f"create database db2 s3_chunkpages 900000000",
             f"create database db2 s3_compact -1",
             f"create database db2 s3_compact 100",
             f"create database db2 duration 1d s3_keeplocal 1d"

tests/army/storage/s3/s3Basic1.json

@@ -20,7 +20,7 @@
                 "replica": 1,
                 "duration":"10d",
                 "s3_keeplocal":"30d", 
-                "s3_chunksize":"131072",
+                "s3_chunkpages":"131072",
                 "tsdb_pagesize":"1",
                 "s3_compact":"1",
                 "wal_retention_size":"1",

tests/army/storage/s3/s3azure.py

@@ -202,13 +202,13 @@ class TDTestCase(TBase):
         if keepLocal is not None:
             kw1 = f"s3_keeplocal {keepLocal}"
         if chunkSize is not None:
-            kw2 = f"s3_chunksize {chunkSize}"
+            kw2 = f"s3_chunkpages {chunkSize}"
         if compact is not None:
             kw3 = f"s3_compact {compact}"
 
         sql = f" create database db1 vgroups 1 duration 1h {kw1} {kw2} {kw3}"
         tdSql.execute(sql, show=True)
-        # sql = f"select name,s3_keeplocal,s3_chunksize,s3_compact from information_schema.ins_databases where name='db1';"
+        # sql = f"select name,s3_keeplocal,s3_chunkpages,s3_compact from information_schema.ins_databases where name='db1';"
         sql = f"select * from information_schema.ins_databases where name='db1';"
         tdSql.query(sql)
         # 29 30 31 -> chunksize keeplocal compact
@@ -228,9 +228,9 @@ class TDTestCase(TBase):
             f"create database db2 s3_keeplocal -1",
             f"create database db2 s3_keeplocal 0",
             f"create database db2 s3_keeplocal 365001",
-            f"create database db2 s3_chunksize -1",
+            f"create database db2 s3_chunkpages -1",
-            f"create database db2 s3_chunksize 0",
+            f"create database db2 s3_chunkpages 0",
-            f"create database db2 s3_chunksize 900000000",
+            f"create database db2 s3_chunkpages 900000000",
             f"create database db2 s3_compact -1",
             f"create database db2 s3_compact 100",
             f"create database db2 duration 1d s3_keeplocal 1d"

tests/army/tmq/drop_lost_comsumers.py

@@ -0,0 +1,337 @@
+import time
+import os
+import threading
+import datetime
+from taos.tmq import Consumer
+from taos.error import TmqError
+
+from frame.log import tdLog
+from frame.cases import tdCases
+from frame.sql import tdSql
+from frame.caseBase import *
+from frame import etool
+from frame.common import tdCom
+
+
+class TaosConsumer:
+    # TODO: Move this class to tq.py and remove it from here
+    def __init__(self):
+        self.sub_once = True
+        self.once_consumer_rows = 0
+        self.sub_log = False
+        self.safe_counter = ThreadSafeCounter()
+
+    def log_info(self, message):
+        if self.sub_log:
+            tdLog.info(message)
+    
+    #TODO merge sub_consumer and sub_consumer_once 
+    def sub_consumer(self, consumer, group_id, topic_name):
+        group_id = int(group_id)
+        if group_id < 100:
+            try:
+                consumer.subscribe([topic_name])
+            except TmqError:
+                tdLog.exit(f"subscribe error")
+        nrows = 0
+        while True:
+            start = datetime.datetime.now()
+            tdLog.info(f"time:{start},consumer:{group_id}, start to consume")
+            message = consumer.poll(timeout=10.0)
+
+            if message:
+                message_offset = message.offset()
+                # topic = message.topic()
+                # database = message.database()
+
+                for block in message:
+                    addrows = block.nrows()
+                    nrows += block.nrows()
+                    ncols = block.ncols()
+                    # values = block.fetchall
+                end = datetime.datetime.now()
+                elapsed_time = end - start
+                tdLog.info(
+                    f"time:{end},consumer:{group_id}, elapsed time:{elapsed_time},"
+                    f"consumer_nrows:{nrows},consumer_addrows:{addrows},"
+                    f"consumer_ncols:{ncols},offset:{id}"
+                )
+            consumer.commit()
+            tdLog.info(f"consumer:{group_id},consumer_nrows:{nrows}")
+            # consumer.unsubscribe()
+            # consumer.close()
+
+    def set_conf(
+        self,
+        td_connect_ip="localhost",
+        group_id=1,
+        client_id="test_consumer_py",
+        enable_auto_commit="false",
+        auto_commit_interval_ms="1000",
+        auto_offset_reset="earliest",
+        msg_with_table_name="true",
+        session_timeout_ms=10000,
+        max_poll_interval_ms=180000,
+        experimental_snapshot_enable="false",
+    ):
+        conf = {
+            # auth options
+            # consume options
+            "td.connect.ip": f"{td_connect_ip}",
+            "group.id": f"{group_id}",
+            "client.id": f"{client_id}",
+            "enable.auto.commit": f"{enable_auto_commit}",
+            "auto.commit.interval.ms": f"{auto_commit_interval_ms}",
+            "auto.offset.reset": f"{auto_offset_reset}",
+            "msg.with.table.name": f"{msg_with_table_name}",
+            "session.timeout.ms": f"{session_timeout_ms}",
+            "max.poll.interval.ms": f"{max_poll_interval_ms}",
+            "experimental.snapshot.enable": f"{experimental_snapshot_enable}",
+        }
+        return conf
+
+    def sub_consumer_once(self, consumer, group_id, topic_name, stop_event):
+        group_id = int(group_id)
+        if group_id < 100:
+            consumer.subscribe([topic_name])
+        nrows = 0
+        consumer_nrows = 0
+        count = 0
+        while not stop_event.is_set():
+            start = datetime.datetime.now()
+            # self.log_info(
+            #     f"time:{start},consumer:{group_id}, start to consume,consumer_nrows:{consumer_nrows}"
+            # )
+            message = None 
+            if consumer_nrows < self.once_consumer_rows:
+                message = consumer.poll(timeout=1.0)
+            elif consumer_nrows >= self.once_consumer_rows:
+                if count == 0:
+                    # when break the loop, the consumer will be closed, so we need to continue to keep consumer alive util the stop_event is set
+                    tdLog.info("stop consumer when consumer all rows")
+                    count += 1
+                    # tdLog.info("stop consumer when consumer all rows")
+                else:
+                    continue
+            if message:
+                message_offset = message.offset()
+                # topic = message.topic()
+                # database = message.database()
+                for block in message:
+                    addrows = block.nrows()
+                    nrows += block.nrows()
+                    self.safe_counter.rows(block.nrows())
+                    ncols = block.ncols()
+                    # values = block.fetchall
+                end = datetime.datetime.now()
+                elapsed_time = end - start
+
+                # self.log_info(
+                #     f"time:{end},consumer:{group_id}, elapsed time:{elapsed_time},consumer_nrows:{nrows},consumer_addrows:{addrows}, consumer_ncols:{ncols},offset:{message_offset}"
+                # )
+            self.log_info(
+                f"consumer:{group_id},consumer_nrows:{nrows},counter.counter:{self.safe_counter.counter},counter.get():{self.safe_counter.get()}"
+            )
+            
+            # consumer.commit()
+            consumer_nrows = nrows
+
+        tdLog.info("Consumer subscription thread is stopping.")
+
+    def taosc_consumer(self, conf: list, topic_name: str, stop_event: threading.Event):
+        try:
+            tdLog.info(conf)
+            tdLog.info("start to config consumer")
+            consumer = Consumer(conf)
+            tdLog.info("start to subscribe")
+            group_id = int(conf["group.id"])
+            tdLog.info(f"{consumer},{group_id}")
+            if self.sub_once:
+                self.sub_consumer_once(consumer, group_id, topic_name, stop_event)
+            else:
+                self.sub_consumer(consumer, group_id, topic_name)
+            # only consumer once
+        except Exception as e:
+            tdLog.exit(f"{e}")
+
+        # consumer.close()
+
+
+class ThreadSafeCounter:
+    def __init__(self):
+        self.counter = 0
+        self.lock = threading.Lock()
+
+    def rows(self, rows):
+        with self.lock:
+            self.counter += rows
+
+    def get(self):
+        with self.lock:
+            return self.counter
+
+
+class TDTestCase:
+    # updatecfgDict = {'debugFlag': 135, 'asynclog': 0}
+    def __init__(self):
+        # db parameter
+        self.table_number = 1000
+        self.rows_per_table = 1000
+        # consumer parameter
+        self.consumer_groups_num = 2
+        self.session_timeout_ms = 180000
+        self.max_poll_interval_ms = 180000
+        # case consumer parameter
+        self.consumer_rows_per_thread = self.table_number * self.rows_per_table
+        self.consumer_all_rows = (
+            self.consumer_rows_per_thread * self.consumer_groups_num
+        )
+        self.topic_name = "select_d1"
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), logSql)
+        self.consumer_instance = TaosConsumer()
+        # tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+
+    def caseDescription(self):
+        """
+        drop_lost_consmuers<hrchen>:
+        1. verifying that the boundary and valid values of session_timeout_ms are in effect
+        2. verifying that the boundary and valid values of max_poll_interval_ms are in effect
+        3. verifying that consumer will be closed when the session_timeout_ms and max_poll_interval_ms is expired
+        """
+        return
+
+    def check_consumer(self, count, rows, stop_event=None):
+        time.sleep(count)
+        try:
+            tdLog.info(
+                f"wait timeout count:{count} and check consumer status whether is closed"
+            )
+            for _ in range(2):
+                tdSql.query("show consumers")
+                anser_rows = tdSql.getRows()
+                if anser_rows == rows:
+                    break
+                time.sleep(1)
+                tdLog.info(
+                    f"wait for {count} seconds to check that consumers number is {anser_rows}"
+                )
+            if anser_rows != rows:
+                if stop_event:
+                    stop_event.set()
+                tdLog.exit(f"consumer number is {anser_rows } but not  expected {rows}")
+        except Exception as e:
+            tdLog.exit(f"{e},check consumer error")
+
+    def drop_session_timeout_consmuers(self):
+        tdSql.execute(f"drop topic if exists {self.topic_name};")
+        tdSql.execute("use db_sub")
+        tdSql.execute(f"create topic {self.topic_name} as  select * from db_sub.meters;")
+
+        # start consumer and config some parameters
+        os.system(
+            f"nohup python3 ./tmq/per_consumer.py  -c {self.consumer_groups_num} -s {self.session_timeout_ms} -p {self.max_poll_interval_ms} -t {self.topic_name} > consumer.log &"
+        )
+        # wait 5s for consuming data
+        time.sleep(5)
+        # kill consumer to simulate session_timeout_ms
+        tdLog.info("kill per_consumer.py")
+        tdCom.kill_signal_process(
+            signal=9, processor_name=r"python3\s*./tmq/per_consumer.py"
+        )
+        self.check_consumer(int(self.session_timeout_ms / 1000), 0)
+        tdSql.execute(f"drop topic if exists {self.topic_name};")
+        os.system("rm -rf consumer.log")
+
+    def drop_max_poll_timeout_consmuers(self):
+        tdSql.execute(f"drop topic if exists {self.topic_name};")
+        tdSql.execute("use db_sub")
+        tdSql.execute(f"create topic {self.topic_name} as  select * from db_sub.meters;")
+
+        threads = []
+        self.safe_counter = ThreadSafeCounter()
+        self.consumer_instance.safe_counter = self.safe_counter
+        stop_event = threading.Event()
+        self.consumer_instance.once_consumer_rows = self.consumer_rows_per_thread
+        tdLog.info(f"consumer_rows:{self.consumer_instance.once_consumer_rows}")
+        self.consumer_instance.sub_once = True
+        for group_id in range(self.consumer_groups_num):
+            conf = self.consumer_instance.set_conf(
+                group_id=group_id,
+                session_timeout_ms=self.session_timeout_ms,
+                max_poll_interval_ms=self.max_poll_interval_ms,
+            )
+            threads.append(
+                threading.Thread(
+                    target=self.consumer_instance.taosc_consumer,
+                    args=(conf, self.topic_name, stop_event),
+                )
+            )
+        for tr in threads:
+            tr.start()
+
+        while True:
+            if self.safe_counter.counter < self.consumer_all_rows:
+                # control print log frequency
+                time.sleep(1)
+                tdLog.info(
+                    f"consumer_all_rows:{self.consumer_all_rows},counter.get():{self.safe_counter.counter}"
+                )
+            elif self.safe_counter.counter == self.consumer_all_rows:
+                # adding 5s is for heartbeat check
+                self.check_consumer(int(self.max_poll_interval_ms / 1000 ) + 5, 0, stop_event)
+                stop_event.set()
+                break
+        
+        time.sleep(1)
+        tdSql.execute(f"drop topic if exists {self.topic_name};")
+
+    def case_session_timeout(self):
+        """
+        TEST CASE: verifying that the boundary and valid values of session_timeout_ms are in effect
+        """
+
+        tdLog.info("start to test session_timeout_ms=12s")
+        # test session_timeout_ms=12s
+        self.session_timeout_ms = 12000
+        self.max_poll_interval_ms = 180000
+        # self.set_session_timeout = int(self.session_timeout_ms / 1000)
+        self.drop_session_timeout_consmuers()
+        tdLog.info("stop to test session_timeout_ms=12s and done ")
+
+    def case_max_poll_timeout(self):
+        """
+        TEST CASE: verifying that the boundary and valid values of max_poll_interval_ms are in effect
+        """
+        tdLog.info("start to test max_poll_interval_ms=20s")
+        # test max_poll_interval_ms=20s
+        self.session_timeout_ms = 300000
+        self.max_poll_interval_ms = 20000
+        self.drop_max_poll_timeout_consmuers()
+        tdLog.info("stop to test max_poll_interval_ms=20s and done ")
+
+    def run(self):
+        """
+        Run the test cases for session timeout and max poll timeout.
+        """
+        vgroups = 4
+        etool.benchMark(
+            command=f"-d db_sub -t {self.table_number} -n {self.rows_per_table} -v {vgroups} -a {self.replicaVar} -y"
+        )
+        # test case start here
+        self.topic_name = "select_d1"
+        # self.case_session_timeout()
+        self.case_max_poll_timeout()
+
+    def stop(self):
+        """
+        Closes the taos connection and logs the success message.
+        """
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())