Merge remote-tracking branch 'origin/main' into feat/TD-22746

Jenkinsfile2

@@ -423,7 +423,7 @@ pipeline {
                             echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
                         }
                         catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                            timeout(time: 120, unit: 'MINUTES'){
+                            timeout(time: 130, unit: 'MINUTES'){
                                 pre_test()
                                 script {
                                     sh '''

cmake/cmake.version

@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.0.2.5")
+  SET(TD_VER_NUMBER "3.0.2.6")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 634399d
+        GIT_TAG 1e15545
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/en/07-develop/07-tmq.mdx

@@ -222,7 +222,7 @@ A database including one supertable and two subtables is created as follows:
 ```sql
 DROP DATABASE IF EXISTS tmqdb;
 CREATE DATABASE tmqdb;
-CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16) TAGS(t1 INT, t3 VARCHAR(16));
+CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16));
 CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0");
 CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1");       
 INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00');

docs/en/07-develop/_sub_java.mdx

@@ -1,11 +1,31 @@
-```java
+import Tabs from '@theme/Tabs';
-{{#include docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java}}
+import TabItem from '@theme/TabItem';
-{{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}}
+
-{{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}}
+<Tabs defaultValue="native">
-```
+  <TabItem value="native" label="native connection">
-```java
+
-{{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}}
+    ```java
-```
+    {{#include docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java}}
-```java
+    ```
-{{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}}
+    ```java
-```
+    {{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}}
+    ```
+    ```java
+    {{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}}
+    ```
+
+  </TabItem>
+  <TabItem value="ws" label="WebSocket connection">
+
+    ```java
+    {{#include docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java}}
+    ```
+    ```java
+    {{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}}
+    ```
+    ```java
+    {{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}}
+    ```
+
+  </TabItem>
+</Tabs>

docs/en/12-taos-sql/10-function.md

@@ -796,19 +796,23 @@ HISTOGRAM(expr，bin_type, bin_description, normalized)
 ### PERCENTILE
 
 ```sql
-PERCENTILE(expr, p)
+PERCENTILE(expr, p [, p1] ...)
 ```
 
 **Description**: The value whose rank in a specific column matches the specified percentage. If such a value matching the specified percentage doesn't exist in the column, an interpolation value will be returned.
 
-**Return value type**: DOUBLE
+**Return value type**: This function takes 2 minumum and 11 maximum parameters, and it can simultaneously return 10 percentiles at most. If 2 parameters are given, a single percentile is returned and the value type is DOUBLE.
+                       If more than 2 parameters are given, the return value type is a VARCHAR string, the format of which is a JSON ARRAY containing all return values.
 
 **Applicable column types**: Numeric
 
 **Applicable table types**: table only
 
-**More explanations**: _p_ is in range [0,100], when _p_ is 0, the result is same as using function MIN; when _p_ is 100, the result is same as function MAX.
+**More explanations**:
 
+- _p_ is in range [0,100], when _p_ is 0, the result is same as using function MIN; when _p_ is 100, the result is same as function MAX.
+- When calculating multiple percentiles of a specific column, a single PERCENTILE function with multiple parameters is adviced, as this can largely reduce the query response time.
+  For example, using SELECT percentile(col, 90, 95, 99) FROM table will perform better than SELECT percentile(col, 90), percentile(col, 95), percentile(col, 99) from table.
 
 ## Selection Functions
 

docs/en/14-reference/03-connector/04-java.mdx

@@ -696,6 +696,9 @@ TaosConsumer consumer = new TaosConsumer<>(config);
 - enable.auto.commit: Specifies whether to commit automatically.
 - group.id: consumer: Specifies the group that the consumer is in.
 - value.deserializer: To deserialize the results, you can inherit `com.taosdata.jdbc.tmq.ReferenceDeserializer` and specify the result set bean. You can also inherit `com.taosdata.jdbc.tmq.Deserializer` and perform custom deserialization based on the SQL result set.
+- td.connect.type: Specifies the type connect with TDengine, `jni` or `WebSocket`. default is `jni`
+- httpConnectTimeout：WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type.
+- messageWaitTimeout：socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type.
 - For more information, see [Consumer Parameters](../../../develop/tmq).
 
 #### Subscribe to consume data
@@ -724,6 +727,11 @@ For more information, see [Data Subscription](../../../develop/tmq).
 
 ### Usage examples
 
+<Tabs defaultValue="native">
+<TabItem value="native" label="native connection">
+
+In addition to the native connection, the Java Connector also supports subscribing via websocket.
+
 ```java
 public abstract class ConsumerLoop {
     private final TaosConsumer<ResultBean> consumer;
@@ -795,6 +803,87 @@ public abstract class ConsumerLoop {
 }
 ```
 
+</TabItem>
+<TabItem value="ws" label="WebSocket connection">
+
+```java
+public abstract class ConsumerLoop {
+    private final TaosConsumer<ResultBean> consumer;
+    private final List<String> topics;
+    private final AtomicBoolean shutdown;
+    private final CountDownLatch shutdownLatch;
+
+    public ConsumerLoop() throws SQLException {
+        Properties config = new Properties();
+        config.setProperty("bootstrap.servers", "localhost:6041");
+        config.setProperty("td.connect.type", "ws");
+        config.setProperty("msg.with.table.name", "true");
+        config.setProperty("enable.auto.commit", "true");
+        config.setProperty("group.id", "group2");
+        config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+
+        this.consumer = new TaosConsumer<>(config);
+        this.topics = Collections.singletonList("topic_speed");
+        this.shutdown = new AtomicBoolean(false);
+        this.shutdownLatch = new CountDownLatch(1);
+    }
+
+    public abstract void process(ResultBean result);
+
+    public void pollData() throws SQLException {
+        try {
+            consumer.subscribe(topics);
+
+            while (!shutdown.get()) {
+                ConsumerRecords<ResultBean> records = consumer.poll(Duration.ofMillis(100));
+                for (ResultBean record : records) {
+                    process(record);
+                }
+            }
+            consumer.unsubscribe();
+        } finally {
+            consumer.close();
+            shutdownLatch.countDown();
+        }
+    }
+
+    public void shutdown() throws InterruptedException {
+        shutdown.set(true);
+        shutdownLatch.await();
+    }
+
+    public static class ResultDeserializer extends ReferenceDeserializer<ResultBean> {
+
+    }
+
+    public static class ResultBean {
+        private Timestamp ts;
+        private int speed;
+
+        public Timestamp getTs() {
+            return ts;
+        }
+
+        public void setTs(Timestamp ts) {
+            this.ts = ts;
+        }
+
+        public int getSpeed() {
+            return speed;
+        }
+
+        public void setSpeed(int speed) {
+            this.speed = speed;
+        }
+    }
+}
+```
+
+</TabItem>
+</Tabs>
+
+> **Note**: The value of value.deserializer should be adjusted based on the package path of the test environment.
+
 ### Use with connection pool
 
 #### HikariCP
@@ -878,8 +967,8 @@ The source code of the sample application is under `TDengine/examples/JDBC`:
 
 | taos-jdbcdriver version |                 major changes                  |
 | :---------------------: | :--------------------------------------------: |
-|       3.0.3        |  fix timestamp resolution error for REST connection in jdk17+ version    |
+|       3.1.0       |       JDBC REST connection supports subscription over WebSocket |
-|       3.0.1 - 3.0.2        |  fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use 3.0.2 in the JDK 8 environment    |
+|       3.0.1 - 3.0.4        |  fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment    |
 |       3.0.0        |   Support for TDengine 3.0    |
 |       2.0.42        |   fix wasNull interface return value in WebSocket connection  |
 |       2.0.41        |   fix decode method of username and password in REST connection |

docs/en/14-reference/11-docker/index.md

@@ -273,49 +273,48 @@ password:             taosdata
 
 ## Start the TDengine cluster with docker-compose
 
-1. The following docker-compose file starts a TDengine cluster with two replicas, two management nodes, two data nodes, and one arbitrator.
+1. The following docker-compose file starts a TDengine cluster with three nodes.
 
-   ```docker
+```yml
-   version: "3"
+version: "3"
-   services:
+services:
-     arbitrator:
+  td-1:
-       image: tdengine/tdengine:$VERSION
+    image: tdengine/tdengine:$VERSION
-       command: tarbitrator
+    environment:
-     td-1:
+      TAOS_FQDN: "td-1"
-       image: tdengine/tdengine:$VERSION
+      TAOS_FIRST_EP: "td-1"
-       environment:
+    volumes:
-         TAOS_FQDN: "td-1"
+      - taosdata-td1:/var/lib/taos/
-         TAOS_FIRST_EP: "td-1"
+      - taoslog-td1:/var/log/taos/
-         TAOS_NUM_OF_MNODES: "2"
+  td-2:
-         TAOS_REPLICA: "2"
+    image: tdengine/tdengine:$VERSION
-         TAOS_ARBITRATOR: arbitrator:6042
+    environment:
-       volumes:
+      TAOS_FQDN: "td-2"
-         - taosdata-td1:/var/lib/taos/
+      TAOS_FIRST_EP: "td-1"
-         - taoslog-td1:/var/log/taos/
+    volumes:
-     td-2:
+      - taosdata-td2:/var/lib/taos/
-       image: tdengine/tdengine:$VERSION
+      - taoslog-td2:/var/log/taos/
-       environment:
+  td-3:
-         TAOS_FQDN: "td-2"
+    image: tdengine/tdengine:$VERSION
-         TAOS_FIRST_EP: "td-1"
+    environment:
-         TAOS_NUM_OF_MNODES: "2"
+      TAOS_FQDN: "td-3"
-         TAOS_REPLICA: "2"
+      TAOS_FIRST_EP: "td-1"
-         TAOS_ARBITRATOR: arbitrator:6042
+    volumes:
-       volumes:
+      - taosdata-td3:/var/lib/taos/
-         - taosdata-td2:/var/lib/taos/
+      - taoslog-td3:/var/log/taos/
-         - taoslog-td2:/var/log/taos/
+volumes:
-   volumes:
+  taosdata-td1:
-     taosdata-td1:
+  taoslog-td1:
-     taoslog-td1:
+  taosdata-td2:
-     taosdata-td2:
+  taoslog-td2:
-     taoslog-td2:
+  taosdata-td3:
-   ```
+  taoslog-td3:
+```
 
 :::note
 
 - The `VERSION` environment variable is used to set the tdengine image tag
 - `TAOS_FIRST_EP` must be set on the newly created instance so that it can join the TDengine cluster; if there is a high availability requirement, `TAOS_SECOND_EP` needs to be used at the same time
-- `TAOS_REPLICA` is used to set the default number of database replicas. Its value range is [1,3]
-  We recommend setting it with `TAOS_ARBITRATOR` to use arbitrator in a two-nodes environment.
   :::
 
 2. Start the cluster
@@ -345,17 +344,18 @@ password:             taosdata
 
 4. Show dnodes via TDengine CLI
 
-   ```shell
+```shell
-   $ docker-compose exec td-1 taos -s "show dnodes"
+$ docker-compose exec td-1 taos -s "show dnodes"
 
-   taos> show dnodes
+taos> show dnodes
-      id   |           end_point            | vnodes | cores  |   status   | role  |       create_time       |      offline reason      |
+     id      |            endpoint            | vnodes | support_vnodes |   status   |       create_time       |              note              |
-   ======================================================================================================================================
+======================================================================================================================================
-         1 | td-1:6030                      |      1 |      8 | ready      | any   | 2022-01-18 02:47:42.871 |                          |
+           1 | td-1:6030                      |      0 |             32 | ready      | 2022-08-19 07:57:29.971 |                                |
-         2 | td-2:6030                      |      0 |      8 | ready      | any   | 2022-01-18 02:47:43.518 |                          |
+           2 | td-2:6030                      |      0 |             32 | ready      | 2022-08-19 07:57:31.415 |                                |
-         0 | arbitrator:6042                |      0 |      0 | ready      | arb   | 2022-01-18 02:47:43.633 | -                        |
+           3 | td-3:6030                      |      0 |             32 | ready      | 2022-08-19 07:57:31.417 |                                |
-   Query OK, 3 row(s) in set (0.000811s)
+Query OK, 3 rows in database (0.021262s)
-   ```
+
+```
 
 ## taosAdapter
 
@@ -373,83 +373,70 @@ password:             taosdata
 
    Suppose you want to deploy multiple taosAdapters to improve throughput and provide high availability. In that case, the recommended configuration method uses a reverse proxy such as Nginx to offer a unified access entry. For specific configuration methods, please refer to the official documentation of Nginx. Here is an example:
 
-   ```docker
+```yml
-   version: "3"
+version: "3"
 
-   networks:
+networks:
-     inter:
+  inter:
-     api:
 
-   services:
+services:
-     arbitrator:
+  td-1:
-       image: tdengine/tdengine:$VERSION
+    image: tdengine/tdengine:$VERSION
-       command: tarbitrator
+    networks:
-       networks:
+      - inter
-         - inter
+    environment:
-     td-1:
+      TAOS_FQDN: "td-1"
-       image: tdengine/tdengine:$VERSION
+      TAOS_FIRST_EP: "td-1"
-       networks:
+    volumes:
-         - inter
+      - taosdata-td1:/var/lib/taos/
-       environment:
+      - taoslog-td1:/var/log/taos/
-         TAOS_FQDN: "td-1"
+  td-2:
-         TAOS_FIRST_EP: "td-1"
+    image: tdengine/tdengine:$VERSION
-         TAOS_NUM_OF_MNODES: "2"
+    networks:
-         TAOS_REPLICA: "2"
+      - inter
-         TAOS_ARBITRATOR: arbitrator:6042
+    environment:
-       volumes:
+      TAOS_FQDN: "td-2"
-         - taosdata-td1:/var/lib/taos/
+      TAOS_FIRST_EP: "td-1"
-         - taoslog-td1:/var/log/taos/
+    volumes:
-     td-2:
+      - taosdata-td2:/var/lib/taos/
-       image: tdengine/tdengine:$VERSION
+      - taoslog-td2:/var/log/taos/
-       networks:
+  adapter:
-         - inter
+    image: tdengine/tdengine:$VERSION
-       environment:
+    entrypoint: "taosadapter"
-         TAOS_FQDN: "td-2"
+    networks:
-         TAOS_FIRST_EP: "td-1"
+      - inter
-         TAOS_NUM_OF_MNODES: "2"
+    environment:
-         TAOS_REPLICA: "2"
+      TAOS_FIRST_EP: "td-1"
-         TAOS_ARBITRATOR: arbitrator:6042
+      TAOS_SECOND_EP: "td-2"
-       volumes:
+    deploy:
-         - taosdata-td2:/var/lib/taos/
+      replicas: 4
-         - taoslog-td2:/var/log/taos/
+  nginx:
-     adapter:
+    image: nginx
-       image: tdengine/tdengine:$VERSION
+    depends_on:
-       command: taosadapter
+      - adapter
-       networks:
+    networks:
-         - inter
+      - inter
-       environment:
+    ports:
-         TAOS_FIRST_EP: "td-1"
+      - 6041:6041
-         TAOS_SECOND_EP: "td-2"
+      - 6044:6044/udp
-       deploy:
+    command: [
-         replicas: 4
+        "sh",
-     nginx:
+        "-c",
-       image: nginx
+        "while true;
-       depends_on:
+        do curl -s http://adapter:6041/-/ping >/dev/null && break;
-         - adapter
+        done;
-       networks:
+        printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}'
-         - inter
+        > /etc/nginx/conf.d/rest.conf;
-         - api
+        printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}'
-       ports:
+        >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf;
-         - 6041:6041
+        nginx -g 'daemon off;'",
-         - 6044:6044/udp
+      ]
-       command: [
+volumes:
-           "sh",
+  taosdata-td1:
-           "-c",
+  taoslog-td1:
-           "while true;
+  taosdata-td2:
-           do curl -s http://adapter:6041/-/ping >/dev/null && break;
+  taoslog-td2:
-           done;
+```
-           printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}'
-           > /etc/nginx/conf.d/rest.conf;
-           printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}'
-           >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf;
-           nginx -g 'daemon off;'",
-         ]
-   volumes:
-     taosdata-td1:
-     taoslog-td1:
-     taosdata-td2:
-     taoslog-td2:
-   ```
 
 ## Deploy with docker swarm
 

docs/en/25-application/_03-immigrate.md

@@ -184,7 +184,7 @@ TDengine supports the standard JDBC 3.0 interface for manipulating databases, bu
 
 To facilitate historical data migration, we provide a plug-in for the data synchronization tool DataX, which can automatically write data into TDengine.The automatic data migration of DataX can only support the data migration process of a single value model.
 
-For the specific usage of DataX and how to use DataX to write data to TDengine, please refer to [DataX-based TDengine Data Migration Tool](https://www.taosdata.com/blog/2021/10/26/3156.html).
+For the specific usage of DataX and how to use DataX to write data to TDengine, please refer to [DataX-based TDengine Data Migration Tool](https://www.taosdata.com/engineering/16401.html).
 
 After migrating via DataX, we found that we can significantly improve the efficiency of migrating historical data by starting multiple processes and migrating numerous metrics simultaneously. The following are some records of the migration process. We provide these as a reference for application migration.
 

docs/en/28-releases/01-tdengine.md

@@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w
 
 import Release from "/components/ReleaseV3";
 
+## 3.0.2.6
+
+<Release type="tdengine" version="3.0.2.6" />
+
 ## 3.0.2.5
 
 <Release type="tdengine" version="3.0.2.5" />

docs/en/28-releases/02-tools.md

@@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat
 
 import Release from "/components/ReleaseV3";
 
+## 2.4.6
+
+<Release type="tools" version="2.4.6" />
+
 ## 2.4.3
 
 <Release type="tools" version="2.4.3" />

docs/examples/java/pom.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
@@ -17,13 +18,13 @@
   </properties>
 
   <dependencies>
-<!--    ANCHOR: dep-->
+    <!--    ANCHOR: dep-->
     <dependency>
       <groupId>com.taosdata.jdbc</groupId>
       <artifactId>taos-jdbcdriver</artifactId>
-      <version>3.0.0</version>
+      <version>3.1.0</version>
     </dependency>
-<!--    ANCHOR_END: dep-->
+    <!--    ANCHOR_END: dep-->
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java

@@ -0,0 +1,79 @@
+package com.taos.example;
+
+import com.taosdata.jdbc.tmq.ConsumerRecords;
+import com.taosdata.jdbc.tmq.TMQConstants;
+import com.taosdata.jdbc.tmq.TaosConsumer;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.Properties;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class WebsocketSubscribeDemo {
+    private static final String TOPIC = "tmq_topic_ws";
+    private static final String DB_NAME = "meters_ws";
+    private static final AtomicBoolean shutdown = new AtomicBoolean(false);
+
+    public static void main(String[] args) {
+        Timer timer = new Timer();
+        timer.schedule(new TimerTask() {
+            public void run() {
+                shutdown.set(true);
+            }
+        }, 3_000);
+        try {
+            // prepare
+            Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
+            String jdbcUrl = "jdbc:TAOS-RS://127.0.0.1:6041/?user=root&password=taosdata&batchfetch=true";
+            try (Connection connection = DriverManager.getConnection(jdbcUrl);
+                    Statement statement = connection.createStatement()) {
+                statement.executeUpdate("drop topic if exists " + TOPIC);
+                statement.executeUpdate("drop database if exists " + DB_NAME);
+                statement.executeUpdate("create database " + DB_NAME);
+                statement.executeUpdate("use " + DB_NAME);
+                statement.executeUpdate(
+                        "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` INT, `location` BINARY(24))");
+                statement.executeUpdate("CREATE TABLE `d0` USING `meters` TAGS(0, 'California.LosAngles')");
+                statement.executeUpdate("INSERT INTO `d0` values(now - 10s, 0.32, 116)");
+                statement.executeUpdate("INSERT INTO `d0` values(now - 8s, NULL, NULL)");
+                statement.executeUpdate(
+                        "INSERT INTO `d1` USING `meters` TAGS(1, 'California.SanFrancisco') values(now - 9s, 10.1, 119)");
+                statement.executeUpdate(
+                        "INSERT INTO `d1` values (now-8s, 10, 120) (now - 6s, 10, 119) (now - 4s, 11.2, 118)");
+                // create topic
+                statement.executeUpdate("create topic " + TOPIC + " as select * from meters");
+            }
+
+            // create consumer
+            Properties properties = new Properties();
+            properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6041");
+            properties.setProperty(TMQConstants.CONNECT_TYPE, "ws");
+            properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true");
+            properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true");
+            properties.setProperty(TMQConstants.GROUP_ID, "test");
+            properties.setProperty(TMQConstants.VALUE_DESERIALIZER,
+                    "com.taos.example.MetersDeserializer");
+
+            // poll data
+            try (TaosConsumer<Meters> consumer = new TaosConsumer<>(properties)) {
+                consumer.subscribe(Collections.singletonList(TOPIC));
+                while (!shutdown.get()) {
+                    ConsumerRecords<Meters> meters = consumer.poll(Duration.ofMillis(100));
+                    for (Meters meter : meters) {
+                        System.out.println(meter);
+                    }
+                }
+                consumer.unsubscribe();
+            }
+        } catch (ClassNotFoundException | SQLException e) {
+            e.printStackTrace();
+        }
+        timer.cancel();
+    }
+}

docs/examples/java/src/test/java/com/taos/test/TestAll.java

@@ -64,21 +64,15 @@ public class TestAll {
 
     @Test
     public void testSubscribe() {
-
-        Thread thread = new Thread(() -> {
-            try {
-                Thread.sleep(1000);
-                insertData();
-            } catch (SQLException e) {
-                e.printStackTrace();
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        });
-        thread.start();
         SubscribeDemo.main(args);
     }
 
+
+    @Test
+    public void testSubscribeOverWebsocket() {
+        WebsocketSubscribeDemo.main(args);
+    }
+
     @Test
     public void testSchemaless() throws SQLException {
         LineProtocolExample.main(args);

docs/zh/07-develop/_sub_java.mdx

@@ -1,3 +1,9 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+<Tabs defaultValue="native">
+<TabItem value="native" label="本地连接">
+
 ```java
 {{#include docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java}}
 ```
@@ -7,3 +13,19 @@
 ```java
 {{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}}
 ```
+
+</TabItem>
+<TabItem value="ws" label="WebSocket 连接">
+
+```java
+{{#include docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java}}
+```
+```java
+{{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}}
+```
+```java
+{{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}}
+```
+
+</TabItem>
+</Tabs>

docs/zh/08-connector/14-java.mdx

@@ -699,7 +699,10 @@ TaosConsumer consumer = new TaosConsumer<>(config);
 - enable.auto.commit: 是否允许自动提交。
 - group.id: consumer: 所在的 group。
 - value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。
-- 其他参数请参考：[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group)
+- td.connect.type: 连接方式。jni：表示使用动态库连接的方式，ws/WebSocket：表示使用 WebSocket 进行数据通信。默认为 jni 方式。
+- httpConnectTimeout：创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。
+- messageWaitTimeout：数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。
+其他参数请参考：[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group)
 
 #### 订阅消费数据
 
@@ -727,6 +730,9 @@ consumer.close()
 
 ### 使用示例如下：
 
+<Tabs defaultValue="native">
+<TabItem value="native" label="原生连接">
+
 ```java
 public abstract class ConsumerLoop {
     private final TaosConsumer<ResultBean> consumer;
@@ -798,6 +804,89 @@ public abstract class ConsumerLoop {
 }
 ```
 
+</TabItem>
+<TabItem value="ws" label="WebSocket 连接">
+
+除了原生的连接方式，Java 连接器还支持通过 WebSocket 订阅数据。
+
+```java
+public abstract class ConsumerLoop {
+    private final TaosConsumer<ResultBean> consumer;
+    private final List<String> topics;
+    private final AtomicBoolean shutdown;
+    private final CountDownLatch shutdownLatch;
+
+    public ConsumerLoop() throws SQLException {
+        Properties config = new Properties();
+        config.setProperty("bootstrap.servers", "localhost:6041");
+        config.setProperty("td.connect.type", "ws");
+        config.setProperty("msg.with.table.name", "true");
+        config.setProperty("enable.auto.commit", "true");
+        config.setProperty("group.id", "group2");
+        config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+
+        this.consumer = new TaosConsumer<>(config);
+        this.topics = Collections.singletonList("topic_speed");
+        this.shutdown = new AtomicBoolean(false);
+        this.shutdownLatch = new CountDownLatch(1);
+    }
+
+    public abstract void process(ResultBean result);
+
+    public void pollData() throws SQLException {
+        try {
+            consumer.subscribe(topics);
+
+            while (!shutdown.get()) {
+                ConsumerRecords<ResultBean> records = consumer.poll(Duration.ofMillis(100));
+                for (ResultBean record : records) {
+                    process(record);
+                }
+            }
+            consumer.unsubscribe();
+        } finally {
+            consumer.close();
+            shutdownLatch.countDown();
+        }
+    }
+
+    public void shutdown() throws InterruptedException {
+        shutdown.set(true);
+        shutdownLatch.await();
+    }
+
+    public static class ResultDeserializer extends ReferenceDeserializer<ResultBean> {
+
+    }
+
+    public static class ResultBean {
+        private Timestamp ts;
+        private int speed;
+
+        public Timestamp getTs() {
+            return ts;
+        }
+
+        public void setTs(Timestamp ts) {
+            this.ts = ts;
+        }
+
+        public int getSpeed() {
+            return speed;
+        }
+
+        public void setSpeed(int speed) {
+            this.speed = speed;
+        }
+    }
+}
+```
+
+</TabItem>
+</Tabs>
+
+> **注意**：这里的 value.deserializer 配置参数值应该根据测试环境的包路径做相应的调整。
+
 ### 与连接池使用
 
 #### HikariCP
@@ -881,8 +970,8 @@ public static void main(String[] args) throws Exception {
 
 | taos-jdbcdriver 版本 |            主要变化            |
 | :------------------: | :----------------------------: |
-|       3.0.3        |  修复 REST 连接在 jdk17+ 版本时间戳解析错误问题    |
+|       3.1.0           |   WebSocket 连接支持订阅功能      |
-|       3.0.1 - 3.0.2        |  修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用 3.0.2 版本    |
+|       3.0.1 - 3.0.4        |  修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用其他版本    |
 |       3.0.0        |   支持 TDengine 3.0    |
 |       2.0.42        |   修在 WebSocket 连接中 wasNull 接口返回值  |
 |       2.0.41        |   修正 REST 连接中用户名和密码转码方式    |

docs/zh/12-taos-sql/10-function.md

@@ -798,18 +798,22 @@ HISTOGRAM(expr，bin_type, bin_description, normalized)
 ### PERCENTILE
 
 ```sql
-PERCENTILE(expr, p)
+PERCENTILE(expr, p [, p1] ... )
 ```
 
 **功能说明**：统计表中某列的值百分比分位数。
 
-**返回数据类型**： DOUBLE。
+**返回数据类型**： 该函数最小参数个数为 2 个，最大参数个数为 11 个。可以最多同时返回 10 个百分比分位数。当参数个数为 2 时， 返回一个分位数， 类型为DOUBLE，当参数个数大于 2 时，返回类型为VARCHAR, 格式为包含多个返回值的JSON数组。
 
 **应用字段**：数值类型。
 
 **适用于**：表。
 
-**使用说明**：*P*值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX。
+**使用说明**：
+
+- *P*值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX;
+- 同时计算针对同一列的多个分位数时，建议使用一个PERCENTILE函数和多个参数的方式，能很大程度上降低查询的响应时间。
+  比如，使用查询SELECT percentile(col, 90, 95, 99) FROM table, 性能会优于SELECT percentile(col, 90), percentile(col, 95), percentile(col, 99) from table。
 
 
 ## 选择函数

docs/zh/12-taos-sql/24-show.md

@@ -191,37 +191,39 @@ SHOW TABLE DISTRIBUTED table_name;
 
 _block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
 
-Total_Blocks :  表d0 占用的 block 个数为 5 个
+Total_Blocks:  表 d0 占用的 block 个数为 5 个
 
-Total_Size.    :  表 d0 所有 block 在文件中占用的大小为 93.65 KB 
+Total_Size:    表 d0 所有 block 在文件中占用的大小为 93.65 KB 
 
 Average_size:  平均每个 block 在文件中占用的空间大小为 18.73 KB
 
-Compression_Ratio: 数据压缩率为 23.98%
+Compression_Ratio: 数据压缩率 23.98%
 
  
 *************************** 2.row ***************************
 
 _block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000]
 
-Total_Rows: 统计表 d0 的所有行数 为20000 行
+Total_Rows:  统计表 d0 的所有行数 为20000 行（该数值仅供参考，不是精确的行数。获得精确的行数需要使用 count 函数）
 
-Inmem_Rows： 表示仍然还存放在内存中的行数，即没有落盘的行数，为 0行，表示没有
+Inmem_Rows： 存储在写缓存中的数据行数（没有落盘），0 行表示内存缓存中没有数据
 
-MinRows：  BLOCK 中最小的行数，为 3616 行
+MinRows：    BLOCK 中最小的行数，为 3616 行
 
-MaxRows： BLOCK 中最大的行数，为 4096行
+MaxRows：    BLOCK 中最大的行数，为 4096 行
 
-Average_Rows： BLOCK 中的平均行数，为4000 行
+Average_Rows： 每个 BLOCK 中的平均行数，此时为 4000 行
 
 
 *************************** 3.row ***************************
 
-_block_dist: Total_Tables=[1] Total_Files=[2]
+_block_dist: Total_Tables=[1] Total_Files=[2] Total_Vgroups=[1]
 
-Total_Tables:  表示子表的个数，这里为1
+Total_Tables:   子表的个数，这里为 1
 
-Total_Files：   表数据保存在几个文件中，这里保存在 2 个文件中
+Total_Files：   表数据被分别保存的数据文件数量，这里是 2 个文件
+
+Total_Vgroups： 表数据分布的虚拟节点（vnode）数量
 
 
 *************************** 5.row ***************************

docs/zh/25-application/03-immigrate.md

@@ -18,82 +18,15 @@ title: OpenTSDB 应用迁移到 TDengine 的最佳实践
 
 如果我们将原本运行在 OpenTSDB 上的应用迁移到 TDengine 上，不仅可以有效地降低计算和存储资源的占用、减少部署服务器的规模，还能够极大减少运行维护的成本的输出，让运维管理工作更简单、更轻松，大幅降低总拥有成本。与 OpenTSDB 一样，TDengine 也已经进行了开源，不同的是，除了单机版，后者还实现了集群版开源，被厂商绑定的顾虑一扫而空。
 
-在下文中我们将就“使用最典型并广泛应用的运维监控（DevOps）场景”来说明，如何在不编码的情况下将 OpenTSDB 的应用快速、安全、可靠地迁移到 TDengine 之上。后续的章节会做更深度的介绍，以便于进行非 DevOps 场景的迁移。
+在下文中我们将说明如何在不编码的情况下将 OpenTSDB 的应用快速、安全、可靠地迁移到 TDengine 之上。
 
-## DevOps 应用快速迁移
+## TDengine 与 OpenTSDB 的差异
 
-### 1、典型应用场景
+本节将详细介绍 OpenTSDB 与 TDengine 在系统功能层面上存在的差异。阅读完本节的内容，你可以全面地评估是否能够将某些基于 OpenTSDB 的复杂应用迁移到 TDengine 上，以及迁移之后应该注意的问题。
-
-一个典型的 DevOps 应用场景的系统整体的架构如下图（图 1） 所示。
-
-**图 1. DevOps 场景中典型架构**
-![TDengine Database IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch](./IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.webp "图1. DevOps 场景中典型架构")
-
-在该应用场景中，包含了部署在应用环境中负责收集机器度量（Metrics）、网络度量（Metrics）以及应用度量（Metrics）的 Agent 工具、汇聚 Agent 收集信息的数据收集器，数据持久化存储和管理的系统以及监控数据可视化工具（例如：Grafana 等）。
-
-其中，部署在应用节点的 Agents 负责向 collectd/Statsd 提供不同来源的运行指标，collectd/StatsD 则负责将汇聚的数据推送到 OpenTSDB 集群系统，然后使用可视化看板 Grafana 将数据可视化呈现出来。
-
-### 2、迁移服务
-
-- **TDengine 安装部署**
-
-首先是 TDengine 的安装，从官网上下载 TDengine 最新稳定版进行安装。各种安装包的使用帮助请参见博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。
-
-注意，安装完成以后，不要立即启动 `taosd` 服务，在正确配置完成参数以后再启动。
-
-- **调整数据收集器配置**
-
-在 TDengine 2.4 版本中，包含一个组件 taosAdapter。taosAdapter 是一个无状态、可快速弹性伸缩的组件，它可以兼容 Influxdb 的 Line Protocol 和 OpenTSDB 的 telnet/JSON 写入协议规范，提供了丰富的数据接入能力，有效的节省用户迁移成本，降低用户应用迁移的难度。
-
-用户可以根据需求弹性部署 taosAdapter 实例，结合场景的需要，快速提升数据写入的吞吐量，为不同应用场景下的数据写入提供保障。
-
-通过 taosAdapter，用户可以将 collectd 或 StatsD 收集的数据直接推送到 TDengine ，实现应用场景的无缝迁移，非常的轻松便捷。taosAdapter 还支持 Telegraf、Icinga、TCollector 、node_exporter 的数据接入，使用详情参考[taosAdapter](/reference/taosadapter/)。
-
-如果使用 collectd，修改其默认位置 `/etc/collectd/collectd.conf` 的配置文件为指向 taosAdapter 部署的节点 IP 地址和端口。假设 taosAdapter 的 IP 地址为 192.168.1.130，端口为 6046，配置如下：
-
-```html
-LoadPlugin write_tsdb
-<Plugin write_tsdb>
-  <Node>
-    Host "192.168.1.130" Port "6046" HostTags "status=production" StoreRates
-    false AlwaysAppendDS false
-  </Node>
-</Plugin>
-```
-
-即可让 collectd 将数据使用推送到 OpenTSDB 的插件方式推送到 taosAdapter， taosAdapter 将调用 API 将数据写入到 TDengine 中，从而完成数据的写入工作。如果你使用的是 StatsD 相应地调整配置文件信息。
-
-- **调整看板（Dashboard）系统**
-
-在数据能够正常写入 TDengine 后，可以调整适配 Grafana 将写入 TDengine 的数据可视化呈现出来。获取和使用 TDengine 提供的 Grafana 插件请参考[与其他工具的连接](/third-party/grafana)。
-
-TDengine 提供了默认的两套 Dashboard 模板，用户只需要将 Grafana 目录下的模板导入到 Grafana 中即可激活使用。
-
-**图 2. 导入 Grafana 模板**
-![TDengine Database IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard](./IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.webp "图2. 导入 Grafana 模板")
-
-操作完以上步骤后，就完成了将 OpenTSDB 替换成为 TDengine 的迁移工作。可以看到整个流程非常简单，不需要写代码，只需要对某些配置文件进行调整即可完成全部的迁移工作。
-
-### 3、迁移后架构
-
-完成迁移以后，此时的系统整体的架构如下图（图 3）所示，而整个过程中采集端、数据写入端、以及监控呈现端均保持了稳定，除了极少的配置调整外，不涉及任何重要的更改和变动。OpenTSDB 大量的应用场景均为 DevOps ，这种场景下，简单的参数设置即可完成 OpenTSDB 到 TDengine 迁移动作，使用上 TDengine 更加强大的处理能力和查询性能。
-
-在绝大多数的 DevOps 场景中，如果你拥有一个小规模的 OpenTSDB 集群（3 台及以下的节点）作为 DevOps 的存储端，依赖于 OpenTSDB 为系统持久化层提供数据存储和查询功能，那么你可以安全地将其替换为 TDengine，并节约更多的计算和存储资源。在同等计算资源配置情况下，单台 TDengine 即可满足 3 ~ 5 台 OpenTSDB 节点提供的服务能力。如果规模比较大，那便需要采用 TDengine 集群。
-
-如果你的应用特别复杂，或者应用领域并不是 DevOps 场景，你可以继续阅读后续的章节，更加全面深入地了解将 OpenTSDB 的应用迁移到 TDengine 的高级话题。
-
-**图 3. 迁移完成后的系统架构**
-![TDengine Database IT-DevOps-Solutions-Immigrate-TDengine-Arch](./IT-DevOps-Solutions-Immigrate-TDengine-Arch.webp "图 3. 迁移完成后的系统架构")
-
-## 其他场景的迁移评估与策略
-
-### 1、TDengine 与 OpenTSDB 的差异
-
-本章将详细介绍 OpenTSDB 与 TDengine 在系统功能层面上存在的差异。阅读完本章的内容，你可以全面地评估是否能够将某些基于 OpenTSDB 的复杂应用迁移到 TDengine 上，以及迁移之后应该注意的问题。
 
 TDengine 当前只支持 Grafana 的可视化看板呈现，所以如果你的应用中使用了 Grafana 以外的前端看板（例如[TSDash](https://github.com/facebook/tsdash)、[Status Wolf](https://github.com/box/StatusWolf)等），那么前端看板将无法直接迁移到 TDengine，需要将前端看板重新适配到 Grafana 才可以正常运行。
 
-在 2.3.0.x 版本中，TDengine 只能够支持 collectd 和 StatsD 作为数据收集汇聚软件，当然后面会陆续提供更多的数据收集聚合软件的接入支持。如果您的收集端使用了其他类型的数据汇聚器，您的应用需要适配到这两个数据汇聚端系统，才能够将数据正常写入。除了上述两个数据汇聚端软件协议以外，TDengine 还支持通过 InfluxDB 的行协议和 OpenTSDB 的数据写入协议、JSON 格式将数据直接写入，您可以重写数据推送端的逻辑，使用 TDengine 支持的行协议来写入数据。
+如果您的收集端使用了像 collectd 和 StatsD 这样的数据采集工具，要重新配置这些数据采集工具将数据写入到 TDengine。TDengine 还支持通过 InfluxDB 的行协议和 OpenTSDB 的数据写入协议、JSON 格式将数据直接写入，您可以重写数据推送端的逻辑，使用 TDengine 支持的行协议来写入数据。
 
 此外，如果你的应用中使用了 OpenTSDB 以下特性，在将应用迁移到 TDengine 之前你还需要了解以下注意事项：
 
@@ -104,11 +37,11 @@ TDengine 当前只支持 Grafana 的可视化看板呈现，所以如果你的
 
 通过上面的介绍，相信你应该能够了解 OpenTSDB 迁移到 TDengine 带来的变化，这些信息也有助于你正确地判断是否可以接受将应用 迁移到 TDengine 之上，体验 TDengine 提供的强大的时序数据处理能力和便捷的使用体验。
 
-### 2、迁移策略
+## 迁移策略
 
 首先将基于 OpenTSDB 的系统进行迁移涉及到的数据模式设计、系统规模估算、数据写入端改造，进行数据分流、应用适配工作；之后将两个系统并行运行一段时间，再将历史数据迁移到 TDengine 中。当然如果你的应用中有部分功能强依赖于上述 OpenTSDB 特性，同时又不希望停止使用，可以考虑保持原有的 OpenTSDB 系统运行，同时启动 TDengine 来提供主要的服务。
 
-## 数据模型设计
+### 数据模型设计
 
 一方面，TDengine 要求其入库的数据具有严格的模式定义。另一方面，TDengine 的数据模型相对于 OpenTSDB 来说又更加丰富，多值模型能够兼容全部的单值模型的建立需求。
 
@@ -150,7 +83,7 @@ insert into memory_vm130_memory_buffered_collectd  using memory tags(‘vm130’
 
 如果你想要利用 TDengine 的多值模型能力，需要首先满足以下要求：不同的采集量具有相同的采集频率，且能够通过消息队列**同时到达**数据写入端，从而确保使用 SQL 语句将多个指标一次性写入。将度量的名称作为超级表的名称，建立具有相同采集频率且能够同时到达的数据多列模型。子表的表名采用具有固定规则的方式进行命名。上述每个度量均只包含一个测量值，因此无法将其转化为多值模型。
 
-## 数据分流与应用适配
+### 数据分流与应用适配
 
 从消息队列中订阅数据，并启动调整后的写入程序写入数据。
 
@@ -166,15 +99,128 @@ TDengine 不支持采用 OpenTSDB 的查询语法进行查询或数据获取处
 
 TDengine 支持标准的 JDBC 3.0 接口操纵数据库，你也可以使用其他类型的高级语言的连接器来查询读取数据，以适配你的应用。具体的操作和使用帮助也请参阅用户手册。
 
-## 历史数据迁移
 
-### 1、使用工具自动迁移数据
+## 使用 DataX 迁移数据
 
-为了方便历史数据的迁移工作，我们为数据同步工具 DataX 提供了插件，能够将数据自动写入到 TDengine 中，需要注意的是 DataX 的自动化数据迁移只能够支持单值模型的数据迁移过程。
+为了方便历史数据的迁移工作，我们为数据同步工具 DataX 提供了适配 TDengine 3.0 的插件，能够将数据自动写入到 TDengine 中，需要注意的是 DataX 的自动化数据迁移只能够支持单值模型的数据迁移过程。
 
-DataX 具体的使用方式及如何使用 DataX 将数据写入 TDengine 请参见[基于 DataX 的 TDengine 数据迁移工具](https://www.taosdata.com/blog/2021/10/26/3156.html)。
+### 安装和部署 TDengine 
 
-在对 DataX 进行迁移实践后，我们发现通过启动多个进程，同时迁移多个 metric 的方式，可以大幅度的提高迁移历史数据的效率，下面是迁移过程中的部分记录，希望这些能为应用迁移工作带来参考。
+在进行数据迁移之前，要有一个正确运行的 TDengine 集群。首先是 TDengine 的安装，从官网上下载 TDengine 最新稳定版进行安装。各种安装包的使用帮助请参考 [安装指南](../../get-started/package)
+
+安装完成后，请根据 [部署指南](../../deployment/deploy) 配置集群。
+
+### 插件功能介绍
+
+1. TDengine30Reader 提供的功能：
+   1. 支持通过 SQL 进行数据筛选；
+   2. 根据时间间隔进行任务切分；
+   3. 支持 TDengine 的全部数据类型；
+   4. 支持批量读取，通过 batchSize 参数控制批量拉取结果集的大小，提高读取性能。
+2. TDengine30Writer 支持的功能：
+   1. 支持 OpenTSDB 的 json 格式的行协议，使用 TDengine 的 schemaless 方式写入 TDengine。
+   2. 支持批量写入，通过 batchSize 参数控制批量写入的数量，提高写入性能。
+
+### DataX 安装环境准备
+
+1. 需要安装 TDengine 客户端
+2. 需要安装 JDK 1.8 环境（运行 DataX）
+3. 需要安装 Python 环境（运行 DataX）
+4. 需要 maven 编译环境（如果不编译 DataX 则可以不安装 maven）
+
+### 安装
+
+1. 下载源码
+~~~
+git clone https://github.com/taosdata/DataX.git
+~~~
+2. 编译打包
+~~~
+cd DataX
+mvn -U clean package assembly:assembly -Dmaven.test.skip=true
+~~~
+3. 安装
+~~~
+cp target/datax.tar.gz your_install_dir
+cd your_install_dir
+tar -zxvf dataX.tar.gz
+~~~
+
+### 数据迁移 Job 的配置
+
+以一个从 OpenTSDB 到 TDengine 3.0 版本的数据迁移任务为例，配置文件 opentsdb2tdengine.json 如下：
+~~~
+{
+   "job":{
+     "content":[{
+       "reader": {
+         "name": "opentsdbreader",
+         "parameter": {
+           "endpoint": "http://192.168.1.180:4242",
+           "column": ["weather_temperature"],
+           "beginDateTime": "2021-01-01 00:00:00",
+           "endDateTime": "2021-01-01 01:00:00"
+         }
+       },
+     "writer": {
+       "name": "tdengine30writer",
+       "parameter": {
+            "username": "root",
+            "password": "taosdata",
+            "connection": [
+              {
+                "table": [
+                  "matric1"
+                ],
+                "jdbcUrl": "jdbc:TAOS://192.168.1.101:6030/test?timestampFormat=TIMESTAMP"
+              }
+            ],
+            "batchSize": 1000,
+            "ignoreTagsUnmatched": true
+          }
+       }
+     }],
+     "setting": {
+       "speed": {
+         "channel": 1
+       }
+     }
+   }
+ } 
+~~~
+配置说明：
+1. 上面的配置表示，从 192.168.1.180 的 OpenTSDB，到 192.168.1.101 的 TDengine 的迁移。迁移 metric 为 weather_temperature，时间从 2021-01-01 00:00:00 开始，到 2021-01-01 01:00:00 结束的数据。
+2. reader 使用 datax 的 opentsdbreader，parameter 的配置请参考：[opentsdbreader.md#配置参数](https://github.com/taosdata/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md)
+3. tdengine30writer 的 parameter 中，user，password 为必须项，没有默认值。batchSize 不是必须项，默认值为 1。详细参考：[tdengine30writer.md#配置参数](https://github.com/taosdata/DataX/blob/master/tdengine30writer/doc/tdengine30writer-CN.md)
+4. TDengine 中，如果 dbname 指定的 database 不存在，则需要在迁移前创建数据库。
+
+### 执行迁移任务
+
+~~~
+python bin/datax.py job/opentsdb2tdengine.json
+~~~
+
+### 限制条件
+
+1. 目前，DataX 自带的 opentsdbreader 仅支持 OpenTSDB-2.3.X 版本。详细参考：[opentsdbreader#约束限制](https://github.com/alibaba/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md#5-%E7%BA%A6%E6%9D%9F%E9%99%90%E5%88%B6)
+2. 数据迁移工具依赖 TDengine 客户端中的 `libtaos.so/taos.dll/libtaos.dylib`，需要与服务端对应版本的 TDengine-client。
+
+### FAQ
+
+1. 如何估算一个数据迁移任务所需要的资源
+   DataX 的每个 reader 按照自己的 task 切分策略进行任务划分，具体请参考 DataX 的任务调度规则。在估算资源是，需要按照数据迁移的数据量，任务切分规则和网络带宽限制等综合考虑，最好以实际数据迁移测试结果为准。
+2. TDengine30Writer 的 batchSize 设置多大效率最高？
+   batchSize 是控制批量写入的参数，在获取 batchSize 行纪录后，TDengineWriter 会向 TDengine 发送一次写入请求，这减少了与 TDengine 交互次数，从而提高了性能。从测试结果来看，batchSize 在 500-1000 范围内效率最高。
+3. job 的配置中 channel 数为多少合适？
+   job 中的 channel 数为流量控制的参数，每个 channel 都需要开辟一块内存，用来缓存数据。如果 channel 设置过大，会引起 OOM，所以 channel 数并不是越大越好。增加 channel 数后，需要提高 JVM 内存大小。从测试结果来看，channel 在 1～6 的范围内都是合适，能够保证 DataX 的流量最大化即可。
+4. java.sql.SQLException: TDengine ERROR (8000060b): Invalid client value
+   配置文件中 column 中没有配置 tbname，此时会触发行协议数据写入（行协议写入只会自动创建子表名，但需要提前创建好超级表），行协议写入的情况下不支持 TAG 数据类型为非 NCHAR，所以此种情况有两种解决方案：1.将 TAG 全部修改为 NCHAR 类型；2.在 Column 中配置好表名称这样不会触发行协议写入。
+5. java.sql.SQLException: TDengine ERROR (8000060b): Timestamp data out of range
+   配置文件中 column 中没有配置 tbname，此时会触发行协议数据写入，且 TAG 全部为 NCHAR 类型，此时需要保证时间戳的一列名称为 _ts，而不能是其他名称（行协议写入下，默认将最后的时间戳写入到 _ts 一列，且不能随意命名）。若想避免请使用 tbname 指定表名以避免触发行协议写入。
+
+### 提升性能
+
+ 在对 DataX 进行迁移实践后，我们发现通过启动多个进程，同时迁移多个 metric 的方式，可以大幅度的提高迁移历史数据的效率，下面是迁移过程中的部分记录，希望这些能为应用迁移工作带来参考。
 
 | DataX 实例个数 (并发进程个数) | 迁移记录速度 （条/秒) |
 | ----------------------------- | --------------------- |
@@ -184,9 +230,9 @@ DataX 具体的使用方式及如何使用 DataX 将数据写入 TDengine 请参
 | 5                             | 约 29.5 万            |
 | 10                            | 约 33 万              |
 
-<br/>（注：测试数据源自 单节点 Intel(R) Core(TM) i7-10700 CPU@2.90GHz 16 核 64G 硬件设备，channel 和 batchSize 分别为 8 和 1000，每条记录包含 10 个 tag)
+（注：测试数据源自 单节点 Intel(R) Core(TM) i7-10700 CPU@2.90GHz 16 核 64G 硬件设备，channel 和 batchSize 分别为 8 和 1000，每条记录包含 10 个 tag)
 
-### 2、手动迁移数据
+## 手动迁移数据
 
 如果你需要使用多值模型进行数据写入，就需要自行开发一个将数据从 OpenTSDB 导出的工具，然后确认哪些时间线能够合并导入到同一个时间线，再将可以同时导入的时间通过 SQL 语句的写入到数据库中。
 
@@ -393,31 +439,11 @@ WHERE ts>=1510560000 AND ts<=1515000009
 
 综上所述，可使用单台 16 核 32GB 的机器，或者使用 2 台 8 核 16GB 机器构成的集群。
 
-## 附录 3: 集群部署及启动
+## 附录 3: 超级表名称
-
-TDengine 提供了丰富的帮助文档说明集群安装、部署的诸多方面的内容，这里提供相应的文档列表，供你参考。
-
-### 集群部署
-
-首先是安装 TDengine，从官网上下载 TDengine 最新稳定版，解压缩后运行 install.sh 进行安装。各种安装包的使用帮助请参见博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。
-
-注意安装完成以后，不要立即启动 `taosd` 服务，在正确配置完成参数以后才启动 `taosd` 服务。
-
-### 设置运行参数并启动服务
-
-为确保系统能够正常获取运行的必要信息。请在服务端正确设置以下关键参数：
-
-FQDN、firstEp、secondEP、dataDir、logDir、tmpDir、serverPort。各参数的具体含义及设置的要求，可参见文档《[TDengine 集群安装、管理](/cluster/)》
-
-按照相同的步骤，在需要运行的节点上设置参数，并启动 `taosd` 服务，然后添加 Dnode 到集群中。
-
-最后启动 `taos` 命令行程序，执行命令 `show dnodes`，如果能看到所有的加入集群的节点，那么集群顺利搭建完成。具体的操作流程及注意事项，请参阅文档《[TDengine 集群安装、管理](/cluster/)》
-
-## 附录 4: 超级表名称
 
 由于 OpenTSDB 的 metric 名称中带有点号（“.”），例如“cpu.usage_user”这种名称的 metric。但是点号在 TDengine 中具有特殊含义，是用来分隔数据库和表名称的分隔符。TDengine 也提供转义符，以允许用户在（超级）表名称中使用关键词或特殊分隔符（如：点号）。为了使用特殊字符，需要采用转义字符将表的名称括起来，例如：`cpu.usage_user`这样就是合法的（超级）表名称。
 
-## 附录 5：参考文章
+## 附录 4：参考文章
 
-1. [使用 TDengine + collectd/StatsD + Grafana 快速搭建 IT 运维监控系统](/application/collectd/)
+1. [使用 TDengine + collectd/StatsD + Grafana 快速搭建 IT 运维监控系统](../collectd/)
-2. [通过 collectd 将采集数据直接写入 TDengine](/third-party/collectd/)
+2. [通过 collectd 将采集数据直接写入 TDengine](../../third-party/collectd/)

docs/zh/28-releases/01-tdengine.md

@@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
 
 import Release from "/components/ReleaseV3";
 
+## 3.0.2.6
+
+<Release type="tdengine" version="3.0.2.6" />
+
 ## 3.0.2.5
 
 <Release type="tdengine" version="3.0.2.5" />

docs/zh/28-releases/02-tools.md

@@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下：
 
 import Release from "/components/ReleaseV3";
 
+## 2.4.6
+
+<Release type="tools" version="2.4.6" />
+
 ## 2.4.3
 
 <Release type="tools" version="2.4.3" />

examples/c/asyncdemo.c

@@ -92,7 +92,7 @@ int main(int argc, char *argv[])
   }
 
   // a simple way to parse input parameters
-  if (argc >= 3) strcpy(db, argv[2]);
+  if (argc >= 3) strncpy(db, argv[2], sizeof(db) - 1);
   if (argc >= 4) points = atoi(argv[3]);
   if (argc >= 5) numOfTables = atoi(argv[4]);
 

examples/c/tmq.c

@@ -61,7 +61,7 @@ static int32_t init_env() {
   printf("create database\n");
   pRes = taos_query(pConn, "drop topic topicname");
   if (taos_errno(pRes) != 0) {
-    printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes));
+    printf("error in drop topicname, reason:%s\n", taos_errstr(pRes));
   }
   taos_free_result(pRes);
 
@@ -189,27 +189,54 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
 
 tmq_t* build_consumer() {
   tmq_conf_res_t code;
-  tmq_conf_t*    conf = tmq_conf_new();
+  tmq_t*         tmq = NULL;
+
+  tmq_conf_t* conf = tmq_conf_new();
   code = tmq_conf_set(conf, "enable.auto.commit", "true");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
   code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
   code = tmq_conf_set(conf, "group.id", "cgrpName");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
   code = tmq_conf_set(conf, "client.id", "user defined name");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
   code = tmq_conf_set(conf, "td.connect.user", "root");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
   code = tmq_conf_set(conf, "td.connect.pass", "taosdata");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
   code = tmq_conf_set(conf, "auto.offset.reset", "earliest");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
   code = tmq_conf_set(conf, "experimental.snapshot.enable", "false");
-  if (TMQ_CONF_OK != code) return NULL;
+  if (TMQ_CONF_OK != code) {
+    tmq_conf_destroy(conf);
+    return NULL;
+  }
 
   tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
+  tmq = tmq_consumer_new(conf, NULL, 0);
 
-  tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
+_end:
   tmq_conf_destroy(conf);
   return tmq;
 }

include/common/systable.h

@@ -13,15 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
  
+#ifndef TDENGINE_SYSTABLE_H
+#define TDENGINE_SYSTABLE_H
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #include "os.h"
 
-#ifndef TDENGINE_SYSTABLE_H
-#define TDENGINE_SYSTABLE_H
-
 #define TSDB_INFORMATION_SCHEMA_DB       "information_schema"
 #define TSDB_INS_TABLE_DNODES            "ins_dnodes"
 #define TSDB_INS_TABLE_MNODES            "ins_mnodes"

include/common/tcommon.h

@@ -205,7 +205,7 @@ typedef struct SDataBlockInfo {
   STimeWindow calWin;     // used for stream, do not serialize
   TSKEY       watermark;  // used for stream
 
-  char    parTbName[TSDB_TABLE_NAME_LEN];  // used for stream partition
+  char parTbName[TSDB_TABLE_NAME_LEN];  // used for stream partition
 } SDataBlockInfo;
 
 typedef struct SSDataBlock {
@@ -291,7 +291,6 @@ typedef struct STableBlockDistInfo {
   uint16_t numOfFiles;
   uint32_t numOfTables;
   uint32_t numOfBlocks;
-  uint32_t numOfVgroups;
   uint64_t totalSize;
   uint64_t totalRows;
   int32_t  maxRows;
@@ -301,6 +300,7 @@ typedef struct STableBlockDistInfo {
   int32_t  firstSeekTimeUs;
   uint32_t numOfInmemRows;
   uint32_t numOfSmallBlocks;
+  uint32_t numOfVgroups;
   int32_t  blockRowsHisto[20];
 } STableBlockDistInfo;
 
@@ -341,7 +341,7 @@ typedef struct SExprInfo {
 
 typedef struct {
   const char* key;
-  size_t     keyLen;
+  size_t      keyLen;
   uint8_t     type;
   union {
     const char* value;
@@ -385,9 +385,9 @@ typedef struct STUidTagInfo {
 #define TABLE_NAME_COLUMN_INDEX         6
 
 // stream create table block column
-#define UD_TABLE_NAME_COLUMN_INDEX     0
+#define UD_TABLE_NAME_COLUMN_INDEX 0
-#define UD_GROUPID_COLUMN_INDEX        1
+#define UD_GROUPID_COLUMN_INDEX    1
-#define UD_TAG_COLUMN_INDEX            2
+#define UD_TAG_COLUMN_INDEX        2
 
 #ifdef __cplusplus
 }

include/common/tdatablock.h

@@ -58,7 +58,6 @@ typedef struct SBlockOrderInfo {
 #define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
 
 #define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
-
 #define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
 // SColumnInfoData, rowNumber
 #define colDataGetData(p1_, r_) \
@@ -83,33 +82,6 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData,
   }
 }
 
-static FORCE_INLINE bool colDataIsNNull_s(const SColumnInfoData* pColumnInfoData, int32_t startIndex,
-                                          uint32_t nRows) {
-  if (!pColumnInfoData->hasNull) {
-    return false;
-  }
-
-  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    for (int32_t i = startIndex; i < nRows; ++i) {
-      if (!colDataIsNull_var(pColumnInfoData, i)) {
-        return false;
-      }
-    }
-  } else {
-    if (pColumnInfoData->nullbitmap == NULL) {
-      return false;
-    }
-
-    for (int32_t i = startIndex; i < nRows; ++i) {
-      if (!colDataIsNull_f(pColumnInfoData->nullbitmap, i)) {
-        return false;
-      }
-    }
-  }
-
-  return true;
-}
-
 static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row,
                                        SColumnDataAgg* pColAgg) {
   if (!pColumnInfoData->hasNull) {
@@ -137,18 +109,18 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
   }
 }
 
-static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
+static FORCE_INLINE void colDataSetNULL(SColumnInfoData* pColumnInfoData, uint32_t rowIndex) {
   // There is a placehold for each NULL value of binary or nchar type.
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    colDataSetNull_var(pColumnInfoData, currentRow);  // it is a null value of VAR type.
+    colDataSetNull_var(pColumnInfoData, rowIndex);  // it is a null value of VAR type.
   } else {
-    colDataSetNull_f_s(pColumnInfoData, currentRow);
+    colDataSetNull_f_s(pColumnInfoData, rowIndex);
   }
 
   pColumnInfoData->hasNull = true;
 }
 
-static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) {
+static FORCE_INLINE void colDataSetNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) {
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     for (int32_t i = start; i < start + nRows; ++i) {
       colDataSetNull_var(pColumnInfoData, i);  // it is a null value of VAR type.
@@ -163,50 +135,50 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui
   pColumnInfoData->hasNull = true;
 }
 
-static FORCE_INLINE void colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) {
+static FORCE_INLINE void colDataSetInt8(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int8_t* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_TINYINT ||
          pColumnInfoData->info.type == TSDB_DATA_TYPE_UTINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_BOOL);
-  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
+  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex;
   *(int8_t*)p = *(int8_t*)v;
 }
 
-static FORCE_INLINE void colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) {
+static FORCE_INLINE void colDataSetInt16(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int16_t* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT ||
          pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT);
-  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
+  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex;
   *(int16_t*)p = *(int16_t*)v;
 }
 
-static FORCE_INLINE void colDataAppendInt32(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int32_t* v) {
+static FORCE_INLINE void colDataSetInt32(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int32_t* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_INT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UINT);
-  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
+  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex;
   *(int32_t*)p = *(int32_t*)v;
 }
 
-static FORCE_INLINE void colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) {
+static FORCE_INLINE void colDataSetInt64(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int64_t* v) {
   int32_t type = pColumnInfoData->info.type;
   ASSERT(type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT || type == TSDB_DATA_TYPE_TIMESTAMP);
-  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
+  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex;
   *(int64_t*)p = *(int64_t*)v;
 }
 
-static FORCE_INLINE void colDataAppendFloat(SColumnInfoData* pColumnInfoData, uint32_t currentRow, float* v) {
+static FORCE_INLINE void colDataSetFloat(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, float* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_FLOAT);
-  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
+  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex;
   *(float*)p = *(float*)v;
 }
 
-static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, uint32_t currentRow, double* v) {
+static FORCE_INLINE void colDataSetDouble(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, double* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_DOUBLE);
-  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
+  char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex;
   *(double*)p = *(double*)v;
 }
 
 int32_t getJsonValueLen(const char* data);
 
-int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
+int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
-int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
+int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
-                            uint32_t numOfRows);
+int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows);
 int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity,
                         const SColumnInfoData* pSource, int32_t numOfRow2);
 int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
@@ -245,7 +217,7 @@ void blockDataEmpty(SSDataBlock* pDataBlock);
 
 size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
 
-int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
+int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n);
 int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n);
 
 int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
@@ -280,13 +252,6 @@ static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
   return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock);
 }
 
-static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,
-                                                 int8_t compressed) {
-  int32_t colSize = colDataGetLength(pColRes, numOfRows);
-  return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data,
-                                                      colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0);
-}
-
 #ifdef __cplusplus
 }
 #endif

include/common/tdataformat.h

@@ -106,7 +106,7 @@ int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData);
 
 // SRow ================================
 int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow);
-void    tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
+int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 void    tRowDestroy(SRow *pRow);
 void    tRowSort(SArray *aRowP);
 int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);

include/common/tmisce.h

@@ -27,13 +27,14 @@ typedef struct SCorEpSet {
   SEpSet  epSet;
 } SCorEpSet;
 
+#define GET_ACTIVE_EP(_eps) (&((_eps)->eps[(_eps)->inUse]))
 int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
 void    addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
 
-bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
+bool    isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
-
+void    epsetAssign(SEpSet* dst, const SEpSet* pSrc);
-void   updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet);
+void    updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet);
-SEpSet getEpSet_s(SCorEpSet* pEpSet);
+SEpSet  getEpSet_s(SCorEpSet* pEpSet);
 
 #ifdef __cplusplus
 }

include/common/tmsg.h

@@ -1904,10 +1904,10 @@ typedef struct {
 } SMqConsumerLostMsg, SMqConsumerRecoverMsg, SMqConsumerClearMsg;
 
 typedef struct {
-  int64_t consumerId;
+  int64_t  consumerId;
-  char    cgroup[TSDB_CGROUP_LEN];
+  char     cgroup[TSDB_CGROUP_LEN];
-  char    clientId[256];
+  char     clientId[256];
-  SArray* topicNames;  // SArray<char**>
+  SArray*  topicNames;  // SArray<char**>
 } SCMSubscribeReq;
 
 static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
@@ -2850,7 +2850,7 @@ typedef struct {
   char   dbFName[TSDB_DB_FNAME_LEN];
   char   stbName[TSDB_TABLE_NAME_LEN];
   char   colName[TSDB_COL_NAME_LEN];
-  char   idxName[TSDB_COL_NAME_LEN];
+  char   idxName[TSDB_INDEX_FNAME_LEN];
   int8_t idxType;
 } SCreateTagIndexReq;
 

include/common/tmsgdef.h

@@ -172,8 +172,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_SERVER_VERSION, "server-version", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL)
+  // TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
+  // TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
 
   TD_NEW_MSG_SEG(TDMT_VND_MSG)

include/common/ttime.h

@@ -80,15 +80,15 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char
 int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
 int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision);
 
-int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
+int32_t taosParseTime(const char* timestr, int64_t* pTime, int32_t len, int32_t timePrec, int8_t dayligth);
 void    deltaToUtcInitOnce();
 char    getPrecisionUnit(int32_t precision);
 
-int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
+int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
-int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
+int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit);
 int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
 
-void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t time, int32_t precision);
+void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);
 
 #ifdef __cplusplus
 }

include/libs/function/function.h

@@ -76,7 +76,7 @@ enum {
 enum {
   MAIN_SCAN = 0x0u,
   REVERSE_SCAN = 0x1u,  // todo remove it
-  REPEAT_SCAN = 0x2u,   // repeat scan belongs to the master scan
+  PRE_SCAN = 0x2u,   // pre-scan belongs to the main scan and occurs before main scan
 };
 
 typedef struct SPoint1 {

include/libs/qcom/query.h

@@ -26,6 +26,7 @@ extern "C" {
 #include "tlog.h"
 #include "tmsg.h"
 #include "tmsgcb.h"
+#include "systable.h"
 
 typedef enum {
   JOB_TASK_STATUS_NULL = 0,
@@ -302,9 +303,10 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 
 #define REQUEST_TOTAL_EXEC_TIMES 2
 
-#define IS_SYS_DBNAME(_dbname)                                                    \
+#define IS_INFORMATION_SCHEMA_DB(_name) ((*(_name) == 'i') && (0 == strcmp(_name, TSDB_INFORMATION_SCHEMA_DB)))
-  (((*(_dbname) == 'i') && (0 == strcmp(_dbname, TSDB_INFORMATION_SCHEMA_DB))) || \
+#define IS_PERFORMANCE_SCHEMA_DB(_name) ((*(_name) == 'p') && (0 == strcmp(_name, TSDB_PERFORMANCE_SCHEMA_DB)))
-   ((*(_dbname) == 'p') && (0 == strcmp(_dbname, TSDB_PERFORMANCE_SCHEMA_DB))))
+
+#define IS_SYS_DBNAME(_dbname) (IS_INFORMATION_SCHEMA_DB(_dbname) || IS_PERFORMANCE_SCHEMA_DB(_dbname))
 
 #define qFatal(...)                                                     \
   do {                                                                  \

include/libs/stream/tstream.h

@@ -175,20 +175,24 @@ typedef struct {
 
 void streamFreeQitem(SStreamQueueItem* data);
 
+#if 0
 bool              streamQueueResEmpty(const SStreamQueueRes* pRes);
 int64_t           streamQueueResSize(const SStreamQueueRes* pRes);
 SStreamQueueNode* streamQueueResFront(SStreamQueueRes* pRes);
 SStreamQueueNode* streamQueueResPop(SStreamQueueRes* pRes);
 void              streamQueueResClear(SStreamQueueRes* pRes);
 SStreamQueueRes   streamQueueBuildRes(SStreamQueueNode* pNode);
+#endif
 
 typedef struct {
   SStreamQueueNode* pHead;
 } SStreamQueue1;
 
+#if 0
 bool            streamQueueHasTask(const SStreamQueue1* pQueue);
 int32_t         streamQueuePush(SStreamQueue1* pQueue, SStreamQueueItem* pItem);
 SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue);
+#endif
 
 typedef struct {
   STaosQueue* queue;
@@ -636,7 +640,7 @@ void         streamMetaClose(SStreamMeta* streamMeta);
 int32_t      streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
 int32_t      streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen);
-SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId);
+// SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId);
 
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);

include/libs/wal/wal.h

@@ -66,6 +66,7 @@ typedef struct {
   int64_t commitVer;
   int64_t appliedVer;
   int64_t lastVer;
+  int64_t logRetention;
 } SWalVer;
 
 #pragma pack(push, 1)
@@ -126,7 +127,7 @@ typedef struct SWal {
 typedef struct {
   int64_t refId;
   int64_t refVer;
-  int64_t refFile;
+//  int64_t refFile;
   SWal   *pWal;
 } SWalRef;
 
@@ -180,7 +181,7 @@ void walFsync(SWal *, bool force);
 int32_t walCommit(SWal *, int64_t ver);
 int32_t walRollback(SWal *, int64_t ver);
 // notify that previous logs can be pruned safely
-int32_t walBeginSnapshot(SWal *, int64_t ver);
+int32_t walBeginSnapshot(SWal *, int64_t ver, int64_t logRetention);
 int32_t walEndSnapshot(SWal *);
 int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
 // for tq

include/os/osMemory.h

@@ -29,6 +29,7 @@ extern "C" {
 #define calloc  CALLOC_FUNC_TAOS_FORBID
 #define realloc REALLOC_FUNC_TAOS_FORBID
 #define free    FREE_FUNC_TAOS_FORBID
+#define strdup  STRDUP_FUNC_TAOS_FORBID
 #endif // ifndef ALLOW_FORBID_FUNC
 
 #endif // if !defined(WINDOWS)
@@ -38,7 +39,7 @@ int32_t taosMemoryDbgInitRestore();
 void   *taosMemoryMalloc(int64_t size);
 void   *taosMemoryCalloc(int64_t num, int64_t size);
 void   *taosMemoryRealloc(void *ptr, int64_t size);
-void   *taosMemoryStrDup(const char *ptr);
+char   *taosStrdup(const char *ptr);
 void    taosMemoryFree(void *ptr);
 int64_t taosMemorySize(void *ptr);
 void    taosPrintBackTrace();

include/util/taoserror.h

@@ -119,6 +119,7 @@ int32_t* taosGetErrno();
 
 #define TSDB_CODE_APP_IS_STARTING               TAOS_DEF_ERROR_CODE(0, 0x0130) //
 #define TSDB_CODE_APP_IS_STOPPING               TAOS_DEF_ERROR_CODE(0, 0x0131) //
+#define TSDB_CODE_IVLD_DATA_FMT                 TAOS_DEF_ERROR_CODE(0, 0x0132) //
 
 //client
 #define TSDB_CODE_TSC_INVALID_OPERATION         TAOS_DEF_ERROR_CODE(0, 0x0200)

include/util/tarray.h

@@ -43,7 +43,7 @@ typedef struct SArray {
  * @return
  */
 SArray* taosArrayInit(size_t size, size_t elemSize);
-SArray* taosArrayInit_s(size_t size, size_t elemSize, size_t initialSize);
+SArray* taosArrayInit_s(size_t elemSize, size_t initialSize);
 
 /**
  *

include/util/tlog.h

@@ -89,20 +89,20 @@ bool taosAssertRelease(bool condition);
 // Disable all asserts that may compromise the performance.
 #if defined DISABLE_ASSERT
 #define ASSERT(condition)
-#define ASSERTS(condition, ...)    (0)
+#define ASSERTS(condition, ...) (0)
 #else
-#define ASSERTS(condition, ...) taosAssertDebug(condition, __FILE__, __LINE__, __VA_ARGS__)
+#define ASSERTS(condition, ...) ((condition) ? false : taosAssertDebug(condition, __FILE__, __LINE__, __VA_ARGS__))
 #ifdef NDEBUG
 #define ASSERT(condition) taosAssertRelease(condition)
 #else
-#define ASSERT(condition) taosAssertDebug(condition, __FILE__, __LINE__, "assert info not provided")
+#define ASSERT(condition) ASSERTS(condition, "assert info not provided")
 #endif
 #endif
 
-void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo);
+void    taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, void *sigInfo);
-void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr* pFd);
+void    taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr *pFd);
-void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile);
+void    taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile);
-int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime);
+int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime);
 
 // clang-format off
 #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
@@ -116,7 +116,7 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
 // clang-format on
-//#define BUF_PAGE_DEBUG
+// #define BUF_PAGE_DEBUG
 #ifdef __cplusplus
 }
 #endif

include/util/tworker.h

@@ -26,12 +26,12 @@ extern "C" {
 typedef struct SQWorkerPool SQWorkerPool;
 typedef struct SWWorkerPool SWWorkerPool;
 
-typedef struct SQWorker {
+typedef struct SQueueWorker {
   int32_t  id;      // worker id
   int64_t  pid;     // thread pid
   TdThread thread;  // thread id
   void    *pool;
-} SQWorker;
+} SQueueWorker;
 
 typedef struct SQWorkerPool {
   int32_t       max;  // max number of workers
@@ -39,7 +39,7 @@ typedef struct SQWorkerPool {
   int32_t       num;  // current number of workers
   STaosQset    *qset;
   const char   *name;
-  SQWorker     *workers;
+  SQueueWorker *workers;
   TdThreadMutex mutex;
 } SQWorkerPool;
 

include/util/xxhash.h

@@ -1,328 +0,0 @@
-/*
-   xxHash - Extremely Fast Hash algorithm
-   Header File
-   Copyright (C) 2012-2016, Yann Collet.
-
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - xxHash source repository : https://github.com/Cyan4973/xxHash
-*/
-
-/* Notice extracted from xxHash homepage :
-
-xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
-It also successfully passes all tests from the SMHasher suite.
-
-Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
-
-Name            Speed       Q.Score   Author
-xxHash          5.4 GB/s     10
-CrapWow         3.2 GB/s      2       Andrew
-MumurHash 3a    2.7 GB/s     10       Austin Appleby
-SpookyHash      2.0 GB/s     10       Bob Jenkins
-SBox            1.4 GB/s      9       Bret Mulvey
-Lookup3         1.2 GB/s      9       Bob Jenkins
-SuperFastHash   1.2 GB/s      1       Paul Hsieh
-CityHash64      1.05 GB/s    10       Pike & Alakuijala
-FNV             0.55 GB/s     5       Fowler, Noll, Vo
-CRC32           0.43 GB/s     9
-MD5-32          0.33 GB/s    10       Ronald L. Rivest
-SHA1-32         0.28 GB/s    10
-
-Q.Score is a measure of quality of the hash function.
-It depends on successfully passing SMHasher test set.
-10 is a perfect score.
-
-A 64-bit version, named XXH64, is available since r35.
-It offers much better speed, but for 64-bit applications only.
-Name     Speed on 64 bits    Speed on 32 bits
-XXH64       13.8 GB/s            1.9 GB/s
-XXH32        6.8 GB/s            6.0 GB/s
-*/
-
-#ifndef XXHASH_H_5627135585666179
-#define XXHASH_H_5627135585666179 1
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************
-*  Definitions
-******************************/
-#include <stddef.h>   /* size_t */
-typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
-
-
-/* ****************************
- *  API modifier
- ******************************/
-/** XXH_INLINE_ALL (and XXH_PRIVATE_API)
- *  This is useful to include xxhash functions in `static` mode
- *  in order to inline them, and remove their symbol from the public list.
- *  Inlining can offer dramatic performance improvement on small keys.
- *  Methodology :
- *     #define XXH_INLINE_ALL
- *     #include "xxhash.h"
- * `xxhash.c` is automatically included.
- *  It's not useful to compile and link it as a separate module.
- */
-#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
-#  ifndef XXH_STATIC_LINKING_ONLY
-#    define XXH_STATIC_LINKING_ONLY
-#  endif
-#  if defined(__GNUC__)
-#    define XXH_PUBLIC_API static __inline __attribute__((unused))
-#  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-#    define XXH_PUBLIC_API static inline
-#  elif defined(_MSC_VER)
-#    define XXH_PUBLIC_API static __inline
-#  else
-     /* this version may generate warnings for unused static functions */
-#    define XXH_PUBLIC_API static
-#  endif
-#else
-#  define XXH_PUBLIC_API   /* do nothing */
-#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
-
-/*! XXH_NAMESPACE, aka Namespace Emulation :
- *
- * If you want to include _and expose_ xxHash functions from within your own library,
- * but also want to avoid symbol collisions with other libraries which may also include xxHash,
- *
- * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
- * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
- *
- * Note that no change is required within the calling program as long as it includes `xxhash.h` :
- * regular symbol name will be automatically translated by this header.
- */
-#ifdef XXH_NAMESPACE
-#  define XXH_CAT(A,B) A##B
-#  define XXH_NAME2(A,B) XXH_CAT(A,B)
-#  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
-#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
-#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
-#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
-#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
-#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
-#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
-#  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
-#  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
-#  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
-#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
-#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
-#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
-#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
-#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
-#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
-#  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
-#  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
-#  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
-#endif
-
-
-/* *************************************
-*  Version
-***************************************/
-#define XXH_VERSION_MAJOR    0
-#define XXH_VERSION_MINOR    6
-#define XXH_VERSION_RELEASE  5
-#define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
-XXH_PUBLIC_API unsigned XXH_versionNumber (void);
-
-
-/*-**********************************************************************
-*  32-bit hash
-************************************************************************/
-typedef unsigned int XXH32_hash_t;
-
-/*! XXH32() :
-    Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input".
-    The memory between input & input+length must be valid (allocated and read-accessible).
-    "seed" can be used to alter the result predictably.
-    Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
-XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
-
-/*======   Streaming   ======*/
-typedef struct XXH32_state_s XXH32_state_t;   /* incomplete type */
-XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
-XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
-XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
-
-XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned int seed);
-XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API XXH32_hash_t  XXH32_digest (const XXH32_state_t* statePtr);
-
-/*
- * Streaming functions generate the xxHash of an input provided in multiple segments.
- * Note that, for small input, they are slower than single-call functions, due to state management.
- * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
- *
- * XXH state must first be allocated, using XXH*_createState() .
- *
- * Start a new hash by initializing state with a seed, using XXH*_reset().
- *
- * Then, feed the hash state by calling XXH*_update() as many times as necessary.
- * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
- *
- * Finally, a hash value can be produced anytime, by using XXH*_digest().
- * This function returns the nn-bits hash as an int or long long.
- *
- * It's still possible to continue inserting input into the hash state after a digest,
- * and generate some new hashes later on, by calling again XXH*_digest().
- *
- * When done, free XXH state space if it was allocated dynamically.
- */
-
-/*======   Canonical representation   ======*/
-
-typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
-XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
-XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
-
-/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
- * The canonical representation uses human-readable write convention, aka big-endian (large digits first).
- * These functions allow transformation of hash result into and from its canonical format.
- * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
- */
-
-
-#ifndef XXH_NO_LONG_LONG
-/*-**********************************************************************
-*  64-bit hash
-************************************************************************/
-typedef unsigned long long XXH64_hash_t;
-
-/*! XXH64() :
-    Calculate the 64-bit hash of sequence of length "len" stored at memory address "input".
-    "seed" can be used to alter the result predictably.
-    This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark).
-*/
-XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
-
-/*======   Streaming   ======*/
-typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
-XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
-XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
-XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
-
-XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
-XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API XXH64_hash_t  XXH64_digest (const XXH64_state_t* statePtr);
-
-/*======   Canonical representation   ======*/
-typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
-XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
-XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
-#endif  /* XXH_NO_LONG_LONG */
-
-
-
-#ifdef XXH_STATIC_LINKING_ONLY
-
-/* ================================================================================================
-   This section contains declarations which are not guaranteed to remain stable.
-   They may change in future versions, becoming incompatible with a different version of the library.
-   These declarations should only be used with static linking.
-   Never use them in association with dynamic linking !
-=================================================================================================== */
-
-/* These definitions are only present to allow
- * static allocation of XXH state, on stack or in a struct for example.
- * Never **ever** use members directly. */
-
-#if !defined (__VMS) \
-  && (defined (__cplusplus) \
-  || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-#   include <stdint.h>
-
-struct XXH32_state_s {
-   uint32_t total_len_32;
-   uint32_t large_len;
-   uint32_t v1;
-   uint32_t v2;
-   uint32_t v3;
-   uint32_t v4;
-   uint32_t mem32[4];
-   uint32_t memsize;
-   uint32_t reserved;   /* never read nor write, might be removed in a future version */
-};   /* typedef'd to XXH32_state_t */
-
-struct XXH64_state_s {
-   uint64_t total_len;
-   uint64_t v1;
-   uint64_t v2;
-   uint64_t v3;
-   uint64_t v4;
-   uint64_t mem64[4];
-   uint32_t memsize;
-   uint32_t reserved[2];          /* never read nor write, might be removed in a future version */
-};   /* typedef'd to XXH64_state_t */
-
-# else
-
-struct XXH32_state_s {
-   unsigned total_len_32;
-   unsigned large_len;
-   unsigned v1;
-   unsigned v2;
-   unsigned v3;
-   unsigned v4;
-   unsigned mem32[4];
-   unsigned memsize;
-   unsigned reserved;   /* never read nor write, might be removed in a future version */
-};   /* typedef'd to XXH32_state_t */
-
-#   ifndef XXH_NO_LONG_LONG  /* remove 64-bit support */
-struct XXH64_state_s {
-   unsigned long long total_len;
-   unsigned long long v1;
-   unsigned long long v2;
-   unsigned long long v3;
-   unsigned long long v4;
-   unsigned long long mem64[4];
-   unsigned memsize;
-   unsigned reserved[2];     /* never read nor write, might be removed in a future version */
-};   /* typedef'd to XXH64_state_t */
-#    endif
-
-# endif
-
-
-#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
-#  include "xxhash.c"   /* include xxhash function bodies as `static`, for inlining */
-#endif
-
-#endif /* XXH_STATIC_LINKING_ONLY */
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* XXHASH_H_5627135585666179 */

packaging/docker/DockerfileCloud

@@ -1,30 +0,0 @@
-FROM ubuntu:18.04
-
-WORKDIR /root
-
-ARG pkgFile
-ARG dirName
-ARG cpuType
-RUN echo ${pkgFile} && echo ${dirName}
-
-RUN apt update
-RUN apt install -y curl
-
-COPY ${pkgFile} /root/
-ENV TINI_VERSION v0.19.0
-ENV TAOS_DISABLE_ADAPTER 1
-ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini
-ENV DEBIAN_FRONTEND=noninteractive
-WORKDIR /root/
-RUN tar -zxf ${pkgFile} && cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root && rm /root/${pkgFile} && rm -rf /root/${dirName} && apt-get update && apt-get install -y locales tzdata netcat && locale-gen en_US.UTF-8 && apt-get clean && rm -rf /var/lib/apt/lists/ && chmod +x /tini
-
-ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" \
-    LC_CTYPE=en_US.UTF-8 \
-    LANG=en_US.UTF-8 \
-    LC_ALL=en_US.UTF-8
-COPY ./run.sh /usr/bin/
-COPY ./bin/* /usr/bin/
-
-ENTRYPOINT ["/tini", "--", "/usr/bin/entrypoint.sh"]
-CMD ["bash", "-c", "/usr/bin/run.sh"]
-VOLUME [ "/var/lib/taos", "/var/log/taos" ]

packaging/docker/README.md

@@ -18,65 +18,58 @@ TDengine is an open-sourced big data platform under [GNU AGPL v3.0](http://www.g
 
 ## How to use this image
 
-### Start a TDengine instance with RESTful API exposed
+### Starting TDengine
 
-Simply, you can use `docker run` to start a TDengine instance and connect it with restful connectors(eg. [JDBC-RESTful](https://www.taosdata.com/cn/documentation/connector/java)).
+The TDengine image starts with the HTTP service activated by default, using the following command:
 
-```bash
+```shell
 docker run -d --name tdengine -p 6041:6041 tdengine/tdengine
 ```
 
-This command starts a docker container by name `tdengine` with TDengine server running, and maps the container's HTTP port 6041 to the host's port 6041. If you have `curl` in your host, you can list the databases by the command:
+The above command starts a container named "tdengine" and maps the HTTP service port 6041 to the host port 6041. You can verify that the HTTP service provided in this container is available using the following command.
 
-```bash
+```shell
 curl -u root:taosdata -d "show databases" localhost:6041/rest/sql
 ```
 
-You can execute the `taos` shell command in the container:
+The TDengine client taos can be executed in this container to access TDengine using the following command.
 
-```bash
+```shell
 $ docker exec -it tdengine taos
 
-Welcome to the TDengine shell from Linux, Client Version:2.4.0.0
-Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.
-
 taos> show databases;
-              name              |      created_time       |   ntables   |   vgroups   | replica | quorum |  days  |           keep           |  cache(MB)  |   blocks    |   minrows   |   maxrows   | wallevel |    fsync    | comp | cachelast | precision | update |   status   |
+              name              |
-====================================================================================================================================================================================================================================================================================
+=================================
- log                            | 2022-01-17 13:57:22.270 |          10 |           1 |       1 |      1 |     10 | 30                       |           1 |           3 |         100 |        4096 |        1 |        3000 |    2 |         0 | us        |      0 | ready      |
+ information_schema             |
-Query OK, 1 row(s) in set (0.002843s)
+ performance_schema             |
+Query OK, 2 row(s) in set (0.002843s)
 ```
 
-Since TDengine use container hostname to establish connections, it's a bit more complex to use TDengine CLI and native connectors(such as JDBC-JNI) with TDengine container instance. This is the recommended way to expose ports and use TDengine with docker in simple cases. If you want to use TDengine CLI or taosc/connectors smoothly outside the `tdengine` container, see next use cases that match you need.
+The TDengine server running in the container uses the container's hostname to establish a connection. Using TDengine CLI or various connectors (such as JDBC-JNI) to access the TDengine inside the container from outside the container is more complicated. So the above is the simplest way to access the TDengine service in the container and is suitable for some simple scenarios. Please refer to the next section if you want to access the TDengine service in the container from outside the container using TDengine CLI or various connectors for complex scenarios.
 
-### Start with host network
+### Start TDengine on the host network
 
-```bash
+```shell
 docker run -d --name tdengine --network host tdengine/tdengine
 ```
 
-Starts container with `host` network will use host's hostname as fqdn instead of container id. It's much like starting natively with `systemd` in host. After installing the client, you can use `taos` shell as normal in host path.
+The above command starts TDengine on the host network and uses the host's FQDN to establish a connection instead of the container's hostname. It is the equivalent of using `systemctl` to start TDengine on the host. If the TDengine client is already installed on the host, you can access it directly with the following command.
 
-```bash
+```shell
 $ taos
 
-Welcome to the TDengine shell from Linux, Client Version:2.4.0.0
-Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.
-
 taos> show dnodes;
    id   |           end_point            | vnodes | cores  |   status   | role  |       create_time       |      offline reason      |
 ======================================================================================================================================
-      1 | host:6030           |      1 |      8 | ready      | any   | 2022-01-17 22:10:32.619 |                          |
+      1 | myhost:6030           |      1 |      8 | ready      | any   | 2022-01-17 22:10:32.619 |                          |
 Query OK, 1 row(s) in set (0.003233s)
 ```
 
-### Start with exposed ports and specified hostname
+### Start TDengine with the specified hostname and port
 
-Set the fqdn explicitly will help you to use in other environment or applications. We provide environment variable `TAOS_FQDN` or `fqdn` config option to explicitly set the hostname used by TDengine container instance(s).
+The `TAOS_FQDN` environment variable or the `fqdn` configuration item in `taos.cfg` allows TDengine to establish a connection at the specified hostname. This approach provides greater flexibility for deployment.
 
-Use `TAOS_FQDN` variable within `docker run` command:
+```shell
-
-```bash
 docker run -d \
    --name tdengine \
    -e TAOS_FQDN=tdengine \
@@ -85,79 +78,58 @@ docker run -d \
    tdengine/tdengine
 ```
 
-This command starts a docker container with TDengine server running and maps the container's TCP ports from 6030 to 6049 to the host's ports from 6030 to 6049 with TCP protocol and UDP ports range 6030-6039 to the host's UDP ports 6030-6039. If the host is already running TDengine server and occupying the same port(s), you need to map the container's port to a different unused port segment. (Please see TDengine 2.0 Port Description for details). In order to support TDengine clients accessing TDengine server services, both TCP and UDP ports need to be exposed by default(unless `rpcForceTcp` is set to `1`).
+The above command starts a TDengine service in the container, which listens to the hostname tdengine, and maps the container's port segment 6030 to 6049 to the host's port segment 6030 to 6049 (both TCP and UDP ports need to be mapped). If the port segment is already occupied on the host, you can modify the above command to specify a free port segment on the host. If `rpcForceTcp` is set to `1`, you can map only the TCP protocol.
 
-If you want to use TDengine CLI or native connectors([JDBC-JNI](https://www.taosdata.com/cn/documentation/connector/java), or [driver-go](https://github.com/taosdata/driver-go)), you need to make sure the `TAOS_FQDN` is resolvable at `/etc/hosts` or with custom DNS service.
+Next, ensure the hostname "tdengine" is resolvable in `/etc/hosts`.
 
-If you set the `TAOS_FQDN` to host's hostname, it will works as using `hosts` network like previous use case. Otherwise, like in `-e TAOS_FQDN=tdengine`, you can add the hostname record `tdengine` into `/etc/hosts` (use `127.0.0.1` here in host path, if use TDengine client/application in other hosts, you should set the right ip to the host eg. `192.168.10.1`(check the real ip in host with `hostname -i` or `ip route list default`) to make the TDengine endpoint resolvable):
+```shell
-
-```bash
 echo 127.0.0.1 tdengine |sudo tee -a /etc/hosts
 ```
 
-Then you can use `taos` with the host `tdengine`:
+Finally, the TDengine service can be accessed from the TDengine CLI or any connector with "tdengine" as the server address.
 
-```bash
+```shell
-taos -h tdengine
+taos -h tdengine -P 6030
 ```
 
-Or develop/test applications with native connectors. As in python:
+If set `TAOS_FQDN` to the same hostname, the effect is the same as "Start TDengine on host network".
 
-```python
+### Start TDengine on the specified network
-import taos;
-conn = taos.connect(host = "tdengine")
-res = conn.query("show databases")
-for row in res.fetch_all_into_dict():
-   print(row)
-```
 
-See the results:
+You can also start TDengine on a specific network. Perform the following steps:
 
-```bash
+1. First, create a docker network named `td-net`
-Python 3.8.10 (default, Nov 26 2021, 20:14:08) 
-[GCC 9.3.0] on linux
-Type "help", "copyright", "credits" or "license" for more information.
->>> import taos;
->>> conn = taos.connect(host = "tdengine")
->>> res = conn.query("show databases")
->>> for row in res.fetch_all_into_dict():
-...    print(row)
-... 
-{'name': 'log', 'created_time': datetime.datetime(2022, 1, 17, 22, 56, 2, 490000), 'ntables': 11, 'vgroups': 1, 'replica': 1, 'quorum': 1, 'days': 10, 'keep': '30', 'cache(MB)': 1, 'blocks': 3, 'minrows': 100, 'maxrows': 4096, 'wallevel': 1, 'fsync': 3000, 'comp': 2, 'cachelast': 0, 'precision': 'us', 'update': 0, 'status': 'ready'}
-```
 
-### Start with specific network
+   ```shell
+   docker network create td-net
+   ```
 
-Alternatively, you can use TDengine natively by using specific network.
+2. Start TDengine
 
-First, create network for TDengine server and client/application.
+   Start the TDengine service on the `td-net` network with the following command:
 
-```bash
+   ```shell
-docker network create td-net
+   docker run -d --name tdengine --network td-net \
-```
+      -e TAOS_FQDN=tdengine \
+      tdengine/tdengine
+   ```
 
-Start TDengine instance with service name as fqdn (explicitly set with `TAOS_FQDN`):
+3. Start the TDengine client in another container on the same network
 
-```bash
+   ```shell
-docker run -d --name tdengine --network td-net \
+   docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine tdengine/tdengine taos
-   -e TAOS_FQDN=tdengine \
+   # or
-   tdengine/tdengine
+   #docker run --rm -it --network td-net -e tdengine/tdengine taos -h tdengine
-```
+   ```
 
-Start TDengine client in another container with the specific network:
+### Launching a client application in a container
 
-```bash
+If you want to start your application in a container, you need to add the corresponding dependencies on TDengine to the image as well, e.g.
-docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine tdengine/tdengine taos
-# or
-docker run --rm -it --network td-net -e tdengine/tdengine taos -h tdengine
-```
 
-When you build your application with docker, you should add the TDengine client in the dockerfile, as based on `ubuntu:20.04` image, install the client like this:
+```docker
-
-```dockerfile
 FROM ubuntu:20.04
 RUN apt-get update && apt-get install -y wget
-ENV TDENGINE_VERSION=2.4.0.0
+ENV TDENGINE_VERSION=3.0.0.0
 RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
@@ -169,10 +141,7 @@ RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENG
 #CMD ["app"]
 ```
 
-Here is an Go example app:
+Here is an example GO program:
-
-<!-- code-spell-checker:disable -->
-<!-- markdownlint-disable MD010 -->
 
 ```go
 /*
@@ -181,19 +150,19 @@ Here is an Go example app:
 package main
 
 import (
-	"database/sql"
+    "database/sql"
-	"flag"
+    "flag"
-	"fmt"
+    "fmt"
-	"time"
+    "time"
 
-	_ "github.com/taosdata/driver-go/v2/taosSql"
+    _ "github.com/taosdata/driver-go/v3/taosSql"
 )
 
 type config struct {
-	hostName   string
+    hostName   string
-	serverPort string
+    serverPort string
-	user       string
+    user       string
-	password   string
+    password   string
 }
 
 var configPara config
@@ -201,70 +170,67 @@ var taosDriverName = "taosSql"
 var url string
 
 func init() {
-	flag.StringVar(&configPara.hostName, "h", "", "The host to connect to TDengine server.")
+    flag.StringVar(&configPara.hostName, "h", "", "The host to connect to TDengine server.")
-	flag.StringVar(&configPara.serverPort, "p", "", "The TCP/IP port number to use for the connection to TDengine server.")
+    flag.StringVar(&configPara.serverPort, "p", "", "The TCP/IP port number to use for the connection to TDengine server.")
-	flag.StringVar(&configPara.user, "u", "root", "The TDengine user name to use when connecting to the server.")
+    flag.StringVar(&configPara.user, "u", "root", "The TDengine user name to use when connecting to the server.")
-	flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.")
+    flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.")
-	flag.Parse()
+    flag.Parse()
 }
 
 func printAllArgs() {
-	fmt.Printf("============= args parse result: =============\n")
+    fmt.Printf("============= args parse result: =============\n")
-	fmt.Printf("hostName:             %v\n", configPara.hostName)
+    fmt.Printf("hostName:             %v\n", configPara.hostName)
-	fmt.Printf("serverPort:           %v\n", configPara.serverPort)
+    fmt.Printf("serverPort:           %v\n", configPara.serverPort)
-	fmt.Printf("usr:                  %v\n", configPara.user)
+    fmt.Printf("usr:                  %v\n", configPara.user)
-	fmt.Printf("password:             %v\n", configPara.password)
+    fmt.Printf("password:             %v\n", configPara.password)
-	fmt.Printf("================================================\n")
+    fmt.Printf("================================================\n")
 }
 
 func main() {
-	printAllArgs()
+    printAllArgs()
 
-	url = "root:taosdata@/tcp(" + configPara.hostName + ":" + configPara.serverPort + ")/"
+    url = "root:taosdata@/tcp(" + configPara.hostName + ":" + configPara.serverPort + ")/"
 
-	taos, err := sql.Open(taosDriverName, url)
+    taos, err := sql.Open(taosDriverName, url)
-	checkErr(err, "open database error")
+    checkErr(err, "open database error")
-	defer taos.Close()
+    defer taos.Close()
 
-	taos.Exec("create database if not exists test")
+    taos.Exec("create database if not exists test")
-	taos.Exec("use test")
+    taos.Exec("use test")
-	taos.Exec("create table if not exists tb1 (ts timestamp, a int)")
+    taos.Exec("create table if not exists tb1 (ts timestamp, a int)")
-	_, err = taos.Exec("insert into tb1 values(now, 0)(now+1s,1)(now+2s,2)(now+3s,3)")
+    _, err = taos.Exec("insert into tb1 values(now, 0)(now+1s,1)(now+2s,2)(now+3s,3)")
-	checkErr(err, "failed to insert")
+    checkErr(err, "failed to insert")
-	rows, err := taos.Query("select * from tb1")
+    rows, err := taos.Query("select * from tb1")
-	checkErr(err, "failed to select")
+    checkErr(err, "failed to select")
 
-	defer rows.Close()
+    defer rows.Close()
-	for rows.Next() {
+    for rows.Next() {
-		var r struct {
+        var r struct {
-			ts time.Time
+            ts time.Time
-			a  int
+            a  int
-		}
+        }
-		err := rows.Scan(&r.ts, &r.a)
+        err := rows.Scan(&r.ts, &r.a)
-		if err != nil {
+        if err != nil {
-			fmt.Println("scan error:\n", err)
+            fmt.Println("scan error:\n", err)
-			return
+            return
-		}
+        }
-		fmt.Println(r.ts, r.a)
+        fmt.Println(r.ts, r.a)
-	}
+    }
 }
 
 func checkErr(err error, prompt string) {
-	if err != nil {
+    if err != nil {
-		fmt.Println("ERROR: %s\n", prompt)
+        fmt.Println("ERROR: %s\n", prompt)
-		panic(err)
+        panic(err)
-	}
+    }
 }
 ```
 
-<!-- markdownlint-enable MD010 -->
+Here is the full Dockerfile:
-<!-- code-spell-checker:enable -->
 
-Full version of dockerfile could be:
+```docker
-
-```dockerfile
 FROM golang:1.17.6-buster as builder
-ENV TDENGINE_VERSION=2.4.0.0
+ENV TDENGINE_VERSION=3.0.0.0
 RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
@@ -274,11 +240,13 @@ RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENG
 WORKDIR /usr/src/app/
 ENV GOPROXY="https://goproxy.io,direct"
 COPY ./main.go ./go.mod ./go.sum /usr/src/app/
-RUN go env && go mod tidy && go build
+RUN go env
+RUN go mod tidy
+RUN go build
 
 FROM ubuntu:20.04
 RUN apt-get update && apt-get install -y wget
-ENV TDENGINE_VERSION=2.4.0.0
+ENV TDENGINE_VERSION=3.0.0.0
 RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
@@ -291,9 +259,9 @@ COPY --from=builder /usr/src/app/app /usr/bin/
 CMD ["app"]
 ```
 
-Suppose you have `main.go`, `go.mod` `go.sum`, `app.dockerfile`, build the app and run it with network `td-net`:
+Now that we have `main.go`, `go.mod`, `go.sum`, `app.dockerfile`, we can build the application and start it on the `td-net` network.
 
-```bash
+```shell
 $ docker build -t app -f app.dockerfile
 $ docker run --rm --network td-net app -h tdengine -p 6030
 ============= args parse result: =============
@@ -316,26 +284,18 @@ password:             taosdata
 2022-01-18 01:43:51.029 +0000 UTC 3
 ```
 
-Now you must be much familiar with developing and testing with TDengine, let's see some more complex cases.
+### Start the TDengine cluster with docker-compose
 
-### Start with docker-compose with multiple nodes(instances)
+1. The following docker-compose file starts a TDengine cluster with three nodes.
 
-Start a 2-replicas-2-mnodes-2-dnodes-1-arbitrator TDengine cluster with `docker-compose` is quite simple. Save the file as `docker-compose.yml`:
+```yml
-
-```yaml
 version: "3"
 services:
-  arbitrator:
-    image: tdengine/tdengine:$VERSION
-    command: tarbitrator
   td-1:
     image: tdengine/tdengine:$VERSION
     environment:
       TAOS_FQDN: "td-1"
       TAOS_FIRST_EP: "td-1"
-      TAOS_NUM_OF_MNODES: "2"
-      TAOS_REPLICA: "2"
-      TAOS_ARBITRATOR: arbitrator:6042
     volumes:
       - taosdata-td1:/var/lib/taos/
       - taoslog-td1:/var/log/taos/
@@ -344,101 +304,95 @@ services:
     environment:
       TAOS_FQDN: "td-2"
       TAOS_FIRST_EP: "td-1"
-      TAOS_NUM_OF_MNODES: "2"
-      TAOS_REPLICA: "2"
-      TAOS_ARBITRATOR: arbitrator:6042
     volumes:
       - taosdata-td2:/var/lib/taos/
       - taoslog-td2:/var/log/taos/
+  td-3:
+    image: tdengine/tdengine:$VERSION
+    environment:
+      TAOS_FQDN: "td-3"
+      TAOS_FIRST_EP: "td-1"
+    volumes:
+      - taosdata-td3:/var/lib/taos/
+      - taoslog-td3:/var/log/taos/
 volumes:
   taosdata-td1:
   taoslog-td1:
   taosdata-td2:
   taoslog-td2:
+  taosdata-td3:
+  taoslog-td3:
 ```
 
-You may notice that:
+:::note
 
-- We use `VERSION` environment variable to set `tdengine` image tag version once.
+- The `VERSION` environment variable is used to set the tdengine image tag
-- **`TAOS_FIRST_EP`** **MUST** be set to join the newly created instances into an existing TDengine cluster. If you want more instances, use `TAOS_SECOND_EP` in case of HA(High Availability) concerns.
+- `TAOS_FIRST_EP` must be set on the newly created instance so that it can join the TDengine cluster; if there is a high availability requirement, `TAOS_SECOND_EP` needs to be used at the same time
-- `TAOS_NUM_OF_MNODES` is for setting number of mnodes for the cluster.
+  :::
-- `TAOS_REPLICA` set the default database replicas, `2` means there're one master and one slave copy of data. The `replica` option should be `1 <= replica <= 3`, and not greater than dnodes number.
-- `TAOS_ARBITRATOR` set the arbitrator entrypoint of the cluster for failover/election stuff. It's better to use arbitrator in a two nodes cluster.
-- The way to start an arbitrator service is as easy as abc: just add command name `tarbitrator`(which is the binary name of arbitrator daemon) in docker-compose service option: `command: tarbitrator`, and everything is ok now.
 
-Now run `docker-compose up -d` with version specified:
+2. Start the cluster
 
-```bash
+   ```shell
-$ VERSION=2.4.0.0 docker-compose up -d
+   $ VERSION=3.0.0.0 docker-compose up -d
-Creating network "test_default" with the default driver
+   Creating network "test_default" with the default driver
-Creating volume "test_taosdata-td1" with default driver
+   Creating volume "test_taosdata-td1" with default driver
-Creating volume "test_taoslog-td1" with default driver
+   Creating volume "test_taoslog-td1" with default driver
-Creating volume "test_taosdata-td2" with default driver
+   Creating volume "test_taosdata-td2" with default driver
-Creating volume "test_taoslog-td2" with default driver
+   Creating volume "test_taoslog-td2" with default driver
-Creating test_td-1_1       ... done
+   Creating test_td-1_1       ... done
-Creating test_arbitrator_1 ... done
+   Creating test_arbitrator_1 ... done
-Creating test_td-2_1       ... done
+   Creating test_td-2_1       ... done
-```
+   ```
 
-Check the status:
+3. Check the status of each node
 
-```bash
+   ```shell
-$ docker-compose ps
+   $ docker-compose ps
-      Name                     Command               State                                                                Ports                                                              
+         Name                     Command               State                                                                Ports
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+   ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-test_arbitrator_1   /usr/bin/entrypoint.sh tar ...   Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
+   test_arbitrator_1   /usr/bin/entrypoint.sh tar ...   Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
-test_td-1_1         /usr/bin/entrypoint.sh taosd     Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
+   test_td-1_1         /usr/bin/entrypoint.sh taosd     Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
-test_td-2_1         /usr/bin/entrypoint.sh taosd     Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
+   test_td-2_1         /usr/bin/entrypoint.sh taosd     Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
-```
+   ```
 
-Check dnodes with TDengine CLI:
+4. Show dnodes via TDengine CLI
 
-```bash
+```shell
 $ docker-compose exec td-1 taos -s "show dnodes"
 
-Welcome to the TDengine shell from Linux, Client Version:2.4.0.0
-Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.
-
 taos> show dnodes
-   id   |           end_point            | vnodes | cores  |   status   | role  |       create_time       |      offline reason      |
+     id      |            endpoint            | vnodes | support_vnodes |   status   |       create_time       |              note              |
 ======================================================================================================================================
-      1 | td-1:6030                      |      1 |      8 | ready      | any   | 2022-01-18 02:47:42.871 |                          |
+           1 | td-1:6030                      |      0 |             32 | ready      | 2022-08-19 07:57:29.971 |                                |
-      2 | td-2:6030                      |      0 |      8 | ready      | any   | 2022-01-18 02:47:43.518 |                          |
+           2 | td-2:6030                      |      0 |             32 | ready      | 2022-08-19 07:57:31.415 |                                |
-      0 | arbitrator:6042                |      0 |      0 | ready      | arb   | 2022-01-18 02:47:43.633 | -                        |
+           3 | td-3:6030                      |      0 |             32 | ready      | 2022-08-19 07:57:31.417 |                                |
-Query OK, 3 row(s) in set (0.000811s)
+Query OK, 3 rows in database (0.021262s)
+
 ```
 
-### Start a TDengine cluster with scaled taosadapter service
+## taosAdapter
 
-In previous use case, you could see the way to start other services built with TDengine(`taosd` as the default command). There's another important service you should know:
+1. taosAdapter is enabled by default in the TDengine container. If you want to disable it, specify the environment variable `TAOS_DISABLE_ADAPTER=true` at startup
 
-> **taosAdapter** is a TDengine’s companion tool and is a bridge/adapter between TDengine cluster and application. It provides an easy-to-use and efficient way to ingest data from data collections agents(like Telegraf, StatsD, CollectD) directly. It also provides InfluxDB/OpenTSDB compatible data ingestion interface to allow InfluxDB/OpenTSDB applications to immigrate to TDengine seamlessly.
+2. At the same time, for flexible deployment, taosAdapter can be started in a separate container
 
-`taosadapter` is running inside `tdengine` image by default, you can disable it by `TAOS_DISABLE_ADAPTER=true`. Running `taosadapter` in a separate container is like how `arbitrator` does:
+   ```docker
+   services:
+     # ...
+     adapter:
+       image: tdengine/tdengine:$VERSION
+       command: taosadapter
+   ```
 
-```yaml
+   Suppose you want to deploy multiple taosAdapters to improve throughput and provide high availability. In that case, the recommended configuration method uses a reverse proxy such as Nginx to offer a unified access entry. For specific configuration methods, please refer to the official documentation of Nginx. Here is an example:
-services:
-  # ...
-  adapter:
-    image: tdengine/tdengine:$VERSION
-    command: taosadapter
-```
 
-`taosadapter` could be scaled with docker-compose, so that you can manage the `taosadapter` nodes easily. Here is an example shows 4-`taosadapter` instances in a TDengine cluster(much like previous use cases):
+```yml
-
-```yaml
 version: "3"
 
 networks:
   inter:
-  api:
 
 services:
-  arbitrator:
-    image: tdengine/tdengine:$VERSION
-    command: tarbitrator
-    networks:
-      - inter
   td-1:
     image: tdengine/tdengine:$VERSION
     networks:
@@ -446,9 +400,6 @@ services:
     environment:
       TAOS_FQDN: "td-1"
       TAOS_FIRST_EP: "td-1"
-      TAOS_NUM_OF_MNODES: "2"
-      TAOS_REPLICA: "2"
-      TAOS_ARBITRATOR: arbitrator:6042
     volumes:
       - taosdata-td1:/var/lib/taos/
       - taoslog-td1:/var/log/taos/
@@ -459,15 +410,12 @@ services:
     environment:
       TAOS_FQDN: "td-2"
       TAOS_FIRST_EP: "td-1"
-      TAOS_NUM_OF_MNODES: "2"
-      TAOS_REPLICA: "2"
-      TAOS_ARBITRATOR: arbitrator:6042
     volumes:
       - taosdata-td2:/var/lib/taos/
       - taoslog-td2:/var/log/taos/
   adapter:
     image: tdengine/tdengine:$VERSION
-    command: taosadapter
+    entrypoint: "taosadapter"
     networks:
       - inter
     environment:
@@ -481,7 +429,6 @@ services:
       - adapter
     networks:
       - inter
-      - api
     ports:
       - 6041:6041
       - 6044:6044/udp
@@ -504,100 +451,14 @@ volumes:
   taoslog-td2:
 ```
 
-Start the cluster:
+## Deploy with docker swarm
 
-```bash
+If you want to deploy a container-based TDengine cluster on multiple hosts, you can use docker swarm. First, to establish a docker swarm cluster on these hosts, please refer to the official docker documentation.
-$ VERSION=2.4.0.0 docker-compose up -d
-Creating network "docker_inter" with the default driver
-Creating network "docker_api" with the default driver
-Creating volume "docker_taosdata-td1" with default driver
-Creating volume "docker_taoslog-td1" with default driver
-Creating volume "docker_taosdata-td2" with default driver
-Creating volume "docker_taoslog-td2" with default driver
-Creating docker_td-2_1       ... done
-Creating docker_arbitrator_1 ... done
-Creating docker_td-1_1       ... done
-Creating docker_adapter_1    ... done
-Creating docker_adapter_2    ... done
-Creating docker_adapter_3    ... done
-```
 
-It will start a TDengine cluster with two dnodes and four taosadapter instances, expose ports 6041/tcp and 6044/udp to host.
+The docker-compose file can refer to the previous section. Here is the command to start TDengine with docker swarm:
 
-`6041` is the RESTful API endpoint port, you can verify that the RESTful interface taosAdapter provides working using the `curl` command.
+```shell
-
+$ VERSION=3.0.0.0 docker stack deploy -c docker-compose.yml taos
-```bash
-$ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' 127.0.0.1:6041/rest/sql
-{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[["log","2022-01-18 04:37:42.902",16,1,1,1,10,"30",1,3,100,4096,1,3000,2,0,"us",0,"ready"]],"rows":1}
-```
-
-If you run curl in batch(here we use [hyperfine](https://github.com/sharkdp/hyperfine) - a command-line benchmarking tool), the requests are balanced into 4 adapter instances.
-
-```bash
-hyperfine -m10 'curl -u root:taosdata localhost:6041/rest/sql -d "describe log.log"'
-```
-
-View the logs with `docker-compose logs`:
-
-```bash
-$ docker-compose logs adapter
-# some logs skipped
-adapter_2     | 01/18 04:57:44.616529 00000039 TAOS_ADAPTER info "| 200 |     162.185µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=18
-adapter_1     | 01/18 04:57:44.627695 00000039 TAOS_ADAPTER info "| 200 |     145.485µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=17
-adapter_3     | 01/18 04:57:44.639165 00000040 TAOS_ADAPTER info "| 200 |     146.913µs |      172.21.0.9 | POST | /rest/sql " sessionID=17 model=web
-adapter_4     | 01/18 04:57:44.650829 00000039 TAOS_ADAPTER info "| 200 |     153.201µs |      172.21.0.9 | POST | /rest/sql " sessionID=17 model=web
-adapter_2     | 01/18 04:57:44.662422 00000039 TAOS_ADAPTER info "| 200 |     211.393µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=19
-adapter_1     | 01/18 04:57:44.673426 00000039 TAOS_ADAPTER info "| 200 |     154.714µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=18
-adapter_3     | 01/18 04:57:44.684788 00000040 TAOS_ADAPTER info "| 200 |     131.876µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=18
-adapter_4     | 01/18 04:57:44.696261 00000039 TAOS_ADAPTER info "| 200 |     162.173µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=18
-adapter_2     | 01/18 04:57:44.707414 00000039 TAOS_ADAPTER info "| 200 |     164.419µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=20
-adapter_1     | 01/18 04:57:44.720842 00000039 TAOS_ADAPTER info "| 200 |     179.374µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=19
-adapter_3     | 01/18 04:57:44.732184 00000040 TAOS_ADAPTER info "| 200 |     141.174µs |      172.21.0.9 | POST | /rest/sql " sessionID=19 model=web
-adapter_4     | 01/18 04:57:44.744024 00000039 TAOS_ADAPTER info "| 200 |     159.774µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=19
-adapter_2     | 01/18 04:57:44.773732 00000039 TAOS_ADAPTER info "| 200 |     178.993µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=21
-adapter_1     | 01/18 04:57:44.796518 00000039 TAOS_ADAPTER info "| 200 |      238.24µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=20
-adapter_3     | 01/18 04:57:44.810744 00000040 TAOS_ADAPTER info "| 200 |     176.133µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=20
-adapter_4     | 01/18 04:57:44.826395 00000039 TAOS_ADAPTER info "| 200 |     149.215µs |      172.21.0.9 | POST | /rest/sql " model=web sessionID=20
-```
-
-`6044/udp` is the [StatsD](https://github.com/statsd/statsd)-compatible port, you can verify this feature with `nc` command(usually provided by `netcat` package).
-
-```bash
-echo "foo:1|c" | nc -u -w0 127.0.0.1 6044
-```
-
-Check the result in `taos` shell with `docker-compose exec`:
-
-```bash
-$ dc exec td-1 taos
-
-Welcome to the TDengine shell from Linux, Client Version:2.4.0.0
-Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.
-
-taos> show databases;
-              name              |      created_time       |   ntables   |   vgroups   | replica | quorum |  days  |           keep           |  cache(MB)  |   blocks    |   minrows   |   maxrows   | wallevel |    fsync    | comp | cachelast | precision | update |   status   |
-====================================================================================================================================================================================================================================================================================
- log                            | 2022-01-18 04:37:42.902 |          17 |           1 |       1 |      1 |     10 | 30                       |           1 |           3 |         100 |        4096 |        1 |        3000 |    2 |         0 | us        |      0 | ready      |
- statsd                         | 2022-01-18 04:45:02.563 |           1 |           1 |       2 |      1 |     10 | 3650                     |          16 |           6 |         100 |        4096 |        1 |        3000 |    2 |         0 | ns        |      2 | ready      |
-Query OK, 2 row(s) in set (0.001838s)
-
-taos> select * from statsd.foo;
-              ts               |         value         |         metric_type          |
-=======================================================================================
- 2022-01-18 04:45:02.563422822 |                     1 | counter                      |
-Query OK, 1 row(s) in set (0.003854s)
-```
-
-Use `docker-compose up -d adapter=1 to reduce the instances to 1
-
-### Deploy TDengine cluster in Docker Swarm with `docker-compose.yml`
-
-If you use docker swarm mode, it will schedule arbitrator/taosd/taosadapter services into different hosts automatically. If you've no experience with k8s/kubernetes, this is the most convenient way to scale out the TDengine cluster with multiple hosts/servers.
-
-Use the `docker-compose.yml` file in previous use case, and deploy with `docker stack` or `docker deploy`:
-
-```bash
-$ VERSION=2.4.0 docker stack deploy -c docker-compose.yml taos
 Creating network taos_inter
 Creating network taos_api
 Creating service taos_arbitrator
@@ -607,39 +468,33 @@ Creating service taos_adapter
 Creating service taos_nginx
 ```
 
-Now you've created a TDengine cluster with multiple host servers.
+Checking status:
 
-Use `docker service` or `docker stack` to manage the cluster:
+```shell
-
-<!-- code-spell-checker:disable -->
-
-```bash
 $ docker stack ps taos
 ID                  NAME                IMAGE                     NODE                DESIRED STATE       CURRENT STATE                ERROR               PORTS
 79ni8temw59n        taos_nginx.1        nginx:latest              TM1701     Running             Running about a minute ago
-3e94u72msiyg        taos_adapter.1      tdengine/tdengine:2.4.0   TM1702     Running             Running 56 seconds ago                           
+3e94u72msiyg        taos_adapter.1      tdengine/tdengine:3.0.0.0   TM1702     Running             Running 56 seconds ago
-100amjkwzsc6        taos_td-2.1         tdengine/tdengine:2.4.0   TM1703     Running             Running about a minute ago                       
+100amjkwzsc6        taos_td-2.1         tdengine/tdengine:3.0.0.0   TM1703     Running             Running about a minute ago
-pkjehr2vvaaa        taos_td-1.1         tdengine/tdengine:2.4.0   TM1704     Running             Running 2 minutes ago                            
+pkjehr2vvaaa        taos_td-1.1         tdengine/tdengine:3.0.0.0   TM1704     Running             Running 2 minutes ago
-tpzvgpsr1qkt        taos_arbitrator.1   tdengine/tdengine:2.4.0   TM1705     Running             Running 2 minutes ago                            
+tpzvgpsr1qkt        taos_arbitrator.1   tdengine/tdengine:3.0.0.0   TM1705     Running             Running 2 minutes ago
-rvss3g5yg6fa        taos_adapter.2      tdengine/tdengine:2.4.0   TM1706     Running             Running 56 seconds ago                           
+rvss3g5yg6fa        taos_adapter.2      tdengine/tdengine:3.0.0.0   TM1706     Running             Running 56 seconds ago
-i2augxamfllf        taos_adapter.3      tdengine/tdengine:2.4.0   TM1707     Running             Running 56 seconds ago                           
+i2augxamfllf        taos_adapter.3      tdengine/tdengine:3.0.0.0   TM1707     Running             Running 56 seconds ago
-lmjyhzccpvpg        taos_adapter.4      tdengine/tdengine:2.4.0   TM1708     Running             Running 56 seconds ago
+lmjyhzccpvpg        taos_adapter.4      tdengine/tdengine:3.0.0.0   TM1708     Running             Running 56 seconds ago
 $ docker service ls
 ID                  NAME                MODE                REPLICAS            IMAGE                     PORTS
-561t4lu6nfw6        taos_adapter        replicated          4/4                 tdengine/tdengine:2.4.0   
+561t4lu6nfw6        taos_adapter        replicated          4/4                 tdengine/tdengine:3.0.0.0
-3hk5ct3q90sm        taos_arbitrator     replicated          1/1                 tdengine/tdengine:2.4.0   
+3hk5ct3q90sm        taos_arbitrator     replicated          1/1                 tdengine/tdengine:3.0.0.0
 d8qr52envqzu        taos_nginx          replicated          1/1                 nginx:latest              *:6041->6041/tcp, *:6044->6044/udp
-2isssfvjk747        taos_td-1           replicated          1/1                 tdengine/tdengine:2.4.0   
+2isssfvjk747        taos_td-1           replicated          1/1                 tdengine/tdengine:3.0.0.0
-9pzw7u02ichv        taos_td-2           replicated          1/1                 tdengine/tdengine:2.4.0
+9pzw7u02ichv        taos_td-2           replicated          1/1                 tdengine/tdengine:3.0.0.0
 ```
 
-<!-- code-spell-checker:enable -->
+From the above output, you can see two dnodes, two taosAdapters, and one Nginx reverse proxy service.
 
-It shows that there are two dnodes, one arbitrator, four taosadapter and one nginx reverse-forward service in this cluster.
+Next, we can reduce the number of taosAdapter services.
 
-You can scale down the taosadapter replicas to `1` by `docker service`:
+```shell
-
-```bash
 $ docker service scale taos_adapter=1
 taos_adapter scaled to 1
 overall progress: 1 out of 1 tasks
@@ -648,17 +503,5 @@ verify: Service converged
 
 $ docker service ls -f name=taos_adapter
 ID                  NAME                MODE                REPLICAS            IMAGE                     PORTS
-561t4lu6nfw6        taos_adapter        replicated          1/1                 tdengine/tdengine:2.4.0
+561t4lu6nfw6        taos_adapter        replicated          1/1                 tdengine/tdengine:3.0.0.0
 ```
-
-Now it remains only 1 taosadapter instance in the cluster.
-
-When you want to remove the cluster, just type:
-
-```bash
-docker stack rm taos
-```
-
-### Environment Variables
-
-When you start `tdengine` image, you can adjust the configuration of TDengine by passing environment variables on the `docker run` command line or in the docker compose file. You can use all of the environment variables that passed to taosd or taosadapter.

packaging/docker/docker-compose.yml

@@ -1,77 +0,0 @@
-version: "3"
-
-networks:
-  inter:
-  api:
-
-services:
-  arbitrator:
-    image: tdengine/tdengine:$VERSION
-    command: tarbitrator
-    networks:
-      - inter
-  td-1:
-    image: tdengine/tdengine:$VERSION
-    networks:
-      - inter
-    environment:
-      TAOS_FQDN: "td-1"
-      TAOS_FIRST_EP: "td-1"
-      TAOS_NUM_OF_MNODES: "2"
-      TAOS_REPLICA: "2"
-      TAOS_ARBITRATOR: arbitrator:6042
-    volumes:
-      - taosdata-td1:/var/lib/taos/
-      - taoslog-td1:/var/log/taos/
-  td-2:
-    image: tdengine/tdengine:$VERSION
-    networks:
-      - inter
-    environment:
-      TAOS_FQDN: "td-2"
-      TAOS_FIRST_EP: "td-1"
-      TAOS_NUM_OF_MNODES: "2"
-      TAOS_REPLICA: "2"
-      TAOS_ARBITRATOR: arbitrator:6042
-    volumes:
-      - taosdata-td2:/var/lib/taos/
-      - taoslog-td2:/var/log/taos/
-  adapter:
-    image: tdengine/tdengine:$VERSION
-    command: taosadapter
-    networks:
-      - inter
-    environment:
-      TAOS_FIRST_EP: "td-1"
-      TOAS_SECOND_EP: "td-2"
-    deploy:
-      replicas: 4
-      update_config:
-        parallelism: 4
-  nginx:
-    image: nginx
-    depends_on:
-      - adapter
-    networks:
-      - inter
-      - api
-    ports:
-      - 6041:6041
-      - 6044:6044/udp
-    command: [
-        "sh",
-        "-c",
-        "while true;
-        do curl -s http://adapter:6041/-/ping >/dev/null && break;
-        done;
-        printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}'
-        > /etc/nginx/conf.d/rest.conf;
-        printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}'
-        >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf;
-        nginx -g 'daemon off;'",
-      ]
-volumes:
-  taosdata-td1:
-  taoslog-td1:
-  taosdata-td2:
-  taoslog-td2:

packaging/docker/dockerManifest.sh

@@ -73,7 +73,7 @@ fi
 
 username="tdengine"
 
-# generate docker verison
+# generate docker version
 echo "generate ${dockerim}:${version}"
 docker manifest create -a ${dockerim}:${version} ${dockeramd64}:${version} ${dockeraarch64}:${version}
 docker manifest inspect  ${dockerim}:${version}

packaging/docker/dockerbuild.sh

@@ -74,7 +74,7 @@ do
 done
 
 
-# Check_verison()
+# Check_version()
 # {
 # }
 
@@ -102,14 +102,14 @@ scriptDir=$(dirname $(readlink -f $0))
 communityDir=${scriptDir}/../../../community
 DockerfilePath=${communityDir}/packaging/docker/
 if [ "$cloudBuild" == "y" ]; then
-  comunityArchiveDir=/nas/TDengine/v$version/cloud
+  communityArchiveDir=/nas/TDengine/v$version/cloud
   Dockerfile=${communityDir}/packaging/docker/DockerfileCloud
 else
-  comunityArchiveDir=/nas/TDengine/v$version/community
+  communityArchiveDir=/nas/TDengine/v$version/community
   Dockerfile=${communityDir}/packaging/docker/Dockerfile
 fi
 cd ${scriptDir}
-cp -f ${comunityArchiveDir}/${pkgFile}  .
+cp -f ${communityArchiveDir}/${pkgFile}  .
 
 echo "dirName=${dirName}"
 

packaging/tools/post.sh

@@ -627,9 +627,16 @@ function install_app() {
   fi
 }
 
-function install_TDengine() {
+function checkDirectory() {
-    echo -e "${GREEN}Start to install TDengine...${NC}"
+    if [ ! -d "${bin_link_dir}" ]; then
-    log_print "start to install TDengine"
+      ${csudo}mkdir -p ${bin_link_dir}
+      log_print "${bin_link_dir} directory created"
+    fi
+
+    if [ ! -d "${lib_link_dir}" ]; then
+      ${csudo}mkdir -p ${lib_link_dir}
+      log_print "${lib_link_dir} directory created"
+    fi
 
     #install log and data dir , then ln to /usr/local/taos
     ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
@@ -640,6 +647,13 @@ function install_TDengine() {
 
     ${csudo}ln -s ${log_dir} ${log_link_dir}     || :
     ${csudo}ln -s ${data_dir} ${data_link_dir}   || :
+}
+
+function install_TDengine() {
+    echo -e "${GREEN}Start to install TDengine...${NC}"
+    log_print "start to install TDengine"
+
+    checkDirectory
 
     # Install include, lib, binary and service
     install_include &&

source/client/src/clientHb.c

@@ -49,6 +49,48 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) {
+  int32_t code = 0;
+  SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo));
+  if (NULL == vgInfo) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    return code;
+  }
+  
+  vgInfo->vgVersion = rsp->vgVersion;
+  vgInfo->stateTs = rsp->stateTs;
+  vgInfo->hashMethod = rsp->hashMethod;
+  vgInfo->hashPrefix = rsp->hashPrefix;
+  vgInfo->hashSuffix = rsp->hashSuffix;
+  vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  if (NULL == vgInfo->vgHash) {
+    taosMemoryFree(vgInfo);
+    tscError("hash init[%d] failed", rsp->vgNum);
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _return;
+  }
+  
+  for (int32_t j = 0; j < rsp->vgNum; ++j) {
+    SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j);
+    if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) {
+      tscError("hash push failed, errno:%d", errno);
+      taosHashCleanup(vgInfo->vgHash);
+      taosMemoryFree(vgInfo);
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _return;
+    }
+  }
+
+_return:
+  if (code) {
+    taosHashCleanup(vgInfo->vgHash);
+    taosMemoryFreeClear(vgInfo);
+  }
+
+  *pInfo = vgInfo;
+  return code;
+}
+
 static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
   int32_t code = 0;
 
@@ -67,37 +109,22 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
     if (rsp->vgVersion < 0) {
       code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid);
     } else {
-      SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo));
+      SDBVgInfo *vgInfo = NULL;
-      if (NULL == vgInfo) {
+      code = hbGenerateVgInfoFromRsp(&vgInfo, rsp);
-        code = TSDB_CODE_OUT_OF_MEMORY;
+      if (TSDB_CODE_SUCCESS != code) {
         goto _return;
       }
 
-      vgInfo->vgVersion = rsp->vgVersion;
-      vgInfo->stateTs = rsp->stateTs;
-      vgInfo->hashMethod = rsp->hashMethod;
-      vgInfo->hashPrefix = rsp->hashPrefix;
-      vgInfo->hashSuffix = rsp->hashSuffix;
-      vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-      if (NULL == vgInfo->vgHash) {
-        taosMemoryFree(vgInfo);
-        tscError("hash init[%d] failed", rsp->vgNum);
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _return;
-      }
-
-      for (int32_t j = 0; j < rsp->vgNum; ++j) {
-        SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j);
-        if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) {
-          tscError("hash push failed, errno:%d", errno);
-          taosHashCleanup(vgInfo->vgHash);
-          taosMemoryFree(vgInfo);
-          code = TSDB_CODE_OUT_OF_MEMORY;
-          goto _return;
-        }
-      }
-
       catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, vgInfo);
+
+      if (IS_SYS_DBNAME(rsp->db)) {
+        code = hbGenerateVgInfoFromRsp(&vgInfo, rsp);
+        if (TSDB_CODE_SUCCESS != code) {
+          goto _return;
+        }
+
+        catalogUpdateDBVgInfo(pCatalog, (rsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB, rsp->uid, vgInfo);
+      }
     }
 
     if (code) {
@@ -347,7 +374,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) {
       continue;
     }
 
-    if (pRequest->killed) {
+    if (pRequest->killed || 0 == pRequest->body.queryJob) {
       releaseRequest(*rid);
       pIter = taosHashIterate(pObj->pRequests, pIter);
       continue;
@@ -491,6 +518,9 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
 
   for (int32_t i = 0; i < dbNum; ++i) {
     SDbVgVersion *db = &dbs[i];
+    tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64 ", vgVersion:%d, numOfTable:%d, startTs:%" PRId64, 
+      i, db->dbFName, db->dbId, db->vgVersion, db->numOfTable, db->stateTs);
+
     db->dbId = htobe64(db->dbId);
     db->vgVersion = htonl(db->vgVersion);
     db->numOfTable = htonl(db->numOfTable);
@@ -758,7 +788,7 @@ static void *hbThreadFunc(void *param) {
       pInfo->msgInfo.pData = buf;
       pInfo->msgInfo.len = tlen;
       pInfo->msgType = TDMT_MND_HEARTBEAT;
-      pInfo->param = strdup(pAppHbMgr->key);
+      pInfo->param = taosStrdup(pAppHbMgr->key);
       pInfo->paramFreeFp = taosMemoryFree;
       pInfo->requestId = generateRequestId();
       pInfo->requestObjRefId = 0;
@@ -826,7 +856,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
   pAppHbMgr->connKeyCnt = 0;
   pAppHbMgr->reportCnt = 0;
   pAppHbMgr->reportBytes = 0;
-  pAppHbMgr->key = strdup(key);
+  pAppHbMgr->key = taosStrdup(key);
 
   // init app info
   pAppHbMgr->pAppInstInfo = pAppInstInfo;

source/client/src/clientImpl.c

@@ -52,7 +52,7 @@ static bool validateDbName(const char* db) { return stringLengthCheck(db, TSDB_D
 static char* getClusterKey(const char* user, const char* auth, const char* ip, int32_t port) {
   char key[512] = {0};
   snprintf(key, sizeof(key), "%s:%s:%s:%d", user, auth, ip, port);
-  return strdup(key);
+  return taosStrdup(key);
 }
 
 bool chkRequestKilled(void* param) {

source/client/src/clientMsgHandler.c

@@ -162,6 +162,22 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
   if (code != TSDB_CODE_SUCCESS) {
     setErrno(pRequest, code);
+  } else {
+    struct SCatalog* pCatalog = NULL;
+    int32_t          code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
+    if (TSDB_CODE_SUCCESS == code) {
+      STscObj*             pTscObj = pRequest->pTscObj;
+
+      SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
+                               .requestId = pRequest->requestId,
+                               .requestObjRefId = pRequest->self,
+                               .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+      char dbFName[TSDB_DB_FNAME_LEN];
+      snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_INFORMATION_SCHEMA_DB);
+      catalogRefreshDBVgInfo(pCatalog, &conn, dbFName);
+      snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_PERFORMANCE_SCHEMA_DB);
+      catalogRefreshDBVgInfo(pCatalog, &conn, dbFName);
+    }  
   }
 
   if (pRequest->body.queryFp) {
@@ -418,12 +434,12 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) {
     char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(name, pInfo->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
     SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
-    colDataAppend(pColInfo, i, name, false);
+    colDataSetVal(pColInfo, i, name, false);
 
     char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE);
     pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
-    colDataAppend(pColInfo, i, value, false);
+    colDataSetVal(pColInfo, i, value, false);
   }
 
   pBlock->info.rows = numOfCfg;
@@ -456,12 +472,13 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
   (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS);
 
   int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS);
+  blockDataDestroy(pBlock);
+
   if(len != rspSize - sizeof(SRetrieveTableRsp)){
     uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, (uint64_t) (rspSize - sizeof(SRetrieveTableRsp)));
     return TSDB_CODE_TSC_INVALID_INPUT;
   }
 
-  blockDataDestroy(pBlock);
   return TSDB_CODE_SUCCESS;
 }
 

source/client/src/clientSmlLine.c

@@ -544,7 +544,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       }
     } else {
       if (currElement->colArray == NULL) {
-        currElement->colArray = taosArrayInit_s(16, sizeof(SSmlKv), 1);
+        currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
       }
       taosArrayPush(currElement->colArray, &kv);  // reserve for timestamp
     }

source/client/src/clientTmq.c

@@ -58,15 +58,14 @@ struct tmq_list_t {
 };
 
 struct tmq_conf_t {
-  char    clientId[256];
+  char           clientId[256];
-  char    groupId[TSDB_CGROUP_LEN];
+  char           groupId[TSDB_CGROUP_LEN];
-  int8_t  autoCommit;
+  int8_t         autoCommit;
-  int8_t  resetOffset;
+  int8_t         resetOffset;
-  int8_t  withTbName;
+  int8_t         withTbName;
-  int8_t  snapEnable;
+  int8_t         snapEnable;
-  int32_t snapBatchSize;
+  int32_t        snapBatchSize;
-  bool    hbBgEnable;
+  bool           hbBgEnable;
-
   uint16_t       port;
   int32_t        autoCommitInterval;
   char*          ip;
@@ -213,6 +212,7 @@ typedef struct {
 typedef struct {
   SMqCommitCbParamSet* params;
   STqOffset*           pOffset;
+  SMqClientVg*         pMqVg;
   /*char                 topicName[TSDB_TOPIC_FNAME_LEN];*/
   /*int32_t              vgId;*/
 } SMqCommitCbParam;
@@ -330,15 +330,15 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
   }
 
   if (strcmp(key, "td.connect.ip") == 0) {
-    conf->ip = strdup(value);
+    conf->ip = taosStrdup(value);
     return TMQ_CONF_OK;
   }
   if (strcmp(key, "td.connect.user") == 0) {
-    conf->user = strdup(value);
+    conf->user = taosStrdup(value);
     return TMQ_CONF_OK;
   }
   if (strcmp(key, "td.connect.pass") == 0) {
-    conf->pass = strdup(value);
+    conf->pass = taosStrdup(value);
     return TMQ_CONF_OK;
   }
   if (strcmp(key, "td.connect.port") == 0) {
@@ -346,7 +346,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     return TMQ_CONF_OK;
   }
   if (strcmp(key, "td.connect.db") == 0) {
-    /*conf->db = strdup(value);*/
+    /*conf->db = taosStrdup(value);*/
     return TMQ_CONF_OK;
   }
 
@@ -361,7 +361,7 @@ tmq_list_t* tmq_list_new() {
 int32_t tmq_list_append(tmq_list_t* list, const char* src) {
   SArray* container = &list->container;
   if (src == NULL || src[0] == 0) return -1;
-  char* topic = strdup(src);
+  char* topic = taosStrdup(src);
   if (topic[0] != '`') {
     strtolower(topic, src);
   }
@@ -422,10 +422,6 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) {
 
 static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet) {
   int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1);
-  if (ASSERT(waitingRspNum >= 0)) {
-    tscError("tmqCommitRspCountDown error:%d", waitingRspNum);
-    return;
-  }
   if (waitingRspNum == 0) {
     tmqCommitDone(pParamSet);
   }
@@ -443,6 +439,17 @@ int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) {
   }
 #endif
 
+  // there may be race condition. fix it
+  if (pBuf->pEpSet != NULL && pParam->pMqVg != NULL) {
+    SMqClientVg* pMqVg = pParam->pMqVg;
+
+    SEp* pEp = GET_ACTIVE_EP(pBuf->pEpSet);
+    SEp* pOld = GET_ACTIVE_EP(&(pMqVg->epSet));
+    uDebug("subKey:%s update the epset vgId:%d, ep:%s:%d, old ep:%s:%d", pParam->pOffset->subKey, pMqVg->vgId,
+           pEp->fqdn, pEp->port, pOld->fqdn, pOld->port);
+    pParam->pMqVg->epSet = *pBuf->pEpSet;
+  }
+
   taosMemoryFree(pParam->pOffset);
   taosMemoryFree(pBuf->pData);
   taosMemoryFree(pBuf->pEpSet);
@@ -451,7 +458,6 @@ int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) {
    * pOffset->version);*/
 
   tmqCommitRspCountDown(pParamSet);
-
   return 0;
 }
 
@@ -461,6 +467,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
+
   pOffset->val = pVg->currentOffset;
 
   int32_t groupLen = strlen(tmq->groupId);
@@ -474,11 +481,13 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
   if (code < 0) {
     return -1;
   }
+
   void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
   if (buf == NULL) {
     taosMemoryFree(pOffset);
     return -1;
   }
+
   ((SMsgHead*)buf)->vgId = htonl(pVg->vgId);
 
   void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
@@ -495,8 +504,10 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
     taosMemoryFree(buf);
     return -1;
   }
+
   pParam->params = pParamSet;
   pParam->pOffset = pOffset;
+  pParam->pMqVg = pVg;  // there may be an race condition
 
   // build send info
   SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
@@ -506,16 +517,18 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
     taosMemoryFree(pParam);
     return -1;
   }
+
   pMsgSendInfo->msgInfo = (SDataBuf){
       .pData = buf,
       .len = sizeof(SMsgHead) + len,
       .handle = NULL,
   };
 
-  tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d offset:%" PRId64, tmq->consumerId, pOffset->subKey, pVg->vgId,
+  SEp* pEp = &pVg->epSet.eps[pVg->epSet.inUse];
-           pOffset->val.version);
+  tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d offset:%" PRId64 " prev:%" PRId64 ", ep:%s:%d", tmq->consumerId,
+           pOffset->subKey, pVg->vgId, pOffset->val.version, pVg->committedOffset.version, pEp->fqdn, pEp->port);
 
-  // TODO: put into cb
+  // TODO: put into cb, the commit offset should be move to the callback function
   pVg->committedOffset = pVg->currentOffset;
 
   pMsgSendInfo->requestId = generateRequestId();
@@ -639,15 +652,16 @@ static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async,
   // init as 1 to prevent concurrency issue
   pParamSet->waitingRspNum = 1;
 
-  for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
+  int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
+  tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics);
+
+  for (int32_t i = 0; i < numOfTopics; i++) {
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
 
     int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
     for (int32_t j = 0; j < numOfVgroups; j++) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
       if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) {
-        tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, current %" PRId64 ", committed %" PRId64, tmq->consumerId,
-                 pTopic->topicName, pVg->vgId, pVg->currentOffset.version, pVg->committedOffset.version);
         if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) {
           continue;
         }
@@ -809,7 +823,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
       int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
       *pRefId = pTmq->refId;
 
-      tscDebug("consumer:0x%" PRIx64 " next retrieve ep from mnode in 1s", pTmq->consumerId);
+      tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
       taosTmrReset(tmqAssignAskEpTask, 1000, pRefId, tmqMgmt.timer, &pTmq->epTimer);
     } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
       tmqCommitInner(pTmq, NULL, 1, 1, pTmq->commitCb, pTmq->commitCbUserParam);
@@ -817,7 +831,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
       int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t));
       *pRefId = pTmq->refId;
 
-      tscDebug("consumer:0x%" PRIx64 " next commit to mnode in %.2fs", pTmq->consumerId,
+      tscDebug("consumer:0x%" PRIx64 " commit to vnode(s) in %.2fs", pTmq->consumerId,
                pTmq->autoCommitInterval / 1000.0);
       taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, pRefId, tmqMgmt.timer, &pTmq->commitTimer);
     } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) {
@@ -981,7 +995,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->qall = taosAllocateQall();
   pTmq->delayedTask = taosOpenQueue();
 
-  if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL) {
+  if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL ||
+      conf->groupId[0] == 0) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     tscError("consumer:0x%" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(),
              pTmq->groupId);
@@ -1060,7 +1075,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   SCMSubscribeReq req = {0};
   int32_t         code = 0;
 
-  tscDebug("consumer:0x%" PRIx64 " tmq subscribe start, numOfTopic %d", tmq->consumerId, sz);
+  tscDebug("consumer:0x%" PRIx64 " subscribe %d topics", tmq->consumerId, sz);
 
   req.consumerId = tmq->consumerId;
   tstrncpy(req.clientId, tmq->clientId, 256);
@@ -1083,7 +1098,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
     }
 
     tNameExtractFullName(&name, topicFName);
-    tscDebug("consumer:0x%" PRIx64 ", subscribe topic: %s", tmq->consumerId, topicFName);
+    tscDebug("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName);
 
     taosArrayPush(req.topicNames, &topicFName);
   }
@@ -1394,7 +1409,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) {
   }
 
   atomic_store_32(&tmq->epoch, epoch);
-  tscDebug("consumer:0x%" PRIx64 ", update topic info completed", tmq->consumerId);
+  tscDebug("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId);
   return set;
 }
 
@@ -1544,7 +1559,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) {
   sendInfo->msgType = TDMT_MND_TMQ_ASK_EP;
 
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
-  tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d", tmq->consumerId, async);
+  tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d, reqId:0x%" PRIx64, tmq->consumerId, async,
+           tmq->consumerId);
 
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
@@ -1559,25 +1575,20 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) {
 }
 
 void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) {
-  /*strcpy(pReq->topic, pTopic->topicName);*/
-  /*strcpy(pReq->cgroup, tmq->groupId);*/
-
   int32_t groupLen = strlen(tmq->groupId);
   memcpy(pReq->subKey, tmq->groupId, groupLen);
   pReq->subKey[groupLen] = TMQ_SEPARATOR;
   strcpy(pReq->subKey + groupLen + 1, pTopic->topicName);
 
   pReq->withTbName = tmq->withTbName;
-  pReq->timeout = timeout;
   pReq->consumerId = tmq->consumerId;
+  pReq->timeout = timeout;
   pReq->epoch = tmq->epoch;
   /*pReq->currentOffset = reqOffset;*/
   pReq->reqOffset = pVg->currentOffset;
-  pReq->reqId = generateRequestId();
-
-  pReq->useSnapshot = tmq->useSnapshot;
-
   pReq->head.vgId = pVg->vgId;
+  pReq->useSnapshot = tmq->useSnapshot;
+  pReq->reqId = generateRequestId();
 }
 
 SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
@@ -1627,6 +1638,76 @@ SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   return pRspObj;
 }
 
+static int32_t handleErrorBeforePoll(SMqClientVg* pVg, tmq_t* pTmq) {
+  atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+  tsem_post(&pTmq->rspSem);
+  return -1;
+}
+
+static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg, int64_t timeout) {
+  SMqPollReq req = {0};
+  tmqBuildConsumeReqImpl(&req, pTmq, timeout, pTopic, pVg);
+
+  int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req);
+  if (msgSize < 0) {
+    return handleErrorBeforePoll(pVg, pTmq);
+  }
+
+  char* msg = taosMemoryCalloc(1, msgSize);
+  if (NULL == msg) {
+    return handleErrorBeforePoll(pVg, pTmq);
+  }
+
+  if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) {
+    taosMemoryFree(msg);
+    return handleErrorBeforePoll(pVg, pTmq);
+  }
+
+  SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
+  if (pParam == NULL) {
+    taosMemoryFree(msg);
+    return handleErrorBeforePoll(pVg, pTmq);
+  }
+
+  pParam->refId = pTmq->refId;
+  pParam->epoch = pTmq->epoch;
+  pParam->pVg = pVg;  // pVg may be released,fix it
+  pParam->pTopic = pTopic;
+  pParam->vgId = pVg->vgId;
+
+  SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
+  if (sendInfo == NULL) {
+    taosMemoryFree(pParam);
+    taosMemoryFree(msg);
+    return handleErrorBeforePoll(pVg, pTmq);
+  }
+
+  sendInfo->msgInfo = (SDataBuf){
+      .pData = msg,
+      .len = msgSize,
+      .handle = NULL,
+  };
+
+  sendInfo->requestId = req.reqId;
+  sendInfo->requestObjRefId = 0;
+  sendInfo->param = pParam;
+  sendInfo->fp = tmqPollCb;
+  sendInfo->msgType = TDMT_VND_TMQ_CONSUME;
+
+  int64_t transporterId = 0;
+  char    offsetFormatBuf[80];
+  tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->currentOffset);
+
+  tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:0x%" PRIx64,
+           pTmq->consumerId, pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId);
+  asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
+
+  pVg->pollCnt++;
+  pTmq->pollCnt++;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 // broadcast the poll request to all related vnodes
 int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
   int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
@@ -1634,7 +1715,9 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
 
   for (int i = 0; i < numOfTopics; i++) {
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-    for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
+    int32_t         numOfVg = taosArrayGetSize(pTopic->vgs);
+
+    for (int j = 0; j < numOfVg; j++) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
       int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus == TMQ_VG_STATUS__WAIT) {
@@ -1653,77 +1736,10 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
       }
 
       atomic_store_32(&pVg->vgSkipCnt, 0);
-
+      int32_t code = doTmqPollImpl(tmq, pTopic, pVg, timeout);
-      SMqPollReq req = {0};
+      if (code != TSDB_CODE_SUCCESS) {
-      tmqBuildConsumeReqImpl(&req, tmq, timeout, pTopic, pVg);
+        return code;
-      int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req);
-      if (msgSize < 0) {
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
-        return -1;
       }
-
-      char* msg = taosMemoryCalloc(1, msgSize);
-      if (NULL == msg) {
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
-        return -1;
-      }
-
-      if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) {
-        taosMemoryFree(msg);
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
-        return -1;
-      }
-
-      SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
-      if (pParam == NULL) {
-        taosMemoryFree(msg);
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
-        return -1;
-      }
-
-      pParam->refId = tmq->refId;
-      pParam->epoch = tmq->epoch;
-
-      pParam->pVg = pVg;
-      pParam->pTopic = pTopic;
-      pParam->vgId = pVg->vgId;
-
-      SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
-      if (sendInfo == NULL) {
-        taosMemoryFree(msg);
-        taosMemoryFree(pParam);
-        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
-        return -1;
-      }
-
-      sendInfo->msgInfo = (SDataBuf){
-          .pData = msg,
-          .len = msgSize,
-          .handle = NULL,
-      };
-
-      sendInfo->requestId = req.reqId;
-      sendInfo->requestObjRefId = 0;
-      sendInfo->param = pParam;
-      sendInfo->fp = tmqPollCb;
-      sendInfo->msgType = TDMT_VND_TMQ_CONSUME;
-
-      int64_t transporterId = 0;
-
-      char offsetFormatBuf[80];
-      tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffset);
-
-      tscDebug("consumer:0x%" PRIx64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:0x%" PRIx64,
-               tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
-      asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
-
-      pVg->pollCnt++;
-      tmq->pollCnt++;
     }
   }
 
@@ -1751,28 +1767,28 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset)
 }
 
 void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
+  tscDebug("consumer:0x%" PRIx64 " start to handle the rsp", tmq->consumerId);
+
   while (1) {
     SMqRspWrapper* rspWrapper = NULL;
     taosGetQitem(tmq->qall, (void**)&rspWrapper);
+
     if (rspWrapper == NULL) {
       taosReadAllQitems(tmq->mqueue, tmq->qall);
       taosGetQitem(tmq->qall, (void**)&rspWrapper);
 
       if (rspWrapper == NULL) {
-        /*tscDebug("consumer %" PRId64 " mqueue empty", tmq->consumerId);*/
         return NULL;
       }
     }
 
-    tscDebug("consumer:0x%" PRIx64 " handle rsp %p", tmq->consumerId, rspWrapper);
-
     if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) {
       taosFreeQitem(rspWrapper);
       terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
       return NULL;
     } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
-      tscDebug("consumer:0x%" PRIx64 " actual process poll rsp", tmq->consumerId);
+      tscDebug("consumer:0x%" PRIx64 " process poll rsp", tmq->consumerId);
       /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
       int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
       if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
@@ -1799,6 +1815,9 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
     } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
       SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
       int32_t            consumerEpoch = atomic_load_32(&tmq->epoch);
+
+      tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId);
+
       if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) {
         SMqClientVg* pVg = pollRspWrapper->vgHandle;
         /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset,
@@ -1863,7 +1882,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
   void*   rspObj;
   int64_t startTime = taosGetTimestampMs();
 
-  tscDebug("consumer:0x%" PRIx64 ", start poll at %" PRId64, tmq->consumerId, startTime);
+  tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64, tmq->consumerId, startTime);
 
 #if 0
   tmqHandleAllDelayedTask(tmq);
@@ -1876,7 +1895,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
 
   // in no topic status, delayed task also need to be processed
   if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) {
-    tscDebug("consumer:0x%" PRIx64 ", poll return since consumer status is init", tmq->consumerId);
+    tscDebug("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId);
     return NULL;
   }
 
@@ -1902,25 +1921,25 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
 
     rspObj = tmqHandleAllRsp(tmq, timeout, false);
     if (rspObj) {
-      tscDebug("consumer:0x%" PRIx64 ", return rsp %p", tmq->consumerId, rspObj);
+      tscDebug("consumer:0x%" PRIx64 " return rsp %p", tmq->consumerId, rspObj);
       return (TAOS_RES*)rspObj;
     } else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
-      tscDebug("consumer:0x%" PRIx64 ", return null since no committed offset", tmq->consumerId);
+      tscDebug("consumer:0x%" PRIx64 " return null since no committed offset", tmq->consumerId);
       return NULL;
     }
 
     if (timeout != -1) {
       int64_t currentTime = taosGetTimestampMs();
-      int64_t passedTime = currentTime - startTime;
+      int64_t elapsedTime = currentTime - startTime;
-      if (passedTime > timeout) {
+      if (elapsedTime > timeout) {
-        tscDebug("consumer:0x%" PRIx64 ", (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64,
+        tscDebug("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64,
                  tmq->consumerId, tmq->epoch, startTime, currentTime);
         return NULL;
       }
       /*tscInfo("consumer:0x%" PRIx64 ", (epoch %d) wait, start time %" PRId64 ", current time %" PRId64*/
       /*", left time %" PRId64,*/
-      /*tmq->consumerId, tmq->epoch, startTime, currentTime, (timeout - passedTime));*/
+      /*tmq->consumerId, tmq->epoch, startTime, currentTime, (timeout - elapsedTime));*/
-      tsem_timewait(&tmq->rspSem, (timeout - passedTime));
+      tsem_timewait(&tmq->rspSem, (timeout - elapsedTime));
     } else {
       // use tsem_timewait instead of tsem_wait to avoid unexpected stuck
       tsem_timewait(&tmq->rspSem, 1000);

source/client/test/clientTests.cpp

@@ -912,7 +912,7 @@ TEST(clientCase, subscription_test) {
   tmq_conf_t* conf = tmq_conf_new();
   tmq_conf_set(conf, "enable.auto.commit", "true");
   tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
-  tmq_conf_set(conf, "group.id", "newabcdefgjhijlm__");
+  tmq_conf_set(conf, "group.id", "cgrpName");
   tmq_conf_set(conf, "td.connect.user", "root");
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
   tmq_conf_set(conf, "auto.offset.reset", "earliest");

source/common/src/tdatablock.c

@@ -64,13 +64,17 @@ int32_t getJsonValueLen(const char* data) {
   return dataLen;
 }
 
-int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) {
+int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) {
+  return colDataSetVal(pColumnInfoData, rowIndex, pData, isNull);
+}
+
+int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull) {
   if (isNull) {
     // There is a placehold for each NULL value of binary or nchar type.
     if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-      pColumnInfoData->varmeta.offset[currentRow] = -1;  // it is a null value of VAR type.
+      pColumnInfoData->varmeta.offset[rowIndex] = -1;  // it is a null value of VAR type.
     } else {
-      colDataSetNull_f_s(pColumnInfoData, currentRow);
+      colDataSetNull_f_s(pColumnInfoData, rowIndex);
     }
 
     pColumnInfoData->hasNull = true;
@@ -95,6 +99,9 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
 
       while (newSize < pAttr->length + dataLen) {
         newSize = newSize * 1.5;
+        if (newSize > UINT32_MAX) {
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
       }
 
       char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize);
@@ -107,12 +114,12 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
     }
 
     uint32_t len = pColumnInfoData->varmeta.length;
-    pColumnInfoData->varmeta.offset[currentRow] = len;
+    pColumnInfoData->varmeta.offset[rowIndex] = len;
 
     memmove(pColumnInfoData->pData + len, pData, dataLen);
     pColumnInfoData->varmeta.length += dataLen;
   } else {
-    memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes);
+    memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex, pData, pColumnInfoData->info.bytes);
   }
 
   return 0;
@@ -173,7 +180,7 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren
   }
 }
 
-int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
+int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
                             uint32_t numOfRows) {
   int32_t len = pColumnInfoData->info.bytes;
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
@@ -513,10 +520,10 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
       }
 
       if (isNull) {
-        colDataAppendNULL(pDstCol, j - startIndex);
+        colDataSetNULL(pDstCol, j - startIndex);
       } else {
         char* p = colDataGetData(pColData, j);
-        colDataAppend(pDstCol, j - startIndex, p, false);
+        colDataSetVal(pDstCol, j - startIndex, p, false);
       }
     }
   }
@@ -619,6 +626,33 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
   return TSDB_CODE_SUCCESS;
 }
 
+static bool colDataIsNNull(const SColumnInfoData* pColumnInfoData, int32_t startIndex,
+                                          uint32_t nRows) {
+  if (!pColumnInfoData->hasNull) {
+    return false;
+  }
+
+  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+    for (int32_t i = startIndex; i < nRows; ++i) {
+      if (!colDataIsNull_var(pColumnInfoData, i)) {
+        return false;
+      }
+    }
+  } else {
+    if (pColumnInfoData->nullbitmap == NULL) {
+      return false;
+    }
+
+    for (int32_t i = startIndex; i < nRows; ++i) {
+      if (!colDataIsNull_f(pColumnInfoData->nullbitmap, i)) {
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
 // todo remove this
 int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) {
   pBlock->info.rows = *(int32_t*)buf;
@@ -661,7 +695,7 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity)
       }
     }
 
-    if (!colDataIsNNull_s(pCol, 0, pBlock->info.rows)) {
+    if (!colDataIsNNull(pCol, 0, pBlock->info.rows)) {
       memcpy(pCol->pData, pStart, colLength);
     }
 
@@ -783,13 +817,13 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co
     SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, i);
 
     if (pSrc->hasNull && colDataIsNull(pSrc, pSrcBlock->info.rows, tupleIndex, pSrcBlock->pBlockAgg[i])) {
-      code = colDataAppend(pDst, numOfRows, NULL, true);
+      code = colDataSetVal(pDst, numOfRows, NULL, true);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
       }
     } else {
       char* p = colDataGetData(pSrc, tupleIndex);
-      code = colDataAppend(pDst, numOfRows, p, false);
+      code = colDataSetVal(pDst, numOfRows, p, false);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
       }
@@ -1033,6 +1067,7 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock*
     offset += pInfo->pColData->info.bytes;
   }
 
+  taosMemoryFree(buf);
   return phelper;
 }
 
@@ -1413,7 +1448,7 @@ SSDataBlock* blockCopyOneRow(const SSDataBlock* pDataBlock, int32_t rowIdx) {
     SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
     void*            pData = colDataGetData(pSrc, rowIdx);
     bool             isNull = colDataIsNull(pSrc, pDataBlock->info.rows, rowIdx, NULL);
-    colDataAppend(pDst, 0, pData, isNull);
+    colDataSetVal(pDst, 0, pData, isNull);
   }
 
   pBlock->info.rows = 1;
@@ -1644,7 +1679,7 @@ static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
   }
 }
 
-int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
+int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) {
   if (n == 0) {
     return TSDB_CODE_SUCCESS;
   }
@@ -2370,7 +2405,11 @@ _end:
   taosArrayDestroy(pVals);
   if (terrno != 0) {
     *ppReq = NULL;
-    if (pReq) tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE);
+    if (pReq) {
+      tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE);
+      taosMemoryFreeClear(pReq);
+    }
+
     return TSDB_CODE_FAILED;
   }
   *ppReq = pReq;
@@ -2528,7 +2567,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
   pStart += sizeof(uint64_t);
 
   if (pBlock->pDataBlock == NULL) {
-    pBlock->pDataBlock = taosArrayInit_s(numOfCols, sizeof(SColumnInfoData), numOfCols);
+    pBlock->pDataBlock = taosArrayInit_s(sizeof(SColumnInfoData), numOfCols);
   }
 
   for (int32_t i = 0; i < numOfCols; ++i) {

source/common/src/tdataformat.c

@@ -139,7 +139,10 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
           nkv += tPutI16v(NULL, -pTColumn->colId);
           nIdx++;
         } else {
-          ASSERT(0);
+          if (ASSERTS(0, "invalid input")) {
+            code = TSDB_CODE_INVALID_PARA;
+            goto _exit;
+          }
         }
 
         pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL;
@@ -176,8 +179,10 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
       ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp;
       break;
     default:
-      ASSERT(0);
+      if (ASSERTS(0, "impossible")) {
-      break;
+        code = TSDB_CODE_INVALID_PARA;
+        goto _exit;
+      }
   }
   if (maxIdx <= UINT8_MAX) {
     nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv;
@@ -306,8 +311,10 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
         pv = pf + pTSchema->flen;
         break;
       default:
-        ASSERT(0);
+        if (ASSERTS(0, "impossible")) {
-        break;
+          code = TSDB_CODE_INVALID_PARA;
+          goto _exit;
+        }
     }
 
     if (pb) {
@@ -370,7 +377,7 @@ _exit:
   return code;
 }
 
-void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
+int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
   ASSERT(iCol < pTSchema->numOfCols);
   ASSERT(pRow->sver == pTSchema->version);
 
@@ -381,17 +388,17 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
     pColVal->type = pTColumn->type;
     pColVal->flag = CV_FLAG_VALUE;
     memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY));
-    return;
+    return 0;
   }
 
   if (pRow->flag == HAS_NONE) {
     *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
-    return;
+    return 0;
   }
 
   if (pRow->flag == HAS_NULL) {
     *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type);
-    return;
+    return 0;
   }
 
   if (pRow->flag >> 4) {  // KV Row
@@ -440,7 +447,7 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
             memcpy(&pColVal->value.val, pData, pTColumn->bytes);
           }
         }
-        return;
+        return 0;
       } else if (TABS(cid) < pTColumn->colId) {
         lidx = mid + 1;
       } else {
@@ -492,16 +499,16 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
           pv = pf + pTSchema->flen;
           break;
         default:
-          ASSERT(0);
+          ASSERTS(0, "invalid row format");
-          break;
+          return TSDB_CODE_IVLD_DATA_FMT;
       }
 
       if (bv == BIT_FLG_NONE) {
         *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
-        return;
+        return 0;
       } else if (bv == BIT_FLG_NULL) {
         *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type);
-        return;
+        return 0;
       }
 
       pColVal->cid = pTColumn->colId;
@@ -520,6 +527,8 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
       }
     }
   }
+
+  return 0;
 }
 
 void tRowDestroy(SRow *pRow) {
@@ -710,7 +719,6 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) {
 _exit:
   if (code) {
     *ppIter = NULL;
-    if (pIter) taosMemoryFree(pIter);
   } else {
     *ppIter = pIter;
   }
@@ -929,8 +937,8 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *
       pv = pf + pTSchema->flen;
       break;
     default:
-      ASSERT(0);
+      ASSERTS(0, "Invalid row flag");
-      break;
+      return TSDB_CODE_IVLD_DATA_FMT;
   }
 
   while (pColData) {
@@ -954,8 +962,8 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *
               bv = GET_BIT2(pb, iTColumn - 1);
               break;
             default:
-              ASSERT(0);
+              ASSERTS(0, "Invalid row flag");
-              break;
+              return TSDB_CODE_IVLD_DATA_FMT;
           }
 
           if (bv == BIT_FLG_NONE) {
@@ -1045,7 +1053,8 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo
           } else if (pRow->flag & KV_FLG_BIG) {
             pData = pv + ((uint32_t *)pKVIdx->idx)[iCol];
           } else {
-            ASSERT(0);
+            ASSERTS(0, "Invalid KV row format");
+            return TSDB_CODE_IVLD_DATA_FMT;
           }
 
           int16_t cid;
@@ -1579,7 +1588,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
   int32_t code = 0;
 
   if (IS_VAR_DATA_TYPE(pColData->type)) {
-    code = tRealloc((uint8_t **)(&pColData->aOffset), (pColData->nVal + 1) << 2);
+    code = tRealloc((uint8_t **)(&pColData->aOffset), ((int64_t)(pColData->nVal + 1)) << 2);
     if (code) goto _exit;
     pColData->aOffset[pColData->nVal] = pColData->nData;
 
@@ -1980,8 +1989,7 @@ static FORCE_INLINE int32_t tColDataUpdateValue20(SColData *pColData, uint8_t *p
   return 0;
 }
 static FORCE_INLINE int32_t tColDataUpdateValue30(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
-  uint8_t bv = GET_BIT1(pColData->pBitMap, pColData->nVal - 1);
+  if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) {  // NONE ==> VALUE
-  if (bv == 0) {  // NONE == > VALUE
     pColData->numOfNone--;
     pColData->nVal--;
     if (pColData->numOfNone) {
@@ -1990,7 +1998,7 @@ static FORCE_INLINE int32_t tColDataUpdateValue30(SColData *pColData, uint8_t *p
       pColData->flag = HAS_NULL;
       return tColDataAppendValue20(pColData, pData, nData);
     }
-  } else if (forward) {  // NULL == > VALUE
+  } else if (forward) {  // NULL ==> VALUE
     pColData->numOfNull--;
     pColData->nVal--;
     if (pColData->numOfNull) {
@@ -2015,36 +2023,43 @@ static FORCE_INLINE int32_t tColDataUpdateValue32(SColData *pColData, uint8_t *p
   return 0;
 }
 static FORCE_INLINE int32_t tColDataUpdateValue40(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
-  if (forward) {
+  if (forward) {  // VALUE ==> VALUE
-    pColData->numOfValue--;
     pColData->nVal--;
-    if (pColData->numOfValue) {
+    if (IS_VAR_DATA_TYPE(pColData->type)) {
-      return tColDataAppendValue40(pColData, pData, nData);
+      pColData->nData = pColData->aOffset[pColData->nVal];
     } else {
-      pColData->flag = 0;
+      pColData->nData -= TYPE_BYTES[pColData->type];
-      return tColDataAppendValue00(pColData, pData, nData);
     }
+    return tColDataPutValue(pColData, pData, nData);
   }
   return 0;
 }
 static FORCE_INLINE int32_t tColDataUpdateValue42(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
-  if (forward) {
+  if (forward) {  // VALUE ==> NULL
     pColData->numOfValue--;
     pColData->nVal--;
     if (pColData->numOfValue) {
+      if (IS_VAR_DATA_TYPE(pColData->type)) {
+        pColData->nData = pColData->aOffset[pColData->nVal];
+      } else {
+        pColData->nData -= TYPE_BYTES[pColData->type];
+      }
       return tColDataAppendValue42(pColData, pData, nData);
     } else {
       pColData->flag = 0;
+      pColData->nData = 0;
       return tColDataAppendValue02(pColData, pData, nData);
     }
   }
   return 0;
 }
 static FORCE_INLINE int32_t tColDataUpdateValue50(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
-  uint8_t bv = GET_BIT1(pColData->pBitMap, pColData->nVal - 1);
+  if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) {  // NONE ==> VALUE
-  if (bv == 0) {  // NONE ==> VALUE
     pColData->numOfNone--;
     pColData->nVal--;
+    if (!IS_VAR_DATA_TYPE(pColData->type)) {
+      pColData->nData -= TYPE_BYTES[pColData->type];
+    }
     if (pColData->numOfNone) {
       return tColDataAppendValue50(pColData, pData, nData);
     } else {
@@ -2052,35 +2067,42 @@ static FORCE_INLINE int32_t tColDataUpdateValue50(SColData *pColData, uint8_t *p
       return tColDataAppendValue40(pColData, pData, nData);
     }
   } else if (forward) {  // VALUE ==> VALUE
-    pColData->numOfValue--;
     pColData->nVal--;
-    if (pColData->numOfValue) {
+    if (IS_VAR_DATA_TYPE(pColData->type)) {
-      return tColDataAppendValue50(pColData, pData, nData);
+      pColData->nData = pColData->aOffset[pColData->nVal];
     } else {
-      pColData->flag = HAS_NONE;
+      pColData->nData -= TYPE_BYTES[pColData->type];
-      return tColDataAppendValue10(pColData, pData, nData);
     }
+    return tColDataPutValue(pColData, pData, nData);
   }
   return 0;
 }
 static FORCE_INLINE int32_t tColDataUpdateValue52(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
-  uint8_t bv = GET_BIT1(pColData->pBitMap, pColData->nVal - 1);
+  if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) {  // NONE ==> NULL
-  if (bv == 0) {  // NONE ==> NULL
     pColData->numOfNone--;
     pColData->nVal--;
+    if (!IS_VAR_DATA_TYPE(pColData->type)) {
+      pColData->nData -= TYPE_BYTES[pColData->type];
+    }
     if (pColData->numOfNone) {
       return tColDataAppendValue52(pColData, pData, nData);
     } else {
-      pColData->flag &= ~HAS_NONE;
+      pColData->flag = HAS_VALUE;
       return tColDataAppendValue42(pColData, pData, nData);
     }
   } else if (forward) {  // VALUE ==> NULL
     pColData->numOfValue--;
     pColData->nVal--;
     if (pColData->numOfValue) {
+      if (IS_VAR_DATA_TYPE(pColData->type)) {
+        pColData->nData = pColData->aOffset[pColData->nVal];
+      } else {
+        pColData->nData -= TYPE_BYTES[pColData->type];
+      }
       return tColDataAppendValue52(pColData, pData, nData);
     } else {
-      pColData->flag &= ~HAS_VALUE;
+      pColData->flag = HAS_NONE;
+      pColData->nData = 0;
       return tColDataAppendValue12(pColData, pData, nData);
     }
   }
@@ -2091,6 +2113,9 @@ static FORCE_INLINE int32_t tColDataUpdateValue60(SColData *pColData, uint8_t *p
     if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) {  // NULL ==> VALUE
       pColData->numOfNull--;
       pColData->nVal--;
+      if (!IS_VAR_DATA_TYPE(pColData->type)) {
+        pColData->nData -= TYPE_BYTES[pColData->type];
+      }
       if (pColData->numOfNull) {
         return tColDataAppendValue60(pColData, pData, nData);
       } else {
@@ -2098,14 +2123,13 @@ static FORCE_INLINE int32_t tColDataUpdateValue60(SColData *pColData, uint8_t *p
         return tColDataAppendValue40(pColData, pData, nData);
       }
     } else {  // VALUE ==> VALUE
-      pColData->numOfValue--;
       pColData->nVal--;
-      if (pColData->numOfValue) {
+      if (IS_VAR_DATA_TYPE(pColData->type)) {
-        return tColDataAppendValue60(pColData, pData, nData);
+        pColData->nData = pColData->aOffset[pColData->nVal];
       } else {
-        pColData->flag = HAS_NULL;
+        pColData->nData -= TYPE_BYTES[pColData->type];
-        return tColDataAppendValue20(pColData, pData, nData);
       }
+      return tColDataPutValue(pColData, pData, nData);
     }
   }
   return 0;
@@ -2115,9 +2139,15 @@ static FORCE_INLINE int32_t tColDataUpdateValue62(SColData *pColData, uint8_t *p
     pColData->numOfValue--;
     pColData->nVal--;
     if (pColData->numOfValue) {
+      if (IS_VAR_DATA_TYPE(pColData->type)) {
+        pColData->nData = pColData->aOffset[pColData->nVal];
+      } else {
+        pColData->nData -= TYPE_BYTES[pColData->type];
+      }
       return tColDataAppendValue62(pColData, pData, nData);
     } else {
       pColData->flag = HAS_NULL;
+      pColData->nData = 0;
       return tColDataAppendValue20(pColData, pData, nData);
     }
   }
@@ -2130,38 +2160,44 @@ static FORCE_INLINE int32_t tColDataUpdateValue70(SColData *pColData, uint8_t *p
   if (bv == 0) {  // NONE ==> VALUE
     pColData->numOfNone--;
     pColData->nVal--;
+    if (!IS_VAR_DATA_TYPE(pColData->type)) {
+      pColData->nData -= TYPE_BYTES[pColData->type];
+    }
     if (pColData->numOfNone) {
       return tColDataAppendValue70(pColData, pData, nData);
     } else {
       for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
         SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) - 1);
       }
-      pColData->flag &= ~HAS_NONE;
+      pColData->flag = (HAS_VALUE | HAS_NULL);
       return tColDataAppendValue60(pColData, pData, nData);
     }
   } else if (bv == 1) {  // NULL ==> VALUE
     if (forward) {
       pColData->numOfNull--;
       pColData->nVal--;
+      if (!IS_VAR_DATA_TYPE(pColData->type)) {
+        pColData->nData -= TYPE_BYTES[pColData->type];
+      }
       if (pColData->numOfNull) {
         return tColDataAppendValue70(pColData, pData, nData);
       } else {
         for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
           SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) ? 1 : 0);
         }
-        pColData->flag &= ~HAS_NULL;
+        pColData->flag = (HAS_VALUE | HAS_NONE);
         return tColDataAppendValue50(pColData, pData, nData);
       }
     }
   } else if (bv == 2) {  // VALUE ==> VALUE
     if (forward) {
-      pColData->numOfValue--;
       pColData->nVal--;
-      if (pColData->numOfValue) {
+      if (IS_VAR_DATA_TYPE(pColData->type)) {
-        return tColDataAppendValue70(pColData, pData, nData);
+        pColData->nData = pColData->aOffset[pColData->nVal];
       } else {
-        return tColDataPutValue(pColData, pData, nData);
+        pColData->nData -= TYPE_BYTES[pColData->type];
       }
+      return tColDataPutValue(pColData, pData, nData);
     }
   } else {
     ASSERT(0);
@@ -2170,29 +2206,37 @@ static FORCE_INLINE int32_t tColDataUpdateValue70(SColData *pColData, uint8_t *p
 }
 static int32_t tColDataUpdateValue72(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
   uint8_t bv = GET_BIT2(pColData->pBitMap, pColData->nVal - 1);
-  ASSERT(bv < 3);
   if (bv == 0) {  // NONE ==> NULL
     pColData->numOfNone--;
     pColData->nVal--;
+    if (!IS_VAR_DATA_TYPE(pColData->type)) {
+      pColData->nData -= TYPE_BYTES[pColData->type];
+    }
     if (pColData->numOfNone) {
       return tColDataAppendValue72(pColData, pData, nData);
     } else {
       for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
         SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) - 1);
       }
-      pColData->flag &= ~HAS_NONE;
+      pColData->flag = (HAS_VALUE | HAS_NULL);
       return tColDataAppendValue62(pColData, pData, nData);
     }
   } else if (bv == 2 && forward) {  // VALUE ==> NULL
     pColData->numOfValue--;
     pColData->nVal--;
     if (pColData->numOfValue) {
+      if (IS_STR_DATA_TYPE(pColData->type)) {
+        pColData->nData = pColData->aOffset[pColData->nVal];
+      } else {
+        pColData->nData -= TYPE_BYTES[pColData->type];
+      }
       return tColDataAppendValue72(pColData, pData, nData);
     } else {
       for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
         SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal));
       }
-      pColData->flag &= ~HAS_VALUE;
+      pColData->flag = (HAS_NULL | HAS_NONE);
+      pColData->nData = 0;
       return tColDataAppendValue32(pColData, pData, nData);
     }
   }
@@ -2312,35 +2356,25 @@ void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
 }
 
 uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) {
-  uint8_t v;
   switch (pColData->flag) {
     case HAS_NONE:
-      v = 0;
+      return 0;
-      break;
     case HAS_NULL:
-      v = 1;
+      return 1;
-      break;
     case (HAS_NULL | HAS_NONE):
-      v = GET_BIT1(pColData->pBitMap, iVal);
+      return GET_BIT1(pColData->pBitMap, iVal);
-      break;
     case HAS_VALUE:
-      v = 2;
+      return 2;
-      break;
     case (HAS_VALUE | HAS_NONE):
-      v = GET_BIT1(pColData->pBitMap, iVal);
+      return (GET_BIT1(pColData->pBitMap, iVal)) ? 2 : 0;
-      if (v) v = 2;
-      break;
     case (HAS_VALUE | HAS_NULL):
-      v = GET_BIT1(pColData->pBitMap, iVal) + 1;
+      return GET_BIT1(pColData->pBitMap, iVal) + 1;
-      break;
     case (HAS_VALUE | HAS_NULL | HAS_NONE):
-      v = GET_BIT2(pColData->pBitMap, iVal);
+      return GET_BIT2(pColData->pBitMap, iVal);
-      break;
     default:
-      ASSERT(0);
+      ASSERTS(0, "not possible");
-      break;
+      return 0;
   }
-  return v;
 }
 
 int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg) {

source/common/src/tglobal.c

@@ -61,7 +61,7 @@ int32_t tsHeartbeatInterval = 1000;
 int32_t tsHeartbeatTimeout = 20 * 1000;
 
 // vnode
-int64_t tsVndCommitMaxIntervalMs = 60 * 1000;
+int64_t tsVndCommitMaxIntervalMs = 600 * 1000;
 
 // monitor
 bool     tsEnableMonitor = true;

source/common/src/tmisce.c

@@ -60,6 +60,19 @@ bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) {
   return true;
 }
 
+void epsetAssign(SEpSet* pDst, const SEpSet* pSrc) {
+  if (pSrc == NULL || pDst == NULL) {
+      return;
+  }
+
+  pDst->inUse = pSrc->inUse;
+  pDst->numOfEps = pSrc->numOfEps;
+  for (int32_t i = 0; i < pSrc->numOfEps; ++i) {
+    pDst->eps[i].port = pSrc->eps[i].port;
+    tstrncpy(pDst->eps[i].fqdn, pSrc->eps[i].fqdn, tListLen(pSrc->eps[i].fqdn));
+  }
+}
+
 void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet) {
   taosCorBeginWrite(&pEpSet->version);
   pEpSet->epSet = *pNewEpSet;

source/common/src/ttime.c

@@ -68,12 +68,12 @@ static int64_t user_mktime64(const uint32_t year0, const uint32_t mon0, const ui
 
 // ==== mktime() kernel code =================//
 static int64_t m_deltaUtc = 0;
-void           deltaToUtcInitOnce() {
-            struct tm tm = {0};
 
-            (void)taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm);
+void deltaToUtcInitOnce() {
-            m_deltaUtc = (int64_t)taosMktime(&tm);
+  struct tm tm = {0};
-            // printf("====delta:%lld\n\n", seconds);
+  (void)taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm);
+  m_deltaUtc = (int64_t)taosMktime(&tm);
+  // printf("====delta:%lld\n\n", seconds);
 }
 
 static int64_t parseFraction(char* str, char** end, int32_t timePrec);

source/common/src/tvariant.c

@@ -185,7 +185,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
     pDst->arr = taosArrayInit(num, sizeof(char *));
     for (size_t i = 0; i < num; i++) {
       char *p = (char *)taosArrayGetP(pSrc->arr, i);
-      char *n = strdup(p);
+      char *n = taosStrdup(p);
       taosArrayPush(pDst->arr, &n);
     }
   } else if (pSrc->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {

source/common/test/commonTests.cpp

@@ -109,14 +109,14 @@ TEST(testCase, Datablock_test) {
     if (i & 0x01) {
       int32_t len = sprintf(buf, str, i);
       STR_TO_VARSTR(varbuf, buf)
-      colDataAppend(p0, i, (const char*)&i, false);
+      colDataSetVal(p0, i, (const char*)&i, false);
-      colDataAppend(p1, i, (const char*)varbuf, false);
+      colDataSetVal(p1, i, (const char*)varbuf, false);
 
       memset(varbuf, 0, sizeof(varbuf));
       memset(buf, 0, sizeof(buf));
     } else {
-      colDataAppend(p0, i, (const char*)&i, true);
+      colDataSetVal(p0, i, (const char*)&i, true);
-      colDataAppend(p1, i, (const char*)varbuf, true);
+      colDataSetVal(p1, i, (const char*)varbuf, true);
     }
 
     b->info.rows++;
@@ -185,8 +185,8 @@ TEST(testCase, non_var_dataBlock_split_test) {
     SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1);
 
     int8_t v = i;
-    colDataAppend(p0, i, (const char*)&i, false);
+    colDataSetVal(p0, i, (const char*)&i, false);
-    colDataAppend(p1, i, (const char*)&v, false);
+    colDataSetVal(p1, i, (const char*)&v, false);
     b->info.rows++;
   }
 
@@ -231,12 +231,12 @@ TEST(testCase, var_dataBlock_split_test) {
     SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1);
 
     int8_t v = i;
-    colDataAppend(p0, i, (const char*)&i, false);
+    colDataSetVal(p0, i, (const char*)&i, false);
 
     sprintf(buf, "the number of row:%d", i);
     int32_t len = sprintf(buf1, buf, i);
     STR_TO_VARSTR(buf1, buf)
-    colDataAppend(p1, i, buf1, false);
+    colDataSetVal(p1, i, buf1, false);
     b->info.rows++;
 
     memset(buf, 0, sizeof(buf));

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -242,19 +242,19 @@ int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) {
     GRANT_CFG_SKIP;
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
-    colDataAppend(pColInfo, i, (const char *)&dnodeId, false);
+    colDataSetVal(pColInfo, i, (const char *)&dnodeId, false);
 
     char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
     pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
-    colDataAppend(pColInfo, i, name, false);
+    colDataSetVal(pColInfo, i, name, false);
 
     char    value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
     int32_t valueLen = 0;
     cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
     varDataSetLen(value, valueLen);
     pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
-    colDataAppend(pColInfo, i, value, false);
+    colDataSetVal(pColInfo, i, value, false);
 
     numOfRows++;
   }

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -53,7 +53,7 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
   pVnode->vgVersion = pCfg->vgVersion;
   pVnode->refCount = 0;
   pVnode->dropped = 0;
-  pVnode->path = tstrdup(pCfg->path);
+  pVnode->path = taosStrdup(pCfg->path);
   pVnode->pImpl = pImpl;
 
   if (pVnode->path == NULL) {

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -166,7 +166,6 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
     dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
             terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
     terrno = (terrno != 0) ? terrno : -1;
-    vmSendResponse(pMsg);
     return terrno;
   }
 

source/dnode/mgmt/node_mgmt/src/dmMgmt.c

@@ -124,7 +124,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
     taosThreadRwlockInit(&pWrapper->lock, NULL);
 
     snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name);
-    pWrapper->path = strdup(path);
+    pWrapper->path = taosStrdup(path);
     if (pWrapper->path == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       goto _OVER;

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -93,18 +93,30 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
       break;
   }
 
-  if (pDnode->status != DND_STAT_RUNNING) {
+/*
-    if (pRpc->msgType == TDMT_DND_SERVER_STATUS) {
+pDnode is null, TD-22618
-      dmProcessServerStartupStatus(pDnode, pRpc);
+at trans.c line 91
-      return;
+before this line, dmProcessRpcMsg callback is set
-    } else {
+after this line, parent is set
-      if (pDnode->status == DND_STAT_INIT) {
+so when dmProcessRpcMsg is called, pDonde is still null.
-        terrno = TSDB_CODE_APP_IS_STARTING;
+*/
+  if (pDnode != NULL){
+    if(pDnode->status != DND_STAT_RUNNING) {
+      if (pRpc->msgType == TDMT_DND_SERVER_STATUS) {
+        dmProcessServerStartupStatus(pDnode, pRpc);
+        return;
       } else {
-        terrno = TSDB_CODE_APP_IS_STOPPING;
+        if (pDnode->status == DND_STAT_INIT) {
+          terrno = TSDB_CODE_APP_IS_STARTING;
+        } else {
+          terrno = TSDB_CODE_APP_IS_STOPPING;
+        }
+        goto _OVER;
       }
-      goto _OVER;
     }   
+  } else {
+    terrno = TSDB_CODE_APP_IS_STARTING;
+    goto _OVER;
   }
 
   if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) {

source/dnode/mgmt/node_util/src/dmEps.c

@@ -402,7 +402,7 @@ static int32_t dmDecodeEpPairs(SJson *pJson, SDnodeData *pData) {
   int32_t code = 0;
 
   SJson *dnodes = tjsonGetObjectItem(pJson, "dnodes");
-  if (dnodes == NULL) return 0;
+  if (dnodes == NULL) return -1;
   int32_t numOfDnodes = tjsonGetArraySize(dnodes);
 
   for (int32_t i = 0; i < numOfDnodes; ++i) {

source/dnode/mnode/impl/inc/mndInt.h

@@ -58,7 +58,7 @@ typedef int32_t (*MndInitFp)(SMnode *pMnode);
 typedef void (*MndCleanupFp)(SMnode *pMnode);
 typedef int32_t (*ShowRetrieveFp)(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter);
-typedef struct SQWorker SQHandle;
+typedef struct SQueueWorker SQHandle;
 
 typedef struct {
   const char  *name;

source/dnode/mnode/impl/inc/mndTopic.h

@@ -27,14 +27,9 @@ void    mndCleanupTopic(SMnode *pMnode);
 
 SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName);
 void         mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic);
-
+int32_t      mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
-SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic);
+bool         mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb);
-SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw);
+const char  *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]);
-
-int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
-int32_t mndCheckTopicExist(SMnode *pMnode, SDbObj *pDb);
-
-const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]);
 
 int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
 

source/dnode/mnode/impl/src/mndCluster.c

@@ -278,31 +278,31 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *
 
     cols = 0;
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->id, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pCluster->id, false);
 
     char buf[tListLen(pCluster->name) + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->pMeta->pSchemas[cols].bytes);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, buf, false);
+    colDataSetVal(pColInfo, numOfRows, buf, false);
 
     int32_t upTime = mndGetClusterUpTimeImp(pCluster);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&upTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&upTime, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false);
 
     char ver[12] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(ver, tsVersionName, pShow->pMeta->pSchemas[cols].bytes);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)ver, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)ver, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     if (tsExpireTime <= 0) {
-      colDataAppendNULL(pColInfo, numOfRows);
+      colDataSetNULL(pColInfo, numOfRows);
     } else {
-      colDataAppend(pColInfo, numOfRows, (const char *)&tsExpireTime, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&tsExpireTime, false);
     }
 
     sdbRelease(pSdb, pCluster);

source/dnode/mnode/impl/src/mndConsumer.c

@@ -100,7 +100,7 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
     return 0;
   }
 
-  mInfo("receive consumer lost msg, consumer id %" PRId64 ", status %s", pLostMsg->consumerId,
+  mInfo("process consumer lost msg, consumer:0x%" PRIx64 " status:%d(%s)", pLostMsg->consumerId, pConsumer->status,
         mndConsumerStatusName(pConsumer->status));
 
   if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
@@ -114,9 +114,17 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
   mndReleaseConsumer(pMnode, pConsumer);
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "lost-csm");
-  if (pTrans == NULL) goto FAIL;
+  if (pTrans == NULL) {
-  if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL;
+    goto FAIL;
-  if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL;
+  }
+
+  if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
+    goto FAIL;
+  }
+
+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    goto FAIL;
+  }
 
   tDeleteSMqConsumerObj(pConsumerNew);
   taosMemoryFree(pConsumerNew);
@@ -138,8 +146,8 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
     return -1;
   }
 
-  mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId,
+  mInfo("receive consumer recover msg, consumer:0x%" PRIx64 " status:%d(%s)", pRecoverMsg->consumerId,
-        mndConsumerStatusName(pConsumer->status));
+        pConsumer->status, mndConsumerStatusName(pConsumer->status));
 
   if (pConsumer->status != MQ_CONSUMER_STATUS__LOST_REBD) {
     mndReleaseConsumer(pMnode, pConsumer);
@@ -153,7 +161,10 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
   mndReleaseConsumer(pMnode, pConsumer);
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm");
-  if (pTrans == NULL) goto FAIL;
+  if (pTrans == NULL) {
+    goto FAIL;
+  }
+
   if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL;
 
@@ -241,25 +252,32 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
   // iterate all consumers, find all modification
   while (1) {
     pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
 
     int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1);
     int32_t status = atomic_load_32(&pConsumer->status);
-    if (status == MQ_CONSUMER_STATUS__READY && hbStatus > MND_CONSUMER_LOST_HB_CNT) {
-      SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg));
 
-      pLostMsg->consumerId = pConsumer->consumerId;
+    mDebug("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", uptime:%" PRId64 ", hbstatus:%d",
-      SRpcMsg rpcMsg = {
+           pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->upTime,
-          .msgType = TDMT_MND_TMQ_CONSUMER_LOST,
+           hbStatus);
-          .pCont = pLostMsg,
-          .contLen = sizeof(SMqConsumerLostMsg),
-      };
-      tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
-    }
 
     if (status == MQ_CONSUMER_STATUS__READY) {
-      // do nothing
+      if (hbStatus > MND_CONSUMER_LOST_HB_CNT) {
+        SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg));
+
+        pLostMsg->consumerId = pConsumer->consumerId;
+        SRpcMsg rpcMsg = {
+            .msgType = TDMT_MND_TMQ_CONSUMER_LOST,
+            .pCont = pLostMsg,
+            .contLen = sizeof(SMqConsumerLostMsg),
+        };
+
+        tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+      }
     } else if (status == MQ_CONSUMER_STATUS__LOST_REBD) {
+      // if the client is lost longer than one day, clear it. Otherwise, do nothing about the lost consumers.
       if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) {
         SMqConsumerClearMsg *pClearMsg = rpcMallocCont(sizeof(SMqConsumerClearMsg));
 
@@ -269,6 +287,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) {
             .pCont = pClearMsg,
             .contLen = sizeof(SMqConsumerClearMsg),
         };
+
         tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
       }
     } else if (status == MQ_CONSUMER_STATUS__LOST) {
@@ -338,7 +357,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
   int64_t         consumerId = req.consumerId;
   SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
   if (pConsumer == NULL) {
-    mError("consumer %" PRId64 " not exist", consumerId);
+    mError("consumer:0x%" PRIx64 " not exist", consumerId);
     terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
     return -1;
   }
@@ -348,7 +367,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
   int32_t status = atomic_load_32(&pConsumer->status);
 
   if (status == MQ_CONSUMER_STATUS__LOST_REBD) {
-    mInfo("try to recover consumer %" PRId64 "", consumerId);
+    mInfo("try to recover consumer:0x%" PRIx64 "", consumerId);
     SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg));
 
     pRecoverMsg->consumerId = consumerId;
@@ -380,6 +399,15 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
 
   SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
   if (pConsumer == NULL) {
+    mError("consumer:0x%" PRIx64 " group:%s not exists in sdb", consumerId, req.cgroup);
+    terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
+    return -1;
+  }
+
+  int32_t ret = strncmp(req.cgroup, pConsumer->cgroup, tListLen(pConsumer->cgroup));
+  if (ret != 0) {
+    mError("consumer:0x%" PRIx64 " group:%s not consistent with data in sdb, saved cgroup:%s", consumerId, req.cgroup,
+           pConsumer->cgroup);
     terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
     return -1;
   }
@@ -391,7 +419,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
 
 #if 1
   if (status == MQ_CONSUMER_STATUS__LOST_REBD) {
-    mInfo("try to recover consumer %" PRId64 "", consumerId);
+    mInfo("try to recover consumer:0x%" PRIx64, consumerId);
     SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg));
 
     pRecoverMsg->consumerId = consumerId;
@@ -400,22 +428,24 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
         .pCont = pRecoverMsg,
         .contLen = sizeof(SMqConsumerRecoverMsg),
     };
+
     tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg);
   }
 #endif
 
   if (status != MQ_CONSUMER_STATUS__READY) {
-    mInfo("consumer %" PRId64 " not ready, status: %s", consumerId, mndConsumerStatusName(status));
+    mInfo("consumer:0x%" PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status));
     terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
     return -1;
   }
 
   int32_t serverEpoch = atomic_load_32(&pConsumer->epoch);
 
-  // 2. check epoch, only send ep info when epoches do not match
+  // 2. check epoch, only send ep info when epochs do not match
   if (epoch != serverEpoch) {
     taosRLockLatch(&pConsumer->lock);
-    mInfo("process ask ep, consumer:%" PRId64 "(epoch %d), server epoch %d", consumerId, epoch, serverEpoch);
+    mInfo("process ask ep, consumer:0x%" PRIx64 "(epoch %d) update with server epoch %d", consumerId, epoch,
+          serverEpoch);
     int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics);
 
     rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp));
@@ -425,7 +455,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
       goto FAIL;
     }
 
-    // handle all topic subscribed by the consumer
+    // handle all topics subscribed by this consumer
     for (int32_t i = 0; i < numOfTopics; i++) {
       char            *topic = taosArrayGetP(pConsumer->currentTopics, i);
       SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
@@ -452,6 +482,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
       SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
       int32_t        vgNum = taosArrayGetSize(pConsumerEp->vgs);
 
+      // this customer assigned vgroups
       topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
       if (topicEp.vgs == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -481,6 +512,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
     }
     taosRUnLockLatch(&pConsumer->lock);
   }
+
   // encode rsp
   int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, &rsp);
   void   *buf = rpcMallocCont(tlen);
@@ -488,6 +520,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
+
   ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP;
   ((SMqRspHead *)buf)->epoch = serverEpoch;
   ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId;
@@ -503,6 +536,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
   pMsg->info.rsp = buf;
   pMsg->info.rspLen = tlen;
   return 0;
+
 FAIL:
   tDeleteSMqAskEpRsp(&rsp);
   mndReleaseConsumer(pMnode, pConsumer);
@@ -525,122 +559,132 @@ int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj
   return 0;
 }
 
-static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
+static int32_t validateTopics(const SArray* pTopicList, SMnode* pMnode, const char* pUser) {
-  SMnode         *pMnode = pMsg->info.node;
+  int32_t numOfTopics = taosArrayGetSize(pTopicList);
-  char           *msgStr = pMsg->pCont;
-  SCMSubscribeReq subscribe = {0};
-  tDeserializeSCMSubscribeReq(msgStr, &subscribe);
-  int64_t         consumerId = subscribe.consumerId;
-  char           *cgroup = subscribe.cgroup;
-  SMqConsumerObj *pConsumerOld = NULL;
-  SMqConsumerObj *pConsumerNew = NULL;
 
-  int32_t code = -1;
+  for (int32_t i = 0; i < numOfTopics; i++) {
-  SArray *newSub = subscribe.topicNames;
+    char        *pOneTopic = taosArrayGetP(pTopicList, i);
-  taosArraySort(newSub, taosArrayCompareString);
+    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pOneTopic);
-  taosArrayRemoveDuplicateP(newSub, taosArrayCompareString, taosMemoryFree);
+    if (pTopic == NULL) {  // terrno has been set by callee function
-
+      return -1;
-  int32_t newTopicNum = taosArrayGetSize(newSub);
-  // check topic existance
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
-  if (pTrans == NULL) goto SUBSCRIBE_OVER;
-
-  for (int32_t i = 0; i < newTopicNum; i++) {
-    char        *topic = taosArrayGetP(newSub, i);
-    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
-    if (pTopic == NULL) {
-      terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
-      goto SUBSCRIBE_OVER;
     }
 
-    if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) {
+    if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) {
       mndReleaseTopic(pMnode, pTopic);
-      goto SUBSCRIBE_OVER;
+      return -1;
-    }
-
-    if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) {
-      goto SUBSCRIBE_OVER;
     }
 
     mndReleaseTopic(pMnode, pTopic);
   }
 
-  pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
+  return 0;
-  if (pConsumerOld == NULL) {
+}
-    mInfo("receive subscribe request from new consumer:%" PRId64, consumerId);
+
+int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
+  SMnode *pMnode = pMsg->info.node;
+  char   *msgStr = pMsg->pCont;
+
+  SCMSubscribeReq subscribe = {0};
+  tDeserializeSCMSubscribeReq(msgStr, &subscribe);
+
+  uint64_t        consumerId = subscribe.consumerId;
+  char           *cgroup = subscribe.cgroup;
+  SMqConsumerObj *pExistedConsumer = NULL;
+  SMqConsumerObj *pConsumerNew = NULL;
+
+  int32_t code = -1;
+  SArray *pTopicList = subscribe.topicNames;
+  taosArraySort(pTopicList, taosArrayCompareString);
+  taosArrayRemoveDuplicateP(pTopicList, taosArrayCompareString, taosMemoryFree);
+
+  int32_t newTopicNum = taosArrayGetSize(pTopicList);
+
+  // check topic existence
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
+  if (pTrans == NULL) {
+    goto _over;
+  }
+
+  code = validateTopics(pTopicList, pMnode, pMsg->info.conn.user);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _over;
+  }
+
+  pExistedConsumer = mndAcquireConsumer(pMnode, consumerId);
+  if (pExistedConsumer == NULL) {
+    mInfo("receive subscribe request from new consumer:0x%" PRIx64" cgroup:%s", consumerId, subscribe.cgroup);
 
     pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
     tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256);
     pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
     taosArrayDestroy(pConsumerNew->rebNewTopics);
-    pConsumerNew->rebNewTopics = newSub;
+    pConsumerNew->rebNewTopics = pTopicList;  // all subscribe topics should re-balance.
     subscribe.topicNames = NULL;
 
     for (int32_t i = 0; i < newTopicNum; i++) {
-      char *newTopicCopy = strdup(taosArrayGetP(newSub, i));
+      char *newTopicCopy = taosStrdup(taosArrayGetP(pTopicList, i));
       taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy);
     }
 
-    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto _over;
-    if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
+    if (mndTransPrepare(pMnode, pTrans) != 0) goto _over;
 
   } else {
-    /*taosRLockLatch(&pConsumerOld->lock);*/
+    /*taosRLockLatch(&pExistedConsumer->lock);*/
+    int32_t status = atomic_load_32(&pExistedConsumer->status);
 
-    int32_t status = atomic_load_32(&pConsumerOld->status);
+    mInfo("receive subscribe request from existed consumer:0x%" PRIx64 " cgroup:%s, current status:%d(%s), subscribe topic num: %d",
-
+          consumerId, subscribe.cgroup, status,mndConsumerStatusName(status), newTopicNum);
-    mInfo("receive subscribe request from existing consumer:%" PRId64 ", current status: %s, subscribe topic num: %d",
-          consumerId, mndConsumerStatusName(status), newTopicNum);
 
     if (status != MQ_CONSUMER_STATUS__READY) {
       terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
-      goto SUBSCRIBE_OVER;
+      goto _over;
     }
 
     pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
     if (pConsumerNew == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _over;
-      goto SUBSCRIBE_OVER;
     }
+
     pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
 
     for (int32_t i = 0; i < newTopicNum; i++) {
-      char *newTopicCopy = strdup(taosArrayGetP(newSub, i));
+      char *newTopicCopy = taosStrdup(taosArrayGetP(pTopicList, i));
       taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy);
     }
 
     int32_t oldTopicNum = 0;
-    if (pConsumerOld->currentTopics) {
+    if (pExistedConsumer->currentTopics) {
-      oldTopicNum = taosArrayGetSize(pConsumerOld->currentTopics);
+      oldTopicNum = taosArrayGetSize(pExistedConsumer->currentTopics);
     }
 
     int32_t i = 0, j = 0;
     while (i < oldTopicNum || j < newTopicNum) {
       if (i >= oldTopicNum) {
-        char *newTopicCopy = strdup(taosArrayGetP(newSub, j));
+        char *newTopicCopy = taosStrdup(taosArrayGetP(pTopicList, j));
         taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
         j++;
         continue;
       } else if (j >= newTopicNum) {
-        char *oldTopicCopy = strdup(taosArrayGetP(pConsumerOld->currentTopics, i));
+        char *oldTopicCopy = taosStrdup(taosArrayGetP(pExistedConsumer->currentTopics, i));
         taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
         i++;
         continue;
       } else {
-        char *oldTopic = taosArrayGetP(pConsumerOld->currentTopics, i);
+        char *oldTopic = taosArrayGetP(pExistedConsumer->currentTopics, i);
-        char *newTopic = taosArrayGetP(newSub, j);
+        char *newTopic = taosArrayGetP(pTopicList, j);
-        int   comp = compareLenPrefixedStr(oldTopic, newTopic);
+        int   comp = strcmp(oldTopic, newTopic);
         if (comp == 0) {
           i++;
           j++;
           continue;
         } else if (comp < 0) {
-          char *oldTopicCopy = strdup(oldTopic);
+          char *oldTopicCopy = taosStrdup(oldTopic);
           taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
           i++;
           continue;
         } else {
-          char *newTopicCopy = strdup(newTopic);
+          char *newTopicCopy = taosStrdup(newTopic);
           taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
           j++;
           continue;
@@ -648,27 +692,28 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
       }
     }
 
-    if (pConsumerOld && taosArrayGetSize(pConsumerNew->rebNewTopics) == 0 &&
+    if (pExistedConsumer && taosArrayGetSize(pConsumerNew->rebNewTopics) == 0 &&
         taosArrayGetSize(pConsumerNew->rebRemovedTopics) == 0) {
       /*if (taosArrayGetSize(pConsumerNew->assignedTopics) == 0) {*/
       /*pConsumerNew->updateType = */
       /*}*/
-      goto SUBSCRIBE_OVER;
+      goto _over;
     }
 
-    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
+    if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto _over;
-    if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER;
+    if (mndTransPrepare(pMnode, pTrans) != 0) goto _over;
   }
 
   code = TSDB_CODE_ACTION_IN_PROGRESS;
 
-SUBSCRIBE_OVER:
+_over:
   mndTransDrop(pTrans);
 
-  if (pConsumerOld) {
+  if (pExistedConsumer) {
-    /*taosRUnLockLatch(&pConsumerOld->lock);*/
+    /*taosRUnLockLatch(&pExistedConsumer->lock);*/
-    mndReleaseConsumer(pMnode, pConsumerOld);
+    mndReleaseConsumer(pMnode, pExistedConsumer);
   }
+
   if (pConsumerNew) {
     tDeleteSMqConsumerObj(pConsumerNew);
     taosMemoryFree(pConsumerNew);
@@ -715,13 +760,14 @@ CM_ENCODE_OVER:
 }
 
 SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
-  terrno = TSDB_CODE_OUT_OF_MEMORY;
   SSdbRow        *pRow = NULL;
   SMqConsumerObj *pConsumer = NULL;
   void           *buf = NULL;
 
   int8_t sver = 0;
-  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
+    goto CM_DECODE_OVER;
+  }
 
   if (sver != MND_CONSUMER_VER_NUMBER) {
     terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
@@ -729,52 +775,63 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
   }
 
   pRow = sdbAllocRow(sizeof(SMqConsumerObj));
-  if (pRow == NULL) goto CM_DECODE_OVER;
+  if (pRow == NULL) {
+    goto CM_DECODE_OVER;
+  }
 
   pConsumer = sdbGetRowObj(pRow);
-  if (pConsumer == NULL) goto CM_DECODE_OVER;
+  if (pConsumer == NULL) {
+    goto CM_DECODE_OVER;
+  }
 
   int32_t dataPos = 0;
   int32_t len;
   SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER);
   buf = taosMemoryMalloc(len);
-  if (buf == NULL) goto CM_DECODE_OVER;
+  if (buf == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto CM_DECODE_OVER;
+  }
+
   SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER);
   SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER);
 
   if (tDecodeSMqConsumerObj(buf, pConsumer) == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;  // TODO set correct error code
     goto CM_DECODE_OVER;
   }
-  tmsgUpdateDnodeEpSet(&pConsumer->ep);
 
-  terrno = TSDB_CODE_SUCCESS;
+  tmsgUpdateDnodeEpSet(&pConsumer->ep);
 
 CM_DECODE_OVER:
   taosMemoryFreeClear(buf);
   if (terrno != TSDB_CODE_SUCCESS) {
-    mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer == NULL ? 0 : pConsumer->consumerId,
+    mError("consumer:0x%" PRIx64 " failed to decode from raw:%p since %s",
-           pRaw, terrstr());
+           pConsumer == NULL ? 0 : pConsumer->consumerId, pRaw, terrstr());
     taosMemoryFreeClear(pRow);
-    return NULL;
   }
 
   return pRow;
 }
 
 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
-  mTrace("consumer:%" PRId64 ", perform insert action", pConsumer->consumerId);
+  mDebug("consumer:0x%" PRIx64 " cgroup:%s status:%d(%s) epoch:%d load from sdb, perform insert action",
+         pConsumer->consumerId, pConsumer->cgroup, pConsumer->status, mndConsumerStatusName(pConsumer->status),
+         pConsumer->epoch);
   pConsumer->subscribeTime = pConsumer->upTime;
   return 0;
 }
 
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
-  mTrace("consumer:%" PRId64 ", perform delete action", pConsumer->consumerId);
+  mDebug("consumer:0x%" PRIx64 " perform delete action, status:%s", pConsumer->consumerId,
+         mndConsumerStatusName(pConsumer->status));
   tDeleteSMqConsumerObj(pConsumer);
   return 0;
 }
 
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
-  mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
+  mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", uptime:%" PRId64,
+         pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime);
 
   taosWLockLatch(&pOldConsumer->lock);
 
@@ -808,20 +865,24 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
     int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
     /*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/
     for (int32_t i = 0; i < sz; i++) {
-      char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i));
+      char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i));
       taosArrayPush(pOldConsumer->rebRemovedTopics, &topic);
     }
 
     pOldConsumer->rebalanceTime = pNewConsumer->upTime;
 
+    int32_t status = pOldConsumer->status;
     pOldConsumer->status = MQ_CONSUMER_STATUS__LOST;
+    mDebug("consumer:0x%" PRIx64 " state %s -> %s, reb-time:%" PRId64 ", reb-removed-topics:%d",
+           pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status),
+           pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics));
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) {
     /*A(taosArrayGetSize(pOldConsumer->currentTopics) == 0);*/
     /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/
 
     int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics);
     for (int32_t i = 0; i < sz; i++) {
-      char *topic = strdup(taosArrayGetP(pOldConsumer->assignedTopics, i));
+      char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i));
       taosArrayPush(pOldConsumer->rebNewTopics, &topic);
     }
 
@@ -837,12 +898,13 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
     /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);*/
     /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);*/
 
-    char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
+    char *addedTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
     // not exist in current topic
 
     bool existing = false;
 #if 1
-    for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) {
+    int32_t numOfExistedTopics = taosArrayGetSize(pOldConsumer->currentTopics);
+    for (int32_t i = 0; i < numOfExistedTopics; i++) {
       char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
       if (strcmp(topic, addedTopic) == 0) {
         existing = true;
@@ -864,30 +926,33 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
     if (!existing) {
       taosArrayPush(pOldConsumer->currentTopics, &addedTopic);
       taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString);
+    } else {
+      taosMemoryFree(addedTopic);
     }
 
     // set status
+    int32_t status = pOldConsumer->status;
     if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
-      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+      if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
-          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
         pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
-      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB ||
+      } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) {
-                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) {
         pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
       }
     } else {
-      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+      if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
-          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
         pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
-      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST ||
+      } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
-                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
         pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
       }
     }
 
+    // the re-balance is triggered when the new consumer is launched.
     pOldConsumer->rebalanceTime = pNewConsumer->upTime;
 
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+    mDebug("consumer:0x%" PRIx64 " state %s -> %s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
+           pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status),
+           pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics));
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
     /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/
     /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/
@@ -926,27 +991,27 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
     /*A(i < sz);*/
 
     // set status
+    int32_t status = pOldConsumer->status;
     if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
-      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+      if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
-          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
         pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
-      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB ||
+      } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) {
-                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST) {
         pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD;
       }
     } else {
-      if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
+      if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
-          pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) {
         pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB;
-      } else if (pOldConsumer->status == MQ_CONSUMER_STATUS__LOST ||
+      } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
-                 pOldConsumer->status == MQ_CONSUMER_STATUS__LOST_IN_REB) {
         pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
       }
     }
 
     pOldConsumer->rebalanceTime = pNewConsumer->upTime;
-
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+
+    mDebug("consumer:0x%" PRIx64 " state %s -> %s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d",
+           pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status),
+           pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics));
   }
 
   taosWUnLockLatch(&pOldConsumer->lock);
@@ -975,16 +1040,19 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
 
   while (numOfRows < rowsCapacity) {
     pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
-    if (pShow->pIter == NULL) break;
+    if (pShow->pIter == NULL) {
+      break;
+    }
+
     if (taosArrayGetSize(pConsumer->assignedTopics) == 0) {
-      mDebug("showing consumer %" PRId64 " no assigned topic, skip", pConsumer->consumerId);
+      mDebug("showing consumer:0x%" PRIx64 " no assigned topic, skip", pConsumer->consumerId);
       sdbRelease(pSdb, pConsumer);
       continue;
     }
 
     taosRLockLatch(&pConsumer->lock);
 
-    mDebug("showing consumer %" PRId64, pConsumer->consumerId);
+    mDebug("showing consumer:0x%" PRIx64, pConsumer->consumerId);
 
     int32_t topicSz = taosArrayGetSize(pConsumer->assignedTopics);
     bool    hasTopic = true;
@@ -1003,56 +1071,55 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
 
       // consumer id
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false);
 
       // consumer group
       char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
       tstrncpy(varDataVal(cgroup), pConsumer->cgroup, TSDB_CGROUP_LEN);
       varDataSetLen(cgroup, strlen(varDataVal(cgroup)));
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false);
 
       // client id
       char clientId[256 + VARSTR_HEADER_SIZE] = {0};
       tstrncpy(varDataVal(clientId), pConsumer->clientId, 256);
       varDataSetLen(clientId, strlen(varDataVal(clientId)));
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)clientId, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false);
 
       // status
       char status[20 + VARSTR_HEADER_SIZE] = {0};
       tstrncpy(varDataVal(status), mndConsumerStatusName(pConsumer->status), 20);
       varDataSetLen(status, strlen(varDataVal(status)));
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)status, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)status, false);
 
       // one subscribed topic
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       if (hasTopic) {
         char        topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
         const char *topicName = mndTopicGetShowName(taosArrayGetP(pConsumer->assignedTopics, i));
-        tstrncpy(varDataVal(topic), topicName, TSDB_TOPIC_FNAME_LEN);
+        STR_TO_VARSTR(topic, topicName);
-        varDataSetLen(topic, strlen(varDataVal(topic)));
+        colDataSetVal(pColInfo, numOfRows, (const char *)topic, false);
-        colDataAppend(pColInfo, numOfRows, (const char *)topic, false);
       } else {
-        colDataAppend(pColInfo, numOfRows, NULL, true);
+        colDataSetVal(pColInfo, numOfRows, NULL, true);
       }
 
       // end point
       /*pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);*/
-      /*colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);*/
+      /*colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);*/
 
       // up time
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false);
 
       // subscribe time
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false);
 
       // rebalance time
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0);
 
       numOfRows++;
     }

source/dnode/mnode/impl/src/mndDb.c

@@ -1039,13 +1039,23 @@ static int32_t mndBuildDropDbRsp(SDbObj *pDb, int32_t *pRspLen, void **ppRsp, bo
 
 static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
   int32_t code = -1;
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-db");
-  if (pTrans == NULL) goto _OVER;
 
-  mInfo("trans:%d, used to drop db:%s", pTrans->id, pDb->name);
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-db");
+  if (pTrans == NULL) {
+    goto _OVER;
+  }
+
+  mInfo("trans:%d start to drop db:%s", pTrans->id, pDb->name);
+
   mndTransSetDbName(pTrans, pDb->name, NULL);
-  if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER;
+  if (mndTrancCheckConflict(pMnode, pTrans) != 0) {
-  if (mndCheckTopicExist(pMnode, pDb) < 0) goto _OVER;
+    goto _OVER;
+  }
+
+  if (mndTopicExistsForDb(pMnode, pDb)) {
+    terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED;
+    goto _OVER;
+  }
 
   if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
@@ -1097,10 +1107,12 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) {
   }
 
   code = mndDropDb(pMnode, pReq, pDb);
-  if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
+  if (code == TSDB_CODE_SUCCESS) {
+    code = TSDB_CODE_ACTION_IN_PROGRESS;
+  }
 
 _OVER:
-  if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
+  if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) {
     mError("db:%s, failed to drop since %s", dropReq.db, terrstr());
   }
 
@@ -1703,46 +1715,46 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
     for (int32_t i = 0; i < pShow->numOfColumns; ++i) {
       SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i);
       if (i == 0) {
-        colDataAppend(pColInfo, rows, buf, false);
+        colDataSetVal(pColInfo, rows, buf, false);
       } else if (i == 1) {
-        colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false);
+        colDataSetVal(pColInfo, rows, (const char *)&pDb->createdTime, false);
       } else if (i == 3) {
-        colDataAppend(pColInfo, rows, (const char *)&numOfTables, false);
+        colDataSetVal(pColInfo, rows, (const char *)&numOfTables, false);
       } else if (i == 14) {
-        colDataAppend(pColInfo, rows, precVstr, false);
+        colDataSetVal(pColInfo, rows, precVstr, false);
       } else if (i == 15) {
-        colDataAppend(pColInfo, rows, statusVstr, false);
+        colDataSetVal(pColInfo, rows, statusVstr, false);
       } else {
-        colDataAppendNULL(pColInfo, rows);
+        colDataSetNULL(pColInfo, rows);
       }
     }
   } else {
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, buf, false);
+    colDataSetVal(pColInfo, rows, buf, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->createdTime, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfVgroups, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.numOfVgroups, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&numOfTables, false);
+    colDataSetVal(pColInfo, rows, (const char *)&numOfTables, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.replications, false);
 
     const char *strictStr = pDb->cfg.strict ? "on" : "off";
     char        strictVstr[24] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(strictVstr, strictStr, 24);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)strictVstr, false);
+    colDataSetVal(pColInfo, rows, (const char *)strictVstr, false);
 
     char    durationVstr[128] = {0};
     int32_t len = sprintf(&durationVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.daysPerFile);
     varDataSetLen(durationVstr, len);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)durationVstr, false);
+    colDataSetVal(pColInfo, rows, (const char *)durationVstr, false);
 
     char keepVstr[128] = {0};
     if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) {
@@ -1754,86 +1766,86 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
     }
     varDataSetLen(keepVstr, len);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)keepVstr, false);
+    colDataSetVal(pColInfo, rows, (const char *)keepVstr, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.buffer, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.buffer, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pageSize, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.pageSize, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pages, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.pages, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.minRows, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.minRows, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.maxRows, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.maxRows, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.compression, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)precVstr, false);
+    colDataSetVal(pColInfo, rows, (const char *)precVstr, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)statusVstr, false);
+    colDataSetVal(pColInfo, rows, (const char *)statusVstr, false);
 
     char *rentensionVstr = buildRetension(pDb->cfg.pRetensions);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     if (rentensionVstr == NULL) {
-      colDataAppendNULL(pColInfo, rows);
+      colDataSetNULL(pColInfo, rows);
     } else {
-      colDataAppend(pColInfo, rows, (const char *)rentensionVstr, false);
+      colDataSetVal(pColInfo, rows, (const char *)rentensionVstr, false);
       taosMemoryFree(rentensionVstr);
     }
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false);
 
     const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast);
     char        cacheModelVstr[24] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, 24);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false);
+    colDataSetVal(pColInfo, rows, (const char *)cacheModelVstr, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastSize, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.cacheLastSize, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walLevel, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walFsyncPeriod, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walFsyncPeriod, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walRetentionPeriod, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRetentionPeriod, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walRetentionSize, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRetentionSize, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walRollPeriod, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walRollPeriod, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walSegmentSize, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.walSegmentSize, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.sstTrigger, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.sstTrigger, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     int16_t hashPrefix = pDb->cfg.hashPrefix;
     if (hashPrefix > 0) {
       hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1;
     }
-    colDataAppend(pColInfo, rows, (const char *)&hashPrefix, false);
+    colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.hashSuffix, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.hashSuffix, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.tsdbPageSize, false);
+    colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.tsdbPageSize, false);
   }
 
   taosMemoryFree(buf);

source/dnode/mnode/impl/src/mndDef.c

@@ -181,7 +181,7 @@ SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
   SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
   if (pVgEpNew == NULL) return NULL;
   pVgEpNew->vgId = pVgEp->vgId;
-  pVgEpNew->qmsg = strdup(pVgEp->qmsg);
+  pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg);
   pVgEpNew->epSet = pVgEp->epSet;
   return pVgEpNew;
 }

source/dnode/mnode/impl/src/mndDnode.c

@@ -1007,11 +1007,11 @@ static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
 
     STR_WITH_MAXSIZE_TO_VARSTR(buf, cfgOpts[i], TSDB_CONFIG_OPTION_LEN);
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)buf, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)buf, false);
 
     STR_WITH_MAXSIZE_TO_VARSTR(bufVal, cfgVals[i], TSDB_CONFIG_VALUE_LEN);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)bufVal, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)bufVal, false);
 
     numOfRows++;
   }
@@ -1039,20 +1039,20 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     cols = 0;
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pDnode->id, false);
 
     char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->pMeta->pSchemas[cols].bytes);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, buf, false);
+    colDataSetVal(pColInfo, numOfRows, buf, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     int16_t id = mndGetVnodesNum(pMnode, pDnode->id);
-    colDataAppend(pColInfo, numOfRows, (const char *)&id, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&id, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false);
 
     const char *status = "ready";
     if (objStatus == SDB_STATUS_CREATING) status = "creating";
@@ -1069,16 +1069,16 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     char b1[16] = {0};
     STR_TO_VARSTR(b1, status);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, b1, false);
+    colDataSetVal(pColInfo, numOfRows, b1, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false);
 
     char *b = taosMemoryCalloc(VARSTR_HEADER_SIZE + strlen(offlineReason[pDnode->offlineReason]) + 1, 1);
     STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, b, false);
+    colDataSetVal(pColInfo, numOfRows, b, false);
     taosMemoryFreeClear(b);
 
     numOfRows++;

source/dnode/mnode/impl/src/mndDump.c

@@ -629,7 +629,7 @@ void mndDumpSdb() {
   }
   taosWriteFile(pFile, pCont, contLen);
   taosWriteFile(pFile, "\n", 1);
-  taosFsyncFile(pFile);
+  UNUSED(taosFsyncFile(pFile));
   taosCloseFile(&pFile);
   tjsonDelete(json);
   taosMemoryFree(pCont);

source/dnode/mnode/impl/src/mndFunc.c

@@ -511,39 +511,39 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
     STR_WITH_MAXSIZE_TO_VARSTR(b1, pFunc->name, pShow->pMeta->pSchemas[cols].bytes);
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)b1, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)b1, false);
 
     if (pFunc->pComment) {
       char *b2 = taosMemoryCalloc(1, pShow->pMeta->pSchemas[cols].bytes);
       STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->pMeta->pSchemas[cols].bytes);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)b2, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)b2, false);
     } else {
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, NULL, true);
+      colDataSetVal(pColInfo, numOfRows, NULL, true);
     }
 
     int32_t isAgg = (pFunc->funcType == TSDB_FUNC_TYPE_AGGREGATE) ? 1 : 0;
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&isAgg, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&isAgg, false);
 
     char b3[TSDB_TYPE_STR_MAX_LEN + 1] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(b3, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->outputType, pFunc->outputLen),
                                pShow->pMeta->pSchemas[cols].bytes);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)b3, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)b3, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pFunc->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->createdTime, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pFunc->codeSize, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->codeSize, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pFunc->bufSize, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->bufSize, false);
 
     numOfRows++;
     sdbRelease(pSdb, pFunc);

source/dnode/mnode/impl/src/mndGrant.c

@@ -29,85 +29,85 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     const char      *src = "community";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "false";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     src = "unlimited";
     STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
-    colDataAppend(pColInfo, numOfRows, tmp, false);
+    colDataSetVal(pColInfo, numOfRows, tmp, false);
 
     numOfRows++;
   }

source/dnode/mnode/impl/src/mndIndex.c

@@ -638,7 +638,7 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
 }
 int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
   // impl later
-  int32_t code = 0;
+  int32_t code = -1;
   SStbObj newStb = {0};
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb-index");
   if (pTrans == NULL) goto _OVER;
@@ -670,6 +670,7 @@ _OVER:
 }
 
 static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) {
+  int32_t code = -1;
   SIdxObj idxObj = {0};
   memcpy(idxObj.name, req->idxName, TSDB_TABLE_FNAME_LEN);
   memcpy(idxObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN);
@@ -681,21 +682,6 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
   idxObj.stbUid = pStb->uid;
   idxObj.dbUid = pStb->dbUid;
 
-  int32_t code = -1;
-  // SField *pField0 = NULL;
-
-  // SStbObj  stbObj = {0};
-  // SStbObj *pNew = &stbObj;
-
-  // taosRLockLatch(&pOld->lock);
-  // memcpy(&stbObj, pOld, sizeof(SStbObj));
-  // taosRUnLockLatch(&pOld->lock);
-
-  // stbObj.pColumns = NULL;
-  // stbObj.pTags = NULL;
-  // stbObj.updateTime = taosGetTimestampMs();
-  // stbObj.lock = 0;
-
   int32_t tag = mndFindSuperTableTagId(pStb, req->colName);
   if (tag < 0) {
     terrno = TSDB_CODE_MND_TAG_NOT_EXIST;

source/dnode/mnode/impl/src/mndMain.c

@@ -133,6 +133,7 @@ static void mndCalMqRebalance(SMnode *pMnode) {
   }
 }
 
+#if 0
 static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) {
   int32_t contLen = 0;
   void   *pReq = mndBuildCheckpointTickMsg(&contLen, sec);
@@ -145,6 +146,7 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) {
     tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
   }
 }
+#endif
 
 static void mndPullupTelem(SMnode *pMnode) {
   mTrace("pullup telem msg");
@@ -320,7 +322,7 @@ static void mndCleanupTimer(SMnode *pMnode) {
 }
 
 static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
-  pMnode->path = strdup(path);
+  pMnode->path = taosStrdup(path);
   if (pMnode->path == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
@@ -714,7 +716,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
   } else if (code == 0) {
     mGTrace("msg:%p, successfully processed", pMsg);
   } else {
-    mGError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle,
+    mGError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, tstrerror(code), pMsg->info.ahandle,
             TMSG_INFO(pMsg->msgType));
   }
 

source/dnode/mnode/impl/src/mndMnode.c

@@ -608,13 +608,13 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
 
     cols = 0;
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false);
 
     char b1[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, TSDB_EP_LEN + VARSTR_HEADER_SIZE);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, b1, false);
+    colDataSetVal(pColInfo, numOfRows, b1, false);
 
     char role[20] = "offline";
     if (pObj->id == pMnode->selfDnodeId) {
@@ -630,7 +630,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     char b2[12 + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(b2, role, pShow->pMeta->pSchemas[cols].bytes);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)b2, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)b2, false);
 
     const char *status = "ready";
     if (objStatus == SDB_STATUS_CREATING) status = "creating";
@@ -639,10 +639,10 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     char b3[9 + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(b3, status, pShow->pMeta->pSchemas[cols].bytes);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)b3, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)b3, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
 
     numOfRows++;
     sdbRelease(pSdb, pObj);

source/dnode/mnode/impl/src/mndProfile.c

@@ -709,32 +709,32 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
     cols = 0;
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConn->id, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->id, false);
 
     char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_TO_VARSTR(user, pConn->user);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)user, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)user, false);
 
     char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE];
     STR_TO_VARSTR(app, pConn->app);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)app, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)app, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConn->pid, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->pid, false);
 
     char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0};
     sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port);
     varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)endpoint, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)endpoint, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConn->loginTimeMs, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->loginTimeMs, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pConn->lastAccessTimeMs, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->lastAccessTimeMs, false);
 
     numOfRows++;
   }
@@ -777,44 +777,44 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
       sprintf(&queryId[VARSTR_HEADER_SIZE], "%x:%" PRIx64, pConn->id, pQuery->reqRid);
       varDataLen(queryId) = strlen(&queryId[VARSTR_HEADER_SIZE]);
       SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)queryId, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)queryId, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->queryId, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->queryId, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pConn->id, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->id, false);
 
       char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE];
       STR_TO_VARSTR(app, pConn->app);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)app, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)app, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pConn->pid, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->pid, false);
 
       char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_TO_VARSTR(user, pConn->user);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)user, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)user, false);
 
       char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0};
       sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port);
       varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)endpoint, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)endpoint, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->stime, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->stime, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->useconds, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->useconds, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->stableQuery, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->stableQuery, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->subPlanNum, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->subPlanNum, false);
 
       char    subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0};
       int32_t strSize = sizeof(subStatus);
@@ -828,12 +828,12 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
       }
       varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, subStatus, false);
+      colDataSetVal(pColInfo, numOfRows, subStatus, false);
 
       char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_TO_VARSTR(sql, pQuery->sql);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)sql, false);
 
       numOfRows++;
     }
@@ -867,55 +867,55 @@ static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo
     cols = 0;
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->appId, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->appId, false);
 
     char ip[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0};
     sprintf(&ip[VARSTR_HEADER_SIZE], "%s", taosIpStr(pApp->ip));
     varDataLen(ip) = strlen(&ip[VARSTR_HEADER_SIZE]);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)ip, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)ip, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->pid, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->pid, false);
 
     char name[TSDB_APP_NAME_LEN + 6 + VARSTR_HEADER_SIZE] = {0};
     sprintf(&name[VARSTR_HEADER_SIZE], "%s", pApp->name);
     varDataLen(name) = strlen(&name[VARSTR_HEADER_SIZE]);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)name, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)name, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->startTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->startTime, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertsReq, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertsReq, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertRows, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertRows, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.insertElapsedTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.insertElapsedTime, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.insertBytes, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.insertBytes, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.fetchBytes, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.fetchBytes, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.queryElapsedTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.queryElapsedTime, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.numOfSlowQueries, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.numOfSlowQueries, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.totalRequests, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.totalRequests, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.currentRequests, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->summary.currentRequests, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pApp->lastAccessTimeMs, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pApp->lastAccessTimeMs, false);
 
     numOfRows++;
   }

source/dnode/mnode/impl/src/mndQnode.c

@@ -507,15 +507,15 @@ static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
 
     cols = 0;
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false);
 
     char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)ep, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)ep, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
 
     numOfRows++;
     sdbRelease(pSdb, pObj);

source/dnode/mnode/impl/src/mndScheduler.c

@@ -529,7 +529,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
   SSdb*       pSdb = pMnode->pSdb;
   SVgObj*     pVgroup = NULL;
   SQueryPlan* pPlan = NULL;
-  SSubplan*   plan = NULL;
+  SSubplan*   pSubplan = NULL;
 
   if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) {
     pPlan = qStringToQueryPlan(pTopic->physicalPlan);
@@ -545,21 +545,25 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
       return -1;
     }
 
-    SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
+    SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
 
-    int32_t opNum = LIST_LENGTH(inner->pNodeList);
+    int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList);
     if (opNum != 1) {
       qDestroyQueryPlan(pPlan);
       terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
       return -1;
     }
-    plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
+
+    pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0);
   }
 
   void* pIter = NULL;
   while (1) {
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
+
     if (!mndVgroupInDb(pVgroup, pTopic->dbUid)) {
       sdbRelease(pSdb, pVgroup);
       continue;
@@ -572,28 +576,27 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     pVgEp->vgId = pVgroup->vgId;
     taosArrayPush(pSub->unassignedVgs, &pVgEp);
 
-    mDebug("init subscription %s, assign vg: %d", pSub->key, pVgEp->vgId);
+    mDebug("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId);
 
     if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) {
       int32_t msgLen;
 
-      plan->execNode.epSet = pVgEp->epSet;
+      pSubplan->execNode.epSet = pVgEp->epSet;
-      plan->execNode.nodeId = pVgEp->vgId;
+      pSubplan->execNode.nodeId = pVgEp->vgId;
 
-      if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) {
+      if (qSubPlanToString(pSubplan, &pVgEp->qmsg, &msgLen) < 0) {
         sdbRelease(pSdb, pVgroup);
         qDestroyQueryPlan(pPlan);
         terrno = TSDB_CODE_QRY_INVALID_INPUT;
         return -1;
       }
     } else {
-      pVgEp->qmsg = strdup("");
+      pVgEp->qmsg = taosStrdup("");
     }
 
     sdbRelease(pSdb, pVgroup);
   }
 
   qDestroyQueryPlan(pPlan);
-
   return 0;
 }

source/dnode/mnode/impl/src/mndSma.c

@@ -552,14 +552,14 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   streamObj.sourceDbUid = pDb->uid;
   streamObj.targetDbUid = pDb->uid;
   streamObj.version = 1;
-  streamObj.sql = strdup(pCreate->sql);
+  streamObj.sql = taosStrdup(pCreate->sql);
   streamObj.smaId = smaObj.uid;
   streamObj.watermark = pCreate->watermark;
   streamObj.deleteMark = pCreate->deleteMark;
   streamObj.fillHistory = STREAM_FILL_HISTORY_ON;
   streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE;
   streamObj.triggerParam = pCreate->maxDelay;
-  streamObj.ast = strdup(smaObj.ast);
+  streamObj.ast = taosStrdup(smaObj.ast);
 
   // check the maxDelay
   if (streamObj.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) {
@@ -1256,19 +1256,19 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
     STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName));
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)n1, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)n1, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)n2, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)n2, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)n3, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)n3, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pSma->dstVgId, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pSma->dstVgId, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);
 
     char col[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_TO_VARSTR(col, (char *)"");

source/dnode/mnode/impl/src/mndSnode.c

@@ -441,16 +441,16 @@ static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
 
     cols = 0;
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->id, false);
 
     char ep[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(ep, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)ep, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)ep, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
 
     numOfRows++;
     sdbRelease(pSdb, pObj);

source/dnode/mnode/impl/src/mndStb.c

@@ -893,7 +893,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
 
 static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) {
   for (int32_t tag = 0; tag < pStb->numOfTags; tag++) {
-    if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) {
+    if (strcmp(pStb->pTags[tag].name, tagName) == 0) {
       return tag;
     }
   }
@@ -903,7 +903,7 @@ static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagNam
 
 static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *colName) {
   for (int32_t col = 0; col < pStb->numOfColumns; col++) {
-    if (strcasecmp(pStb->pColumns[col].name, colName) == 0) {
+    if (strcmp(pStb->pColumns[col].name, colName) == 0) {
       return col;
     }
   }
@@ -1734,7 +1734,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
   pRsp->ttl = pStb->ttl;
   pRsp->commentLen = pStb->commentLen;
   if (pStb->commentLen > 0) {
-    pRsp->pComment = strdup(pStb->comment);
+    pRsp->pComment = taosStrdup(pStb->comment);
   }
 
   for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
@@ -2743,7 +2743,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t
 //        STR_TO_VARSTR(comment, "");
 //        colDataAppend(pColInfo, numOfRows, comment, false);
 //      } else {
-//        colDataAppendNULL(pColInfo, numOfRows);
+//        colDataSetNULL(pColInfo, numOfRows);
 //      }
 //
 //      char watermark[64 + VARSTR_HEADER_SIZE] = {0};
@@ -2932,7 +2932,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
     varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE]));
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)stbName, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)stbName, false);
 
     char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
     tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB);
@@ -2940,31 +2940,31 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
     varDataSetLen(db, strlen(varDataVal(db)));
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)db, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)db, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pStb->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->createdTime, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false);  // number of tables
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pStb->updateTime, false);  // number of tables
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     if (pStb->commentLen > 0) {
       char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_TO_VARSTR(comment, pStb->comment);
-      colDataAppend(pColInfo, numOfRows, comment, false);
+      colDataSetVal(pColInfo, numOfRows, comment, false);
     } else if (pStb->commentLen == 0) {
       char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0};
       STR_TO_VARSTR(comment, "");
-      colDataAppend(pColInfo, numOfRows, comment, false);
+      colDataSetVal(pColInfo, numOfRows, comment, false);
     } else {
-      colDataAppendNULL(pColInfo, numOfRows);
+      colDataSetNULL(pColInfo, numOfRows);
     }
 
     char watermark[64 + VARSTR_HEADER_SIZE] = {0};
@@ -2972,14 +2972,14 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
     varDataSetLen(watermark, strlen(varDataVal(watermark)));
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)watermark, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)watermark, false);
 
     char maxDelay[64 + VARSTR_HEADER_SIZE] = {0};
     sprintf(varDataVal(maxDelay), "%" PRId64 "a,%" PRId64 "a", pStb->maxdelay[0], pStb->maxdelay[1]);
     varDataSetLen(maxDelay, strlen(varDataVal(maxDelay)));
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)maxDelay, false);
 
     char    rollup[160 + VARSTR_HEADER_SIZE] = {0};
     int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs);
@@ -2998,7 +2998,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
     varDataSetLen(rollup, strlen(varDataVal(rollup)));
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)rollup, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)rollup, false);
 
     numOfRows++;
     sdbRelease(pSdb, pStb);
@@ -3070,7 +3070,7 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *p
       colDataAppend(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false);
       for (int32_t k = 6; k <= 8; ++k) {
         pColInfoData = taosArrayGet(p->pDataBlock, k);
-        colDataAppendNULL(pColInfoData, numOfRows);
+        colDataSetNULL(pColInfoData, numOfRows);
       }
 
       numOfRows += 1;
@@ -3174,7 +3174,7 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
       colDataAppend(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false);
       while (cols < pShow->numOfColumns) {
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppendNULL(pColInfo, numOfRows);
+        colDataSetNULL(pColInfo, numOfRows);
       }
       numOfRows++;
     }

source/dnode/mnode/impl/src/mndStream.c

@@ -39,7 +39,7 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj
 static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
 static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
 static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
-static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
+// static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
 /*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/
 static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq);
 static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
@@ -66,8 +66,8 @@ int32_t mndInitStream(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp);
   mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp);
 
-  mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr);
+  // mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr);
-  mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
+  // mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
   mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
 
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream);
@@ -354,7 +354,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
     int32_t dataIndex = 0;
     for (int16_t i = 0; i < pObj->outputSchema.nCols; i++) {
       SColLocation *pos = taosArrayGet(pCreate->fillNullCols, nullIndex);
-      if (i < pos->slotId) {
+      if (nullIndex >= numOfNULL || i < pos->slotId) {
         pFullSchema[i].bytes = pObj->outputSchema.pSchema[dataIndex].bytes;
         pFullSchema[i].colId = i + 1;  // pObj->outputSchema.pSchema[dataIndex].colId;
         pFullSchema[i].flags = pObj->outputSchema.pSchema[dataIndex].flags;
@@ -507,7 +507,8 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
   SMCreateStbReq createReq = {0};
   tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN);
   createReq.numOfColumns = pStream->outputSchema.nCols;
-  createReq.pColumns = taosArrayInit_s(createReq.numOfColumns, sizeof(SField), createReq.numOfColumns);
+  createReq.numOfTags = 1;  // group id
+  createReq.pColumns = taosArrayInit_s(sizeof(SField), createReq.numOfColumns);
   // build fields
   for (int32_t i = 0; i < createReq.numOfColumns; i++) {
     SField *pField = taosArrayGet(createReq.pColumns, i);
@@ -519,7 +520,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
 
   if (pStream->tagSchema.nCols == 0) {
     createReq.numOfTags = 1;
-    createReq.pTags = taosArrayInit_s(createReq.numOfTags, sizeof(SField), 1);
+    createReq.pTags = taosArrayInit_s(sizeof(SField), 1);
     // build tags
     SField *pField = taosArrayGet(createReq.pTags, 0);
     strcpy(pField->name, "group_id");
@@ -528,7 +529,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
     pField->bytes = 8;
   } else {
     createReq.numOfTags = pStream->tagSchema.nCols;
-    createReq.pTags = taosArrayInit_s(createReq.numOfTags, sizeof(SField), createReq.numOfTags);
+    createReq.pTags = taosArrayInit_s(sizeof(SField), createReq.numOfTags);
     for (int32_t i = 0; i < createReq.numOfTags; i++) {
       SField *pField = taosArrayGet(createReq.pTags, i);
       pField->bytes = pStream->tagSchema.pSchema[i].bytes;
@@ -722,8 +723,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
   mInfo("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name);
 
   mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb);
-  if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER;
+  if (mndTrancCheckConflict(pMnode, pTrans) != 0) {
-
+    mndTransDrop(pTrans);
+    goto _OVER;
+  }
   // create stb for stream
   if (createStreamReq.createStb == STREAM_CREATE_STABLE_TRUE &&
       mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) {
@@ -778,6 +781,9 @@ _OVER:
   tFreeStreamObj(&streamObj);
   return code;
 }
+
+#if 0
+
 static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
   SMnode     *pMnode = pReq->info.node;
   SSdb       *pSdb = pMnode->pSdb;
@@ -942,6 +948,8 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
   return 0;
 }
 
+#endif
+
 static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
   SMnode     *pMnode = pReq->info.node;
   SStreamObj *pStream = NULL;
@@ -1099,52 +1107,52 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName));
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)streamName, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->createTime, false);
 
     char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql));
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)sql, false);
 
     char status[20 + VARSTR_HEADER_SIZE] = {0};
     char status2[20] = {0};
     mndShowStreamStatus(status2, pStream);
     STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status));
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&status, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&status, false);
 
     char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(sourceDB, mndGetDbStr(pStream->sourceDb), sizeof(sourceDB));
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&sourceDB, false);
 
     char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(targetDB, mndGetDbStr(pStream->targetDb), sizeof(targetDB));
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&targetDB, false);
 
     if (pStream->targetSTbName[0] == 0) {
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, NULL, true);
+      colDataSetVal(pColInfo, numOfRows, NULL, true);
     } else {
       char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(targetSTB, mndGetStbStr(pStream->targetSTbName), sizeof(targetSTB));
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&targetSTB, false);
     }
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->watermark, false);
 
     char trigger[20 + VARSTR_HEADER_SIZE] = {0};
     char trigger2[20] = {0};
     mndShowStreamTrigger(trigger2, pStream);
     STR_WITH_MAXSIZE_TO_VARSTR(trigger, trigger2, sizeof(trigger));
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&trigger, false);
 
     numOfRows++;
     sdbRelease(pSdb, pStream);
@@ -1196,11 +1204,11 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
         STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName));
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppend(pColInfo, numOfRows, (const char *)streamName, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false);
 
         // task id
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppend(pColInfo, numOfRows, (const char *)&pTask->taskId, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->taskId, false);
 
         // node type
         char nodeType[20 + VARSTR_HEADER_SIZE] = {0};
@@ -1211,12 +1219,12 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         } else {
           memcpy(varDataVal(nodeType), "snode", 5);
         }
-        colDataAppend(pColInfo, numOfRows, nodeType, false);
+        colDataSetVal(pColInfo, numOfRows, nodeType, false);
 
         // node id
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
         int32_t nodeId = TMAX(pTask->nodeId, 0);
-        colDataAppend(pColInfo, numOfRows, (const char *)&nodeId, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false);
 
         // level
         char level[20 + VARSTR_HEADER_SIZE] = {0};
@@ -1232,7 +1240,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         } else if (pTask->taskLevel == TASK_LEVEL__SINK) {
         }
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppend(pColInfo, numOfRows, (const char *)&level, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)&level, false);
 
         // status
         char status[20 + VARSTR_HEADER_SIZE] = {0};
@@ -1240,7 +1248,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         strcpy(status, "normal");
         STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status));
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppend(pColInfo, numOfRows, (const char *)&status, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)&status, false);
 
         numOfRows++;
       }

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -39,12 +39,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
 static int32_t  mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *);
 static int32_t  mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *);
 static int32_t  mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub);
-
+static int32_t  mndProcessRebalanceReq(SRpcMsg *pMsg);
-static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg);
+static int32_t  mndProcessDropCgroupReq(SRpcMsg *pMsg);
-static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg);
+static int32_t  mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
-
+static void     mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter);
-static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
-static void    mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter);
 
 static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) {
   SSdbRaw *pRedoRaw = mndSubActionEncode(pSub);
@@ -85,12 +83,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
   return sdbSetTable(pMnode->pSdb, table);
 }
 
-static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) {
+static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey) {
   SMqSubscribeObj *pSub = tNewSubscribeObj(subKey);
   if (pSub == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
+
   pSub->dbUid = pTopic->dbUid;
   pSub->stbUid = pTopic->stbUid;
   pSub->subType = pTopic->subType;
@@ -123,6 +122,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
+
   SMsgHead *pMsgHead = (SMsgHead *)buf;
 
   pMsgHead->contLen = htonl(tlen);
@@ -205,7 +205,7 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
 static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
   int32_t     totalVgNum = pOutput->pSub->vgNum;
   const char *sub = pOutput->pSub->key;
-  mInfo("sub:%s, mq rebalance vgNum:%d", sub, pOutput->pSub->vgNum);
+  mInfo("sub:%s mq re-balance %d vgroups", sub, pOutput->pSub->vgNum);
 
   // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
   SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
@@ -214,7 +214,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
   int32_t actualRemoved = 0;
   for (int32_t i = 0; i < removedNum; i++) {
-    int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
+    uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
 
     SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
 
@@ -229,7 +229,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
             .pVgEp = pVgEp,
         };
         taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-        mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64, sub, pVgEp->vgId, consumerId);
+        mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRId64, sub, pVgEp->vgId, consumerId);
       }
       taosArrayDestroy(pConsumerEp->vgs);
       taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
@@ -239,7 +239,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   }
 
   if (removedNum != actualRemoved) {
-    mError("sub:%s, mq rebalance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved);
+    mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved);
   }
 
   // if previously no consumer, there are vgs not assigned
@@ -253,7 +253,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
           .pVgEp = pVgEp,
       };
       taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
-      mInfo("sub:%s, mq rebalance remove vgId:%d from unassigned", sub, pVgEp->vgId);
+      mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", sub, pVgEp->vgId);
     }
   }
 
@@ -267,7 +267,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     minVgCnt = totalVgNum / afterRebConsumerNum;
     imbConsumerNum = totalVgNum % afterRebConsumerNum;
   }
-  mInfo("sub:%s, mq rebalance %d consumer after rebalance, at least %d vg each, %d consumer has more vg", sub,
+
+  mInfo("sub:%s mq re-balance %d consumers: at least %d vg each, %d consumer has more vg", sub,
         afterRebConsumerNum, minVgCnt, imbConsumerNum);
 
   // 4. first scan: remove consumer more than wanted, put to remove hash
@@ -275,7 +276,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   void   *pIter = NULL;
   while (1) {
     pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
+
     SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
 
     int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
@@ -297,7 +301,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
                 .pVgEp = pVgEp,
             };
             taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-            mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId,
+            mInfo("sub:%s mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId,
                   pConsumerEp->consumerId);
           }
           imbCnt++;
@@ -312,7 +316,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
               .pVgEp = pVgEp,
           };
           taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-          mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId,
+          mInfo("sub:%s mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId,
                 pConsumerEp->consumerId);
         }
       }
@@ -330,7 +334,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
       taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
       taosArrayPush(pOutput->newConsumers, &consumerId);
-      mInfo("sub:%s, mq rebalance add new consumer:%" PRId64, sub, consumerId);
+      mInfo("sub:%s mq rebalance add new consumer:%" PRId64, sub, consumerId);
     }
   }
 
@@ -349,7 +353,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       // iter hash and find one vg
       pRemovedIter = taosHashIterate(pHash, pRemovedIter);
       if (pRemovedIter == NULL) {
-        mError("sub:%s, removed iter is null", sub);
+        mError("sub:%s removed iter is null", sub);
         continue;
       }
 
@@ -402,33 +406,36 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     SMqRebOutputVg *pRebOutput = NULL;
     while (1) {
       pIter = taosHashIterate(pHash, pIter);
-      if (pIter == NULL) break;
+      if (pIter == NULL) {
+        break;
+      }
+
       pRebOutput = (SMqRebOutputVg *)pIter;
 
       taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
       taosArrayPush(pOutput->rebVgs, pRebOutput);
-      mInfo("sub:%s, mq rebalance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId);
+      mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId);
     }
   }
 
   // 8. generate logs
-  mInfo("sub:%s, mq rebalance calculation completed, rebalanced vg", sub);
+  mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", sub);
   for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
     SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
-    mInfo("sub:%s, mq rebalance vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, sub,
+    mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, sub,
           pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId);
   }
   {
-    void *pIter = NULL;
+    pIter = NULL;
     while (1) {
       pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
       if (pIter == NULL) break;
       SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
       int32_t        sz = taosArrayGetSize(pConsumerEp->vgs);
-      mInfo("sub:%s, mq rebalance final cfg: consumer %" PRId64 " has %d vg", sub, pConsumerEp->consumerId, sz);
+      mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRId64 " has %d vg", sub, pConsumerEp->consumerId, sz);
       for (int32_t i = 0; i < sz; i++) {
         SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
-        mInfo("sub:%s, mq rebalance final cfg: vg %d to consumer %" PRId64 "", sub, pVgEp->vgId,
+        mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRId64, sub, pVgEp->vgId,
               pConsumerEp->consumerId);
       }
     }
@@ -552,11 +559,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
   SMqDoRebalanceMsg *pReq = pMsg->pCont;
   void              *pIter = NULL;
 
-  mInfo("mq rebalance start");
+  mInfo("mq re-balance start");
 
   while (1) {
     pIter = taosHashIterate(pReq->rebSubHash, pIter);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
+
     SMqRebInputObj rebInput = {0};
 
     SMqRebOutputObj rebOutput = {0};
@@ -577,12 +587,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
       mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true);
       SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
       if (pTopic == NULL) {
-        mError("mq rebalance %s failed since topic %s not exist, abort", pRebInfo->key, topic);
+        mError("mq re-balance %s ignored since topic %s not exist", pRebInfo->key, topic);
         continue;
       }
+
       taosRLockLatch(&pTopic->lock);
 
-      rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key);
+      rebOutput.pSub = mndCreateSubscription(pMnode, pTopic, pRebInfo->key);
 
       if (rebOutput.pSub == NULL) {
         mError("mq rebalance %s failed create sub since %s, abort", pRebInfo->key, terrstr());
@@ -605,15 +616,16 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
     }
 
     if (mndDoRebalance(pMnode, &rebInput, &rebOutput) < 0) {
-      mError("mq rebalance internal error");
+      mError("mq re-balance internal error");
     }
 
     // if add more consumer to balanced subscribe,
     // possibly no vg is changed
 
     if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) {
-      mError("mq rebalance persist rebalance output error, possibly vnode splitted or dropped");
+      mError("mq re-balance persist re-balance output error, possibly vnode splitted or dropped");
     }
+
     taosArrayDestroy(pRebInfo->lostConsumers);
     taosArrayDestroy(pRebInfo->newConsumers);
     taosArrayDestroy(pRebInfo->removedConsumers);
@@ -627,19 +639,18 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
   }
 
   // reset flag
-  mInfo("mq rebalance completed successfully");
+  mInfo("mq re-balance completed successfully");
   taosHashCleanup(pReq->rebSubHash);
   mndRebEnd();
 
   return 0;
 }
 
-static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) {
+static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
-  SMnode         *pMnode = pReq->info.node;
+  SMnode         *pMnode = pMsg->info.node;
-  SSdb           *pSdb = pMnode->pSdb;
   SMDropCgroupReq dropReq = {0};
 
-  if (tDeserializeSMDropCgroupReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+  if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;
   }
@@ -663,7 +674,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pReq) {
     return -1;
   }
 
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-cgroup");
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "drop-cgroup");
   if (pTrans == NULL) {
     mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr());
     mndReleaseSubscribe(pMnode, pSub);
@@ -956,7 +967,7 @@ END:
   return code;
 }
 
-static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
+int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
   SMnode          *pMnode = pReq->info.node;
   SSdb            *pSdb = pMnode->pSdb;
   int32_t          numOfRows = 0;
@@ -998,18 +1009,18 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         varDataSetLen(cgroup, strlen(varDataVal(cgroup)));
 
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppend(pColInfo, numOfRows, (const char *)topic, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)topic, false);
 
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false);
 
         // vg id
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppend(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false);
 
         // consumer id
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppend(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false);
 
         mDebug("mnd show subscriptions: topic %s, consumer %" PRId64 " cgroup %s vgid %d", varDataVal(topic),
                pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId);
@@ -1018,11 +1029,11 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
 #if 0
       // subscribe time
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false);
 
       // rebalance time
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0);
 #endif
 
         numOfRows++;
@@ -1046,18 +1057,18 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       varDataSetLen(cgroup, strlen(varDataVal(cgroup)));
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)topic, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)topic, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false);
 
       // vg id
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pVgEp->vgId, false);
 
       // consumer id
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, NULL, true);
+      colDataSetVal(pColInfo, numOfRows, NULL, true);
 
       mDebug("mnd show subscriptions(unassigned): topic %s, cgroup %s vgid %d", varDataVal(topic), varDataVal(cgroup),
              pVgEp->vgId);
@@ -1066,11 +1077,11 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
 #if 0
       // subscribe time
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->subscribeTime, false);
 
       // rebalance time
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pSub->rebalanceTime, pConsumer->rebalanceTime == 0);
 #endif
 
       numOfRows++;
@@ -1090,7 +1101,7 @@ static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
   return numOfRows;
 }
 
-static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter) {
+void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter) {
   SSdb *pSdb = pMnode->pSdb;
   sdbCancelFetch(pSdb, pIter);
 }

source/dnode/mnode/impl/src/mndTopic.c

@@ -31,9 +31,12 @@
 #define MND_TOPIC_VER_NUMBER   2
 #define MND_TOPIC_RESERVE_SIZE 64
 
+SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic);
+SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw);
+
 static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic);
 static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic);
-static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic);
+static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic);
 static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq);
 static int32_t mndProcessDropTopicReq(SRpcMsg *pReq);
 
@@ -79,6 +82,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   if (pTopic->physicalPlan) {
     physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
   }
+
   int32_t schemaLen = 0;
   if (pTopic->schema.nCols) {
     schemaLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
@@ -88,7 +92,9 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   int32_t size = sizeof(SMqTopicObj) + physicalPlanLen + pTopic->sqlLen + pTopic->astLen + schemaLen + ntbColLen +
                  MND_TOPIC_RESERVE_SIZE;
   SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size);
-  if (pRaw == NULL) goto TOPIC_ENCODE_OVER;
+  if (pRaw == NULL) {
+    goto TOPIC_ENCODE_OVER;
+  }
 
   int32_t dataPos = 0;
   SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_ENCODE_OVER);
@@ -106,6 +112,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER);
   SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER);
   SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER);
+
   if (pTopic->astLen) {
     SDB_SET_BINARY(pRaw, dataPos, pTopic->ast, pTopic->astLen, TOPIC_ENCODE_OVER);
   }
@@ -123,6 +130,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
     taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema);
     SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER);
   }
+
   SDB_SET_INT64(pRaw, dataPos, pTopic->ntbUid, TOPIC_ENCODE_OVER);
   if (pTopic->ntbUid != 0) {
     int32_t sz = taosArrayGetSize(pTopic->ntbColIds);
@@ -132,6 +140,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
       SDB_SET_INT16(pRaw, dataPos, colId, TOPIC_ENCODE_OVER);
     }
   }
+
   SDB_SET_INT64(pRaw, dataPos, pTopic->ctbStbUid, TOPIC_ENCODE_OVER);
 
   SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER);
@@ -247,10 +256,9 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
     SDB_GET_INT16(pRaw, dataPos, &colId, TOPIC_DECODE_OVER);
     taosArrayPush(pTopic->ntbColIds, &colId);
   }
+
   SDB_GET_INT64(pRaw, dataPos, &pTopic->ctbStbUid, TOPIC_DECODE_OVER);
-
   SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER);
-
   terrno = TSDB_CODE_SUCCESS;
 
 TOPIC_DECODE_OVER:
@@ -266,12 +274,12 @@ TOPIC_DECODE_OVER:
 }
 
 static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic) {
-  mTrace("topic:%s, perform insert action", pTopic->name);
+  mTrace("topic:%s perform insert action", pTopic->name);
   return 0;
 }
 
 static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) {
-  mTrace("topic:%s, perform delete action", pTopic->name);
+  mTrace("topic:%s perform delete action", pTopic->name);
   taosMemoryFreeClear(pTopic->sql);
   taosMemoryFreeClear(pTopic->ast);
   taosMemoryFreeClear(pTopic->physicalPlan);
@@ -281,7 +289,7 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) {
 }
 
 static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) {
-  mTrace("topic:%s, perform update action", pOldTopic->name);
+  mTrace("topic:%s perform update action", pOldTopic->name);
   atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime);
   atomic_exchange_32(&pOldTopic->version, pNewTopic->version);
 
@@ -364,7 +372,8 @@ static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) {
 
 static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb,
                               const char *userName) {
-  mInfo("topic:%s to create", pCreate->name);
+  mInfo("start to create topic:%s", pCreate->name);
+
   SMqTopicObj topicObj = {0};
   tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
   tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
@@ -379,23 +388,22 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
   topicObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
   topicObj.dbUid = pDb->uid;
   topicObj.version = 1;
-  topicObj.sql = strdup(pCreate->sql);
+  topicObj.sql = taosStrdup(pCreate->sql);
   topicObj.sqlLen = strlen(pCreate->sql) + 1;
   topicObj.subType = pCreate->subType;
   topicObj.withMeta = pCreate->withMeta;
-  if (topicObj.withMeta) {
+
-    if (topicObj.subType == TOPIC_SUB_TYPE__COLUMN) {
+  if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
+    if (pCreate->withMeta) {
       terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
       mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
       return -1;
     }
-  }
 
-  if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
+    topicObj.ast = taosStrdup(pCreate->ast);
-    topicObj.ast = strdup(pCreate->ast);
     topicObj.astLen = strlen(pCreate->ast) + 1;
 
-    qDebugL("ast %s", topicObj.ast);
+    qDebugL("topic:%s ast %s", topicObj.name, topicObj.ast);
 
     SNode *pAst = NULL;
     if (nodesStringToNode(pCreate->ast, &pAst) != 0) {
@@ -409,18 +417,20 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
 
     SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
     if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) {
-      mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+      mError("failed to create topic:%s since %s", pCreate->name, terrstr());
       taosMemoryFree(topicObj.ast);
       taosMemoryFree(topicObj.sql);
       return -1;
     }
 
-    int64_t ntbUid;
     topicObj.ntbColIds = taosArrayInit(0, sizeof(int16_t));
     if (topicObj.ntbColIds == NULL) {
+      taosMemoryFree(topicObj.ast);
+      taosMemoryFree(topicObj.sql);
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
+
     extractTopicTbInfo(pAst, &topicObj);
 
     if (topicObj.ntbUid == 0) {
@@ -449,6 +459,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
       terrno = TSDB_CODE_MND_STB_NOT_EXIST;
       return -1;
     }
+
     topicObj.stbUid = pStb->uid;
     mndReleaseStb(pMnode, pStb);
   }
@@ -467,7 +478,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
     taosMemoryFreeClear(topicObj.physicalPlan);
     return -1;
   }
-  mInfo("trans:%d, used to create topic:%s", pTrans->id, pCreate->name);
+
+  mInfo("trans:%d to create topic:%s", pTrans->id, pCreate->name);
 
   SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj);
   if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
@@ -476,6 +488,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
     mndTransDrop(pTrans);
     return -1;
   }
+
   (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
 
   if (topicObj.ntbUid != 0) {
@@ -544,7 +557,11 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
   taosMemoryFreeClear(topicObj.sql);
   taosMemoryFreeClear(topicObj.ast);
   taosArrayDestroy(topicObj.ntbColIds);
-  if (topicObj.schema.nCols) taosMemoryFreeClear(topicObj.schema.pSchema);
+
+  if (topicObj.schema.nCols) {
+    taosMemoryFreeClear(topicObj.schema.pSchema);
+  }
+
   mndTransDrop(pTrans);
   return TSDB_CODE_ACTION_IN_PROGRESS;
 }
@@ -561,17 +578,17 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  mInfo("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql);
+  mInfo("topic:%s start to create, sql:%s", createTopicReq.name, createTopicReq.sql);
 
   if (mndCheckCreateTopicReq(&createTopicReq) != 0) {
-    mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr());
+    mError("topic:%s failed to create since %s", createTopicReq.name, terrstr());
     goto _OVER;
   }
 
   pTopic = mndAcquireTopic(pMnode, createTopicReq.name);
   if (pTopic != NULL) {
     if (createTopicReq.igExists) {
-      mInfo("topic:%s, already exist, ignore exist is set", createTopicReq.name);
+      mInfo("topic:%s already exist, ignore exist is set", createTopicReq.name);
       code = 0;
       goto _OVER;
     } else {
@@ -589,11 +606,13 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
   }
 
   code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb, pReq->info.conn.user);
-  if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
+  if (code == 0) {
+    code = TSDB_CODE_ACTION_IN_PROGRESS;
+  }
 
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
-    mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr());
+    mError("failed to create topic:%s since %s", createTopicReq.name, terrstr());
   }
 
   mndReleaseTopic(pMnode, pTopic);
@@ -605,13 +624,18 @@ _OVER:
 
 static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) {
   int32_t code = -1;
-  if (mndUserRemoveTopic(pMnode, pTrans, pTopic->name) != 0) goto _OVER;
+  if (mndUserRemoveTopic(pMnode, pTrans, pTopic->name) != 0) {
+    goto _OVER;
+  }
 
   SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic);
   if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER;
   (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
 
-  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    goto _OVER;
+  }
+
   code = 0;
 
 _OVER:
@@ -650,7 +674,9 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
   SMqConsumerObj *pConsumer;
   while (1) {
     pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
 
     if (pConsumer->status == MQ_CONSUMER_STATUS__LOST_REBD) continue;
 
@@ -661,8 +687,8 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
         mndReleaseConsumer(pMnode, pConsumer);
         mndReleaseTopic(pMnode, pTopic);
         terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
-        mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s", dropReq.name,
+        mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s",
-               pConsumer->consumerId, pConsumer->cgroup);
+               dropReq.name, pConsumer->consumerId, pConsumer->cgroup);
         return -1;
       }
     }
@@ -726,8 +752,9 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
 
   if (pTopic->ntbUid != 0) {
     // broadcast to all vnode
-    void   *pIter = NULL;
+    pIter = NULL;
     SVgObj *pVgroup = NULL;
+
     while (1) {
       pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
       if (pIter == NULL) break;
@@ -767,6 +794,8 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
 }
 
 static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
+  *pNumOfTopics = 0;
+
   SSdb   *pSdb = pMnode->pSdb;
   SDbObj *pDb = mndAcquireDb(pMnode, dbName);
   if (pDb == NULL) {
@@ -779,7 +808,9 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo
   while (1) {
     SMqTopicObj *pTopic = NULL;
     pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
 
     if (pTopic->dbUid == pDb->uid) {
       numOfTopics++;
@@ -807,29 +838,29 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
     SName            n;
     int32_t          cols = 0;
 
-    char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0};
+    char        topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0};
-    tstrncpy(varDataVal(topicName), mndGetDbStr(pTopic->name), sizeof(topicName) - 2);
+    const char *pName = mndGetDbStr(pTopic->name);
-    /*tNameFromString(&n, pTopic->name, T_NAME_ACCT | T_NAME_DB);*/
+    STR_TO_VARSTR(topicName, pName);
-    /*tNameGetDbName(&n, varDataVal(topicName));*/
+
-    varDataSetLen(topicName, strlen(varDataVal(topicName)));
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false);
 
     char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
     tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB);
     tNameGetDbName(&n, varDataVal(dbName));
     varDataSetLen(dbName, strlen(varDataVal(dbName)));
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)dbName, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)dbName, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pTopic->createTime, false);
 
     char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
-    tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN);
+    STR_TO_VARSTR(sql, pTopic->sql);
-    varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
+
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)sql, false);
 
     numOfRows++;
     sdbRelease(pSdb, pTopic);
@@ -862,24 +893,26 @@ static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) {
   sdbCancelFetch(pSdb, pIter);
 }
 
-int32_t mndCheckTopicExist(SMnode *pMnode, SDbObj *pDb) {
+bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb) {
-  SSdb *pSdb = pMnode->pSdb;
+  SSdb        *pSdb = pMnode->pSdb;
-
   void        *pIter = NULL;
   SMqTopicObj *pTopic = NULL;
+
   while (1) {
     pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
 
     if (pTopic->dbUid == pDb->uid) {
       sdbRelease(pSdb, pTopic);
-      terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED;
+      return true;
-      return -1;
     }
 
     sdbRelease(pSdb, pTopic);
   }
-  return 0;
+
+  return false;
 }
 
 #if 0

source/dnode/mnode/impl/src/mndTrans.c

@@ -655,6 +655,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
     taosArrayPush(pTrans->pRpcArray, &pReq->info);
     pTrans->originRpcType = pReq->msgType;
   }
+
   mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans);
   return pTrans;
 }
@@ -1664,36 +1665,36 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
     cols = 0;
 
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->id, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->id, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->createdTime, false);
 
     char stage[TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(stage, mndTransStr(pTrans->stage), pShow->pMeta->pSchemas[cols].bytes);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)stage, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)stage, false);
 
     char opername[TSDB_TRANS_OPER_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(opername, pTrans->opername, pShow->pMeta->pSchemas[cols].bytes);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)opername, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)opername, false);
 
     char dbname[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndGetDbStr(pTrans->dbname), pShow->pMeta->pSchemas[cols].bytes);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)dbname, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)dbname, false);
 
     char stbname[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(stbname, mndGetDbStr(pTrans->stbname), pShow->pMeta->pSchemas[cols].bytes);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)stbname, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)stbname, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->failedTimes, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->failedTimes, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->lastExecTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->lastExecTime, false);
 
     char    lastInfo[TSDB_TRANS_ERROR_LEN + VARSTR_HEADER_SIZE] = {0};
     char    detail[TSDB_TRANS_ERROR_LEN + 1] = {0};
@@ -1709,7 +1710,7 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
     }
     STR_WITH_MAXSIZE_TO_VARSTR(lastInfo, detail, pShow->pMeta->pSchemas[cols].bytes);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)lastInfo, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)lastInfo, false);
 
     numOfRows++;
     sdbRelease(pSdb, pTrans);

source/dnode/mnode/impl/src/mndUser.c

@@ -795,23 +795,23 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     char             name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
-    colDataAppend(pColInfo, numOfRows, (const char *)name, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)name, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pUser->superUser, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->superUser, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pUser->enable, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->enable, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->createdTime, false);
 
     numOfRows++;
     sdbRelease(pSdb, pUser);
@@ -848,17 +848,17 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
       SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)userName, false);
 
       char privilege[20] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(privilege, "all", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false);
 
       char objName[20] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)objName, false);
 
       numOfRows++;
     }
@@ -869,12 +869,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
       SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)userName, false);
 
       char privilege[20] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(privilege, "read", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false);
 
       SName name = {0};
       char  objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -882,7 +882,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       tNameGetDbName(&name, varDataVal(objName));
       varDataSetLen(objName, strlen(varDataVal(objName)));
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)objName, false);
 
       numOfRows++;
       db = taosHashIterate(pUser->readDbs, db);
@@ -894,12 +894,12 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
       SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)userName, false);
 
       char privilege[20] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(privilege, "write", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false);
 
       SName name = {0};
       char  objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -907,7 +907,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       tNameGetDbName(&name, varDataVal(objName));
       varDataSetLen(objName, strlen(varDataVal(objName)));
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)objName, false);
 
       numOfRows++;
       db = taosHashIterate(pUser->writeDbs, db);
@@ -919,18 +919,18 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
       SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)userName, false);
 
       char privilege[20] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(privilege, "subscribe", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false);
 
       char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0};
       tstrncpy(varDataVal(topicName), mndGetDbStr(topic), TSDB_TOPIC_NAME_LEN - 2);
       varDataSetLen(topicName, strlen(varDataVal(topicName)));
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false);
 
       numOfRows++;
       topic = taosHashIterate(pUser->topics, topic);
@@ -1062,10 +1062,14 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) {
 
   while (1) {
     pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
 
     code = -1;
-    if (mndUserDupObj(pUser, &newUser) != 0) break;
+    if (mndUserDupObj(pUser, &newUser) != 0) {
+      break;
+    }
 
     bool inTopic = (taosHashGet(newUser.topics, topic, len) != NULL);
     if (inTopic) {

source/dnode/mnode/impl/src/mndVgroup.c

@@ -529,10 +529,10 @@ SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) {
 }
 
 static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) {
-    if (*dnode1Id == *dnode2Id) {
+  if (*dnode1Id == *dnode2Id) {
-        return 0;
+    return 0;
-    }
+  }
-    return *dnode1Id > *dnode2Id ? 1 : -1;
+  return *dnode1Id > *dnode2Id ? 1 : -1;
 }
 
 static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float ratio) {
@@ -541,12 +541,12 @@ static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float r
 }
 
 static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) {
-    float d1Score = mndGetDnodeScore(pDnode1, 0, 0.9);
+  float d1Score = mndGetDnodeScore(pDnode1, 0, 0.9);
-    float d2Score = mndGetDnodeScore(pDnode2, 0, 0.9);
+  float d2Score = mndGetDnodeScore(pDnode2, 0, 0.9);
-    if (d1Score == d2Score) {
+  if (d1Score == d2Score) {
-        return 0;
+    return 0;
-    }
+  }
-    return d1Score > d2Score ? 1 : -1;
+  return d1Score > d2Score ? 1 : -1;
 }
 
 void mndSortVnodeGid(SVgObj *pVgroup) {
@@ -739,7 +739,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
 
     cols = 0;
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false);
 
     SName name = {0};
     char  db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -748,17 +748,17 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
     varDataSetLen(db, strlen(varDataVal(db)));
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)db, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)db, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->numOfTables, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->numOfTables, false);
 
     // default 3 replica, add 1 replica if move vnode
     for (int32_t i = 0; i < 4; ++i) {
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       if (i < pVgroup->replica) {
         int16_t dnodeId = (int16_t)pVgroup->vnodeGid[i].dnodeId;
-        colDataAppend(pColInfo, numOfRows, (const char *)&dnodeId, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)&dnodeId, false);
 
         bool       exist = false;
         bool       online = false;
@@ -789,24 +789,24 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
         STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes);
 
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppend(pColInfo, numOfRows, (const char *)buf1, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)buf1, false);
       } else {
-        colDataAppendNULL(pColInfo, numOfRows);
+        colDataSetNULL(pColInfo, numOfRows);
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataAppendNULL(pColInfo, numOfRows);
+        colDataSetNULL(pColInfo, numOfRows);
       }
     }
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     int32_t cacheUsage = (int32_t)pVgroup->cacheUsage;
-    colDataAppend(pColInfo, numOfRows, (const char *)&cacheUsage, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&cacheUsage, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false);
+    colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false);
 
     // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     // if (pDb == NULL || pDb->compactStartTime <= 0) {
-    //   colDataAppendNULL(pColInfo, numOfRows);
+    //   colDataSetNULL(pColInfo, numOfRows);
     // } else {
     //   colDataAppend(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false);
     // }
@@ -905,15 +905,15 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
       cols = 0;
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->replica, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->replica, false);
 
       char buf[20] = {0};
       STR_TO_VARSTR(buf, syncStr(pVgid->syncState));
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)buf, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)buf, false);
 
       const char *dbname = mndGetDbStr(pVgroup->dbName);
       char        b1[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -923,10 +923,10 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
         STR_WITH_MAXSIZE_TO_VARSTR(b1, "NULL", TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE);
       }
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)b1, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)b1, false);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)&pVgid->dnodeId, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pVgid->dnodeId, false);
 
       SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
       char       b2[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -936,7 +936,7 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
         STR_WITH_MAXSIZE_TO_VARSTR(b2, "NULL", TSDB_EP_LEN + VARSTR_HEADER_SIZE);
       }
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataAppend(pColInfo, numOfRows, (const char *)b2, false);
+      colDataSetVal(pColInfo, numOfRows, (const char *)b2, false);
 
       numOfRows++;
     }

source/dnode/mnode/impl/test/trans/trans2.cpp

@@ -124,7 +124,7 @@ class MndTestTrans2 : public ::testing::Test {
     mndTransAppendUndolog(pTrans, pUndoRaw);
     sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
 
-    char *param = strdup("====> test log <=====");
+    char *param = taosStrdup("====> test log <=====");
     mndTransSetCb(pTrans, TRANS_START_FUNC_TEST, TRANS_STOP_FUNC_TEST, param, strlen(param) + 1);
 
     if (pDb != NULL) {
@@ -157,7 +157,7 @@ class MndTestTrans2 : public ::testing::Test {
     mndTransAppendUndolog(pTrans, pUndoRaw);
     sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
 
-    char *param = strdup("====> test action <=====");
+    char *param = taosStrdup("====> test action <=====");
     mndTransSetCb(pTrans, TRANS_START_FUNC_TEST, TRANS_STOP_FUNC_TEST, param, strlen(param) + 1);
 
     {
@@ -229,7 +229,7 @@ class MndTestTrans2 : public ::testing::Test {
     mndTransAppendUndolog(pTrans, pUndoRaw);
     sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
 
-    char *param = strdup("====> test log <=====");
+    char *param = taosStrdup("====> test log <=====");
     mndTransSetCb(pTrans, TRANS_START_FUNC_TEST, TRANS_STOP_FUNC_TEST, param, strlen(param) + 1);
 
     int32_t code = mndTransPrepare(pMnode, pTrans);

source/dnode/mnode/sdb/src/sdb.c

@@ -30,9 +30,9 @@ SSdb *sdbInit(SSdbOpt *pOption) {
 
   char path[PATH_MAX + 100] = {0};
   snprintf(path, sizeof(path), "%s%sdata", pOption->path, TD_DIRSEP);
-  pSdb->currDir = strdup(path);
+  pSdb->currDir = taosStrdup(path);
   snprintf(path, sizeof(path), "%s%stmp", pOption->path, TD_DIRSEP);
-  pSdb->tmpDir = strdup(path);
+  pSdb->tmpDir = taosStrdup(path);
   if (pSdb->currDir == NULL || pSdb->tmpDir == NULL) {
     sdbCleanup(pSdb);
     terrno = TSDB_CODE_OUT_OF_MEMORY;

source/dnode/mnode/sdb/src/sdbFile.c

@@ -472,7 +472,7 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) {
 
   taosThreadMutexLock(&pSdb->filelock);
   if (pSdb->pWal != NULL) {
-    // code = walBeginSnapshot(pSdb->pWal, pSdb->applyIndex);
+    // code = walBeginSnapshot(pSdb->pWal, pSdb->applyIndex, 0);
     if (pSdb->sync == 0) {
       code = 0;
     } else {
@@ -521,7 +521,7 @@ static SSdbIter *sdbCreateIter(SSdb *pSdb) {
 
   char name[PATH_MAX + 100] = {0};
   snprintf(name, sizeof(name), "%s%ssdb.data.%" PRIu64, pSdb->tmpDir, TD_DIRSEP, (uint64_t)pIter);
-  pIter->name = strdup(name);
+  pIter->name = taosStrdup(name);
   if (pIter->name == NULL) {
     taosMemoryFree(pIter);
     terrno = TSDB_CODE_OUT_OF_MEMORY;

source/dnode/qnode/inc/qndInt.h

@@ -29,7 +29,7 @@
 extern "C" {
 #endif
 
-typedef struct SQWorker SQHandle;
+typedef struct SQueueWorker SQHandle;
 
 typedef struct SQnode {
   int32_t   qndId;

source/dnode/snode/src/snode.c

@@ -104,7 +104,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
-  pSnode->path = strdup(path);
+  pSnode->path = taosStrdup(path);
   if (pSnode->path == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     goto FAIL;

source/dnode/vnode/src/inc/vnodeInt.h

@@ -58,7 +58,7 @@ typedef struct STQ                STQ;
 typedef struct SVState            SVState;
 typedef struct SVStatis           SVStatis;
 typedef struct SVBufPool          SVBufPool;
-typedef struct SQWorker           SQHandle;
+typedef struct SQueueWorker       SQHandle;
 typedef struct STsdbKeepCfg       STsdbKeepCfg;
 typedef struct SMetaSnapReader    SMetaSnapReader;
 typedef struct SMetaSnapWriter    SMetaSnapWriter;

source/dnode/vnode/src/meta/metaQuery.c

@@ -706,7 +706,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
     }
   }
 
-  if (ASSERTS(sver > 0, __FILE__, __LINE__, "failed to get table schema version: %d", sver)) {
+  if (ASSERTS(sver > 0, "failed to get table schema version: %d", sver)) {
     code = TSDB_CODE_NOT_FOUND;
     goto _exit;
   }

source/dnode/vnode/src/meta/metaSnapshot.c

@@ -252,7 +252,7 @@ static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo)
     return;
   }
   STableInfoForChildTable dataTmp = {0};
-  dataTmp.tableName = strdup(me->name);
+  dataTmp.tableName = taosStrdup(me->name);
 
   dataTmp.schemaRow = tCloneSSchemaWrapper(&me->stbEntry.schemaRow);
   dataTmp.tagRow = tCloneSSchemaWrapper(&me->stbEntry.schemaTag);

source/dnode/vnode/src/sma/smaEnv.c

@@ -291,10 +291,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
     // step 1: set rsma trigger stat cancelled
     atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
 
-    // step 2: destroy the rsma info and associated fetch tasks
+    // step 2: wait for all triggered fetch tasks to finish
-    taosHashCleanup(RSMA_INFO_HASH(pStat));
-
-    // step 3: wait for all triggered fetch tasks to finish
     int32_t nLoops = 0;
     while (1) {
       if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) {
@@ -310,9 +307,12 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
       }
     }
 
-    // step 4:
+    // step 3:
     tdRsmaStopExecutor(pSma);
 
+    // step 4: destroy the rsma info and associated fetch tasks
+    taosHashCleanup(RSMA_INFO_HASH(pStat));
+
     // step 5:
     tdRSmaFSClose(RSMA_FS(pStat));