Merge branch '3.0' into docs/daniel-3.0

Jenkinsfile2

@@ -43,6 +43,7 @@ def pre_test(){
         cd ${WKC}
         git reset --hard
         git clean -fxd
+        rm -rf examples/rust/
         git remote prune origin
         git fetch
     '''

cmake/cmake.install

@@ -1,38 +1,8 @@
-IF (EXISTS /var/lib/taos/dnode/dnodeCfg.json)
+IF (TD_LINUX)
-  INSTALL(CODE "MESSAGE(\"The default data directory /var/lib/taos contains old data of tdengine 2.x, please clear it before installing!\")")
-ELSEIF (EXISTS C:/TDengine/data/dnode/dnodeCfg.json)
-  INSTALL(CODE "MESSAGE(\"The default data directory C:/TDengine/data contains old data of tdengine 2.x, please clear it before installing!\")")
-ELSEIF (TD_LINUX)
   SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh")
   INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
   INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})")
 ELSEIF (TD_WINDOWS)
-  SET(CMAKE_INSTALL_PREFIX C:/TDengine)
-
-  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go     DESTINATION connector)
-  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector)
-  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector)
-  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector)
-  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .)
-  INSTALL(CODE "IF (NOT EXISTS ${CMAKE_INSTALL_PREFIX}/cfg/taos.cfg)
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg ${CMAKE_INSTALL_PREFIX}/cfg/taos.cfg)
-  ENDIF ()")
-  INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include)
-  INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include)
-  INSTALL(FILES ${TD_SOURCE_DIR}/include/libs/function/taosudf.h DESTINATION include)
-  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
-  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver)
-  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)
-  INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .)
-  INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .)
-  INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .)
-  IF (BUILD_TOOLS)
-    INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosBenchmark.exe DESTINATION .)
-  ENDIF ()
-
-  IF (TD_MVN_INSTALLED)
-    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc)
-  ENDIF ()
   SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat")
   INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
   INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})")

cmake/taosadapter_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG 3d21433
+        GIT_TAG abed566
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

contrib/CMakeLists.txt

@@ -136,24 +136,6 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
 execute_process(COMMAND "${CMAKE_COMMAND}" --build .
         WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
 
-# clear submodule
-execute_process(COMMAND git submodule deinit -f tools/taos-tools
-        WORKING_DIRECTORY "${TD_SOURCE_DIR}")
-execute_process(COMMAND git rm --cached tools/taos-tools
-        WORKING_DIRECTORY "${TD_SOURCE_DIR}")
-execute_process(COMMAND git submodule deinit -f tools/taosadapter
-        WORKING_DIRECTORY "${TD_SOURCE_DIR}")
-execute_process(COMMAND git rm --cached tools/taosadapter
-        WORKING_DIRECTORY "${TD_SOURCE_DIR}")
-execute_process(COMMAND git submodule deinit -f tools/taosws-rs
-        WORKING_DIRECTORY "${TD_SOURCE_DIR}")
-execute_process(COMMAND git rm --cached tools/taosws-rs
-        WORKING_DIRECTORY "${TD_SOURCE_DIR}")
-execute_process(COMMAND git submodule deinit -f examples/rust
-        WORKING_DIRECTORY "${TD_SOURCE_DIR}")
-execute_process(COMMAND git rm --cached examples/rust
-        WORKING_DIRECTORY "${TD_SOURCE_DIR}")
-
 # ================================================================================================
 # Build
 # ================================================================================================
@@ -349,9 +331,11 @@ endif(${BUILD_WITH_TRAFT})
 
 # LIBUV
 if(${BUILD_WITH_UV})
-    if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+    if (TD_WINDOWS)
-        MESSAGE("Windows need set no-sign-compare")
+        # There is no GetHostNameW function on win7.
-        add_compile_options(-Wno-sign-compare)
+        file(READ "libuv/src/win/util.c" LIBUV_WIN_UTIL_CONTENT)
+        string(REPLACE "if (GetHostNameW(buf, UV_MAXHOSTNAMESIZE" "DWORD  nSize = UV_MAXHOSTNAMESIZE;\n  if (GetComputerNameW(buf, &nSize" LIBUV_WIN_UTIL_CONTENT "${LIBUV_WIN_UTIL_CONTENT}")
+        file(WRITE "libuv/src/win/util.c" "${LIBUV_WIN_UTIL_CONTENT}")
     endif ()
     add_subdirectory(libuv EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_UV})

docs/en/01-index.md

@@ -4,7 +4,8 @@ sidebar_label: Documentation Home
 slug: /
 ---
 
-TDengine is a [high-performance](https://tdengine.com/fast), [scalable](https://tdengine.com/scalable) time series database with [SQL support](https://tdengine.com/sql-support). This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design and other topics. It’s written mainly for architects, developers and system administrators.
+
+TDengine is an open source, cloud native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors.  This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design and other topics. It’s written mainly for architects, developers and system administrators.
 
 To get an overview of TDengine, such as a feature list, benchmarks, and competitive advantages, please browse through the [Introduction](./intro) section.
 

docs/en/02-intro/index.md

@@ -3,7 +3,7 @@ title: Introduction
 toc_max_heading_level: 2
 ---
 
-TDengine is a high-performance, scalable time-series database with SQL support. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](/develop/cache), [stream processing](/develop/continuous-query), [data subscription](/develop/subscribe)  and other functionalities to reduce the complexity and cost of development and operation.
+TDengine is an open source, high-performance, cloud native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](/develop/cache), [stream processing](/develop/continuous-query), [data subscription](/develop/subscribe) and other functionalities to reduce the system complexity and cost of development and operation.
 
 This section introduces the major features, competitive advantages, typical use-cases and benchmarks to help you get a high level overview of TDengine.
 
@@ -31,25 +31,21 @@ For more details on features, please read through the entire documentation.
 
 ## Competitive Advantages
 
-Time-series data is structured, not transactional, and is rarely deleted or updated. TDengine makes full use of [these characteristics of time series data](https://tdengine.com/2019/07/09/86.html) to build its own innovative storage engine and computing engine to differentiate itself from other time series databases, with the following advantages.
+By making full use of [characteristics of time series data](https://tdengine.com/tsdb/characteristics-of-time-series-data/), TDengine differentiates itself from other time series databases, with the following advantages.
 
-- **[High Performance](https://tdengine.com/fast)**: With an innovatively designed and purpose-built storage engine, TDengine outperforms other time series databases in data ingestion and querying while significantly reducing storage costs and compute costs.
+- **High-Performance**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression.
 
-- **[Scalable](https://tdengine.com/scalable)**: TDengine provides out-of-box scalability and high-availability through its native distributed design. Nodes can be added through simple configuration to achieve greater data processing power. In addition, this feature is open source.
+- **Simplified Solution**: Through built-in caching, stream processing and data subscription features, TDengine provides a simplified solution for time-series data processing. It reduces system design complexity and operation costs significantly.
 
-- **[SQL Support](https://tdengine.com/sql-support)**: TDengine uses SQL as the query language, thereby reducing learning and migration costs, while adding SQL extensions to better handle time-series. Keeping NoSQL developers in mind, TDengine also supports convenient and flexible, schemaless data ingestion.
+- **Cloud Native**: Through native distributed design, sharding and partitioning, separation of compute and storage, RAFT, support for kubernetes deployment and full observability, TDengine is a cloud native Time-Series Database and can be deployed on public, private or hybrid clouds.
 
-- **All in One**: TDengine has built-in caching, stream processing and data subscription functions. It is no longer necessary to integrate Kafka/Redis/HBase/Spark or other software in some scenarios. It makes the system architecture much simpler, cost-effective and easier to maintain.
+- **Ease of Use**: For administrators, TDengine significantly reduces the effort to deploy and maintain. For developers, it provides a simple interface, simplified solution and seamless integrations for third party tools. For data users, it gives easy data access. 
 
-- **Seamless Integration**: Without a single line of code, TDengine provide seamless, configurable integration with third-party tools such as Telegraf, Grafana, EMQX, Prometheus, StatsD, collectd, etc. More third-party tools are being integrated.
+- **Easy Data Analytics**: Through super tables, storage and compute separation, data partitioning by time interval, pre-computation and other means, TDengine makes it easy to explore, format, and get access to data in a highly efficient way. 
 
-- **Zero Management**: Installation and cluster setup can be done in seconds. Data partitioning and sharding are executed automatically. TDengine’s running status can be monitored via Grafana or other DevOps tools.
+- **Open Source**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 18.8k stars on GitHub. There is an active developer community, and over 139k running instances worldwide.
 
-- **Zero Learning Costs**: With SQL as the query language and support for ubiquitous tools like Python, Java, C/C++, Go, Rust, and Node.js connectors, and a REST API, there are zero learning costs.
+With TDengine, the total cost of ownership of your time-series data platform can be greatly reduced. 1: With its superior performance, the computing and storage resources are reduced significantly；2: With SQL support, it can be seamlessly integrated with many third party tools, and learning costs/migration costs are reduced significantly；3: With its simplified solution and nearly zero management, the operation and maintenance costs are reduced significantly. 
-
-- **Interactive Console**: TDengine provides convenient console access to the database, through a CLI, to run ad hoc queries, maintain the database, or manage the cluster, without any programming.
-
-With TDengine, the total cost of ownership of your time-series data platform can be greatly reduced. 1: With its superior performance, the computing and storage resources are reduced significantly 2: With SQL support, it can be seamlessly integrated with many third party tools, and learning costs/migration costs are reduced significantly 3: With its simple architecture and zero management, the operation and maintenance costs are reduced. 
 
 ## Technical Ecosystem
 This is how TDengine would be situated, in a typical time-series data processing platform:

docs/en/04-concept/index.md

@@ -2,7 +2,7 @@
 title: Concepts
 ---
 
-In order to explain the basic concepts and provide some sample code, the TDengine documentation smart meters as a typical time series use case. We assume the following: 1. Each smart meter collects three metrics i.e. current, voltage, and phase 2. There are multiple smart meters, and 3. Each meter has static attributes like location and group ID. Based on this, collected data will look similar to the following table:
+In order to explain the basic concepts and provide some sample code, the TDengine documentation smart meters as a typical time series use case. We assume the following: 1. Each smart meter collects three metrics i.e. current, voltage, and phase; 2. There are multiple smart meters; 3. Each meter has static attributes like location and group ID. Based on this, collected data will look similar to the following table:
 
 <div className="center-table">
 <table>

docs/en/07-develop/02-model/index.mdx

@@ -2,7 +2,9 @@
 title: Data Model
 ---
 
-The data model employed by TDengine is similar to that of a relational database. You have to create databases and tables. You must design the data model based on your own business and application requirements. You should design the STable (an abbreviation for super table) schema to fit your data. This chapter will explain the big picture without getting into syntactical details.
+The data model employed by TDengine is similar to that of a relational database. You have to create databases and tables. You must design the data model based on your own business and application requirements. You should design the [STable](/concept/#super-table-stable) (an abbreviation for super table) schema to fit your data. This chapter will explain the big picture without getting into syntactical details. 
+
+Note: before you read this chapter, please make sure you have already read through [Key Concepts](/concept/), since TDengine introduces new concepts like "one table for one [data collection point](/concept/#data-collection-point)" and "[super table](/concept/#super-table-stable)".
 
 
 
@@ -11,7 +13,7 @@ The data model employed by TDengine is similar to that of a relational database.
 The characteristics of time-series data from different data collection points may be different. Characteristics include collection frequency, retention policy and others which determine how you create and configure the database. For e.g. days to keep, number of replicas, data block size, whether data updates are allowed and other configurable parameters would be determined by the characteristics of your data and your business requirements. For TDengine to operate with the best performance, we strongly recommend that you create and configure different databases for data with different characteristics. This allows you, for example, to set up different storage and retention policies. When creating a database, there are a lot of parameters that can be configured such as, the days to keep data, the number of replicas, the size of the cache, time precision, the minimum and maximum number of rows in each data block, whether compression is enabled, the time range of the data in single data file and so on. An example is shown as follows:
 
 ```sql
-CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 VGROUPS 100 WAL 1;
+CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 WAL_LEVEL 1;
 ```
 
 In the above SQL statement:

docs/en/07-develop/04-query-data/index.mdx

@@ -54,21 +54,21 @@ In most use cases, there are always multiple kinds of data collection points. A
 In TDengine CLI `taos`, use the SQL below to get the average voltage of all the meters in California grouped by location.
 
 ```
-taos> SELECT AVG(voltage) FROM meters GROUP BY location;
+taos> SELECT AVG(voltage), location FROM meters GROUP BY location;
        avg(voltage)        |                             location                             |
-=============================================================
+===============================================================================================
-             222.000000000 | California.LosAngeles                |
              219.200000000 | California.SanFrancisco                                          |
-Query OK, 2 row(s) in set (0.002136s)
+             221.666666667 | California.LosAngeles                                            |
+Query OK, 2 rows in database (0.005995s)
 ```
 
 ### Example 2
 
-In TDengine CLI `taos`, use the SQL below to get the number of rows and the maximum current in the past 24 hours from meters whose groupId is 2.
+In TDengine CLI `taos`, use the SQL below to get the number of rows and the maximum current from meters whose groupId is 2.
 
 ```
-taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - 24h;
+taos> SELECT count(*), max(current) FROM meters where groupId = 2;
-     cunt(*)  |    max(current)  |
+     count(*)  |    max(current)  |
 ==================================
             5 |             13.4 |
 Query OK, 1 row(s) in set (0.002136s)
@@ -81,40 +81,41 @@ In [Select](../../taos-sql././select), all query operations are marked as to whe
 In IoT use cases, down sampling is widely used to aggregate data by time range. The `INTERVAL` keyword in TDengine can be used to simplify the query by time window. For example, the SQL statement below can be used to get the sum of current every 10 seconds from meters table d1001.
 
 ```
-taos> SELECT sum(current) FROM d1001 INTERVAL(10s);
+taos> SELECT _wstart, sum(current) FROM d1001 INTERVAL(10s);
-           ts            |       sum(current)        |
+         _wstart         |       sum(current)        |
 ======================================================
  2018-10-03 14:38:00.000 |              10.300000191 |
  2018-10-03 14:38:10.000 |              24.900000572 |
-Query OK, 2 row(s) in set (0.000883s)
+Query OK, 2 rows in database (0.003139s)
 ```
 
 Down sampling can also be used for STable. For example, the below SQL statement can be used to get the sum of current from all meters in California.
 
 ```
-taos> SELECT SUM(current) FROM meters where location like "California%" INTERVAL(1s);
+taos> SELECT _wstart, SUM(current) FROM meters where location like "California%" INTERVAL(1s);
-           ts            |       sum(current)        |
+         _wstart         |       sum(current)        |
 ======================================================
  2018-10-03 14:38:04.000 |              10.199999809 |
- 2018-10-03 14:38:05.000 |              32.900000572 |
+ 2018-10-03 14:38:05.000 |              23.699999809 |
  2018-10-03 14:38:06.000 |              11.500000000 |
  2018-10-03 14:38:15.000 |              12.600000381 |
- 2018-10-03 14:38:16.000 |              36.000000000 |
+ 2018-10-03 14:38:16.000 |              34.400000572 |
-Query OK, 5 row(s) in set (0.001538s)
+Query OK, 5 rows in database (0.007413s)
 ```
 
 Down sampling also supports time offset. For example, the below SQL statement can be used to get the sum of current from all meters but each time window must start at the boundary of 500 milliseconds.
 
 ```
-taos> SELECT SUM(current) FROM meters INTERVAL(1s, 500a);
+taos> SELECT _wstart, SUM(current) FROM meters INTERVAL(1s, 500a);
-           ts            |       sum(current)        |
+         _wstart         |       sum(current)        |
 ======================================================
- 2018-10-03 14:38:04.500 |              11.189999809 |
+ 2018-10-03 14:38:03.500 |              10.199999809 |
- 2018-10-03 14:38:05.500 |              31.900000572 |
+ 2018-10-03 14:38:04.500 |              10.300000191 |
- 2018-10-03 14:38:06.500 |              11.600000000 |
+ 2018-10-03 14:38:05.500 |              13.399999619 |
- 2018-10-03 14:38:15.500 |              12.300000381 |
+ 2018-10-03 14:38:06.500 |              11.500000000 |
- 2018-10-03 14:38:16.500 |              35.000000000 |
+ 2018-10-03 14:38:14.500 |              12.600000381 |
-Query OK, 5 row(s) in set (0.001521s)
+ 2018-10-03 14:38:16.500 |              34.400000572 |
+Query OK, 6 rows in database (0.005515s)
 ```
 
 In many use cases, it's hard to align the timestamp of the data collected by each collection point. However, a lot of algorithms like FFT require the data to be aligned with same time interval and application programs have to handle this by themselves. In TDengine, it's easy to achieve the alignment using down sampling.

docs/examples/node/nativeexample/subscribe_demo.js

@@ -28,8 +28,7 @@ function runConsumer() {
         console.log(msg.topicPartition);
         console.log(msg.block);
         console.log(msg.fields)
-        // fixme(@xiaolei): commented temp, should be fixed.
+        consumer.commit(msg);
-        //consumer.commit(msg);
         console.log(`=======consumer ${i} done`)
     }
 

docs/examples/node/package.json

@@ -4,7 +4,7 @@
   "main": "index.js",
   "license": "MIT",
   "dependencies": {
-    "@tdengine/client": "^3.0.0",
+    "@tdengine/client": "^3.0.1",
     "@tdengine/rest": "^3.0.0"
   }
 }

docs/zh/07-develop/02-model/index.mdx

@@ -11,10 +11,10 @@ TDengine 采用类关系型数据模型，需要建库、建表。因此对于
 不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为了在各种场景下 TDengine 都能最大效率的工作，TDengine 建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除 SQL 标准的选项外，还可以指定保留时长、副本数、缓存大小、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如：
 
 ```sql
-CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 VGROUPS 100 WAL 1;
+CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 WAL_LEVEL 1;
 ```
 
-上述语句将创建一个名为 power 的库，这个库的数据将保留 365 天（超过 365 天将被自动删除），每 10 天一个数据文件，每个 VNODE 的写入内存池的大小为 16 MB，数据库的 VGROUPS 数量，对该数据库入会写 WAL 但不执行 FSYNC。详细的语法及参数请见 [数据库管理](/taos-sql/database) 章节。
+上述语句将创建一个名为 power 的库，这个库的数据将保留 365 天（超过 365 天将被自动删除），每 10 天一个数据文件，每个 VNODE 的写入内存池的大小为 16 MB，对该数据库入会写 WAL 但不执行 FSYNC。详细的语法及参数请见 [数据库管理](/taos-sql/database) 章节。
 
 创建库之后，需要使用 SQL 命令 `USE` 将当前库切换过来，例如：
 

docs/zh/07-develop/04-query-data/index.mdx

@@ -54,20 +54,20 @@ Query OK, 2 row(s) in set (0.001100s)
 在 TAOS Shell，查找加利福尼亚州所有智能电表采集的电压平均值，并按照 location 分组。
 
 ```
-taos> SELECT AVG(voltage) FROM meters GROUP BY location;
+taos> SELECT AVG(voltage), location FROM meters GROUP BY location;
        avg(voltage)        |                             location                             |
-=============================================================
+===============================================================================================
-             222.000000000 | California.LosAngeles                |
              219.200000000 | California.SanFrancisco                                          |
-Query OK, 2 row(s) in set (0.002136s)
+             221.666666667 | California.LosAngeles                                            |
+Query OK, 2 rows in database (0.005995s)
 ```
 
 ### 示例二
 
-在 TAOS shell, 查找 groupId 为 2 的所有智能电表过去 24 小时的记录条数，电流的最大值。
+在 TAOS shell, 查找 groupId 为 2 的所有智能电表的记录条数，电流的最大值。
 
 ```
-taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - 24h;
+taos> SELECT count(*), max(current) FROM meters where groupId = 2;
      cunt(*)  |    max(current)  |
 ==================================
             5 |             13.4 |
@@ -81,40 +81,41 @@ Query OK, 1 row(s) in set (0.002136s)
 物联网场景里，经常需要通过降采样（down sampling）将采集的数据按时间段进行聚合。TDengine 提供了一个简便的关键词 interval 让按照时间窗口的查询操作变得极为简单。比如，将智能电表 d1001 采集的电流值每 10 秒钟求和
 
 ```
-taos> SELECT sum(current) FROM d1001 INTERVAL(10s);
+taos> SELECT _wstart, sum(current) FROM d1001 INTERVAL(10s);
-           ts            |       sum(current)        |
+         _wstart         |       sum(current)        |
 ======================================================
  2018-10-03 14:38:00.000 |              10.300000191 |
  2018-10-03 14:38:10.000 |              24.900000572 |
-Query OK, 2 row(s) in set (0.000883s)
+Query OK, 2 rows in database (0.003139s)
 ```
 
 降采样操作也适用于超级表，比如：将加利福尼亚州所有智能电表采集的电流值每秒钟求和
 
 ```
-taos> SELECT SUM(current) FROM meters where location like "California%" INTERVAL(1s);
+taos> SELECT _wstart, SUM(current) FROM meters where location like "California%" INTERVAL(1s);
-           ts            |       sum(current)        |
+         _wstart         |       sum(current)        |
 ======================================================
  2018-10-03 14:38:04.000 |              10.199999809 |
- 2018-10-03 14:38:05.000 |              32.900000572 |
+ 2018-10-03 14:38:05.000 |              23.699999809 |
  2018-10-03 14:38:06.000 |              11.500000000 |
  2018-10-03 14:38:15.000 |              12.600000381 |
- 2018-10-03 14:38:16.000 |              36.000000000 |
+ 2018-10-03 14:38:16.000 |              34.400000572 |
-Query OK, 5 row(s) in set (0.001538s)
+Query OK, 5 rows in database (0.007413s)
 ```
 
 降采样操作也支持时间偏移，比如：将所有智能电表采集的电流值每秒钟求和，但要求每个时间窗口从 500 毫秒开始
 
 ```
-taos> SELECT SUM(current) FROM meters INTERVAL(1s, 500a);
+taos> SELECT _wstart, SUM(current) FROM meters INTERVAL(1s, 500a);
-           ts            |       sum(current)        |
+         _wstart         |       sum(current)        |
 ======================================================
- 2018-10-03 14:38:04.500 |              11.189999809 |
+ 2018-10-03 14:38:03.500 |              10.199999809 |
- 2018-10-03 14:38:05.500 |              31.900000572 |
+ 2018-10-03 14:38:04.500 |              10.300000191 |
- 2018-10-03 14:38:06.500 |              11.600000000 |
+ 2018-10-03 14:38:05.500 |              13.399999619 |
- 2018-10-03 14:38:15.500 |              12.300000381 |
+ 2018-10-03 14:38:06.500 |              11.500000000 |
- 2018-10-03 14:38:16.500 |              35.000000000 |
+ 2018-10-03 14:38:14.500 |              12.600000381 |
-Query OK, 5 row(s) in set (0.001521s)
+ 2018-10-03 14:38:16.500 |              34.400000572 |
+Query OK, 6 rows in database (0.005515s)
 ```
 
 物联网场景里，每个数据采集点采集数据的时间是难同步的，但很多分析算法(比如 FFT)需要把采集的数据严格按照时间等间隔的对齐，在很多系统里，需要应用自己写程序来处理，但使用 TDengine 的降采样操作就轻松解决。

docs/zh/12-taos-sql/03-table.md

@@ -110,7 +110,7 @@ alter_table_option: {
 对普通表可以进行如下修改操作
 1. ADD COLUMN：添加列。
 2. DROP COLUMN：删除列。
-3. ODIFY COLUMN：修改列定义，如果数据列的类型是可变长类型，那么可以使用此指令修改其宽度，只能改大，不能改小。
+3. MODIFY COLUMN：修改列定义，如果数据列的类型是可变长类型，那么可以使用此指令修改其宽度，只能改大，不能改小。
 4. RENAME COLUMN：修改列名称。
 
 ### 增加列

docs/zh/14-reference/11-docker/index.md

@@ -25,10 +25,11 @@ curl -u root:taosdata -d "show databases" localhost:6041/rest/sql
 $ docker exec -it tdengine taos
 
 taos> show databases;
-              name              |      created_time       |   ntables   |   vgroups   | replica | quorum |  days  |           keep           |  cache(MB)  |   blocks    |   minrows   |   maxrows   | wallevel |    fsync    | comp | cachelast | precision | update |   status   |
+              name              |
-====================================================================================================================================================================================================================================================================================
+=================================
- log                            | 2022-01-17 13:57:22.270 |          10 |           1 |       1 |      1 |     10 | 30                       |           1 |           3 |         100 |        4096 |        1 |        3000 |    2 |         0 | us        |      0 | ready      |
+ information_schema             |
-Query OK, 1 row(s) in set (0.002843s)
+ performance_schema             |
+Query OK, 2 rows in database (0.033802s)
 ```
 
 因为运行在容器中的 TDengine 服务端使用容器的 hostname 建立连接，使用 taos shell 或者各种连接器（例如 JDBC-JNI）从容器外访问容器内的 TDengine 比较复杂，所以上述方式是访问容器中 TDengine 服务的最简单的方法，适用于一些简单场景。如果在一些复杂场景下想要从容器化使用 taos shell 或者各种连接器访问容器中的 TDengine 服务，请参考下一节。
@@ -45,10 +46,11 @@ docker run -d --name tdengine --network host tdengine/tdengine
 $ taos
 
 taos> show dnodes;
-   id   |           end_point            | vnodes | cores  |   status   | role  |       create_time       |      offline reason      |
+     id      |            endpoint            | vnodes | support_vnodes |   status   |       create_time       |              note              |
-======================================================================================================================================
+=================================================================================================================================================
-      1 | myhost:6030           |      1 |      8 | ready      | any   | 2022-01-17 22:10:32.619 |                          |
+           1 | vm98:6030                      |      0 |             32 | ready      | 2022-08-19 14:50:05.337 |                                |
-Query OK, 1 row(s) in set (0.003233s)
+Query OK, 1 rows in database (0.010654s)
+
 ```
 
 ## 以指定的 hostname 和 port 启动 TDengine
@@ -59,12 +61,13 @@ Query OK, 1 row(s) in set (0.003233s)
 docker run -d \
    --name tdengine \
    -e TAOS_FQDN=tdengine \
-   -p 6030-6049:6030-6049 \
+   -p 6030:6030 \
-   -p 6030-6049:6030-6049/udp \
+   -p 6041-6049:6041-6049 \
+   -p 6041-6049:6041-6049/udp \
    tdengine/tdengine
 ```
 
-上面的命令在容器中启动一个 TDengine 服务，其所监听的 hostname 为 tdengine ，并将容器的 6030 到 6049 端口段映射到主机的 6030 到 6049 端口段 （tcp 和 udp 都需要映射)。如果主机上该端口段已经被占用，可以修改上述命令指定一个主机上空闲的端口段。如果 `rpcForceTcp` 被设置为 `1` ，可以只映射 tcp 协议。
+上面的命令在容器中启动一个 TDengine 服务，其所监听的 hostname 为 tdengine ，并将容器的 6030 端口映射到主机的 6030 端口（TCP，只能映射主机 6030 端口），6041-6049 端口段映射到主机 6041-6049 端口段（tcp 和 udp 都需要映射，如果主机上该端口段已经被占用，可以修改上述命令指定一个主机上空闲的端口段）。
 
 接下来，要确保 "tdengine" 这个 hostname 在 `/etc/hosts` 中可解析。
 
@@ -103,9 +106,9 @@ taos -h tdengine -P 6030
 3. 在同一网络上的另一容器中启动 TDengine 客户端
 
    ```shell
-   docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine tdengine/tdengine taos
+   docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine --entrypoint=taos tdengine/tdengine
    # or
-   #docker run --rm -it --network td-net -e tdengine/tdengine taos -h tdengine
+   #docker run --rm -it --network td-net --entrypoint=taos tdengine/tdengine -h tdengine
    ```
 
 ## 在容器中启动客户端应用
@@ -115,7 +118,7 @@ taos -h tdengine -P 6030
 ```docker
 FROM ubuntu:20.04
 RUN apt-get update && apt-get install -y wget
-ENV TDENGINE_VERSION=2.4.0.0
+ENV TDENGINE_VERSION=3.0.0.0
 RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
@@ -129,6 +132,14 @@ RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_
 
 以下是一个 go 应用程序的示例：
 
+* 创建 go mod 项目：
+
+```bash
+go mod init app
+```
+
+* 创建 main.go：
+
 ```go
 /*
  * In this test program, we'll create a database and insert 4 records then select out.
@@ -212,12 +223,18 @@ func checkErr(err error, prompt string) {
 }
 ```
 
-如下是完整版本的 dockerfile
+* 更新 go mod
 
-```docker
+```bash
-FROM golang:1.17.6-buster as builder
+go mod tidy
-ENV TDENGINE_VERSION=2.4.0.0
+```
-RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
+
+如下是完整版本的 dockerfile：
+
+```dockerfile
+FROM golang:1.19.0-buster as builder
+ENV TDENGINE_VERSION=3.0.0.0
+RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
    && ./install_client.sh \
@@ -232,8 +249,8 @@ RUN go build
 
 FROM ubuntu:20.04
 RUN apt-get update && apt-get install -y wget
-ENV TDENGINE_VERSION=2.4.0.0
+ENV TDENGINE_VERSION=3.0.0.0
-RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
+RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
    && ./install_client.sh \
@@ -248,46 +265,32 @@ CMD ["app"]
 目前我们已经有了 `main.go`, `go.mod`, `go.sum`, `app.dockerfile`， 现在可以构建出这个应用程序并在 `td-net` 网络上启动它
 
 ```shell
-$ docker build -t app -f app.dockerfile
+$ docker build -t app -f app.dockerfile .
-$ docker run --rm --network td-net app -h tdengine -p 6030
+$ docker run --rm --network td-net app app -h tdengine -p 6030
 ============= args parse result: =============
 hostName:             tdengine
 serverPort:           6030
 usr:                  root
 password:             taosdata
 ================================================
-2022-01-17 15:56:55.48 +0000 UTC 0
+2022-08-19 07:43:51.68 +0000 UTC 0
-2022-01-17 15:56:56.48 +0000 UTC 1
+2022-08-19 07:43:52.68 +0000 UTC 1
-2022-01-17 15:56:57.48 +0000 UTC 2
+2022-08-19 07:43:53.68 +0000 UTC 2
-2022-01-17 15:56:58.48 +0000 UTC 3
+2022-08-19 07:43:54.68 +0000 UTC 3
-2022-01-17 15:58:01.842 +0000 UTC 0
-2022-01-17 15:58:02.842 +0000 UTC 1
-2022-01-17 15:58:03.842 +0000 UTC 2
-2022-01-17 15:58:04.842 +0000 UTC 3
-2022-01-18 01:43:48.029 +0000 UTC 0
-2022-01-18 01:43:49.029 +0000 UTC 1
-2022-01-18 01:43:50.029 +0000 UTC 2
-2022-01-18 01:43:51.029 +0000 UTC 3
 ```
 
 ## 用 docker-compose 启动 TDengine 集群
 
-1. 如下 docker-compose 文件启动一个 2 副本、2 管理节点、2 数据节点以及 1 个 arbitrator 的 TDengine 集群。
+1. 如下 docker-compose 文件启动一个 三节点 TDengine 集群。
 
-   ```docker
+```yml
-   version: "3"
+version: "3"
-   services:
+services:
-     arbitrator:
-       image: tdengine/tdengine:$VERSION
-       command: tarbitrator
   td-1:
     image: tdengine/tdengine:$VERSION
     environment:
       TAOS_FQDN: "td-1"
       TAOS_FIRST_EP: "td-1"
-         TAOS_NUM_OF_MNODES: "2"
-         TAOS_REPLICA: "2"
-         TAOS_ARBITRATOR: arbitrator:6042
     volumes:
       - taosdata-td1:/var/lib/taos/
       - taoslog-td1:/var/log/taos/
@@ -296,65 +299,78 @@ password:             taosdata
     environment:
       TAOS_FQDN: "td-2"
       TAOS_FIRST_EP: "td-1"
-         TAOS_NUM_OF_MNODES: "2"
-         TAOS_REPLICA: "2"
-         TAOS_ARBITRATOR: arbitrator:6042
     volumes:
       - taosdata-td2:/var/lib/taos/
       - taoslog-td2:/var/log/taos/
+  td-3:
+    image: tdengine/tdengine:$VERSION
+    environment:
+      TAOS_FQDN: "td-3"
+      TAOS_FIRST_EP: "td-1"
     volumes:
+      - taosdata-td3:/var/lib/taos/
+      - taoslog-td3:/var/log/taos/    
+volumes:
   taosdata-td1:
   taoslog-td1:
   taosdata-td2:
   taoslog-td2:
-   ```
+  taosdata-td3:
+  taoslog-td3:
+```
 
 :::note
 
-- `VERSION` 环境变量被用来设置 tdengine image tag
+* `VERSION` 环境变量被用来设置 tdengine image tag
-- 在新创建的实例上必须设置 `TAOS_FIRST_EP` 以使其能够加入 TDengine 集群；如果有高可用需求，则需要同时使用 `TAOS_SECOND_EP`
+* 在新创建的实例上必须设置 `TAOS_FIRST_EP` 以使其能够加入 TDengine 集群；如果有高可用需求，则需要同时使用 `TAOS_SECOND_EP`
-- `TAOS_REPLICA` 用来设置缺省的数据库副本数量，其取值范围为[1,3]
+:::
-  在双副本环境下，推荐使用 arbitrator, 用 TAOS_ARBITRATOR 来设置
-  :::
 
 2. 启动集群
 
-   ```shell
+```shell
-   $ VERSION=2.4.0.0 docker-compose up -d
+$ VERSION=3.0.0.0 docker-compose up -d
-   Creating network "test_default" with the default driver
+Creating network "test-docker_default" with the default driver
-   Creating volume "test_taosdata-td1" with default driver
+Creating volume "test-docker_taosdata-td1" with default driver
-   Creating volume "test_taoslog-td1" with default driver
+Creating volume "test-docker_taoslog-td1" with default driver
-   Creating volume "test_taosdata-td2" with default driver
+Creating volume "test-docker_taosdata-td2" with default driver
-   Creating volume "test_taoslog-td2" with default driver
+Creating volume "test-docker_taoslog-td2" with default driver
-   Creating test_td-1_1       ... done
+Creating volume "test-docker_taosdata-td3" with default driver
-   Creating test_arbitrator_1 ... done
+Creating volume "test-docker_taoslog-td3" with default driver
-   Creating test_td-2_1       ... done
+
-   ```
+Creating test-docker_td-3_1 ... done
+Creating test-docker_td-1_1 ... done
+Creating test-docker_td-2_1 ... done
+```
 
 3. 查看节点状态
 
-   ```shell
+```shell
-   $ docker-compose ps
+   docker-compose ps
        Name                     Command               State   Ports
-   ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
-   test_arbitrator_1   /usr/bin/entrypoint.sh tar ...   Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
+-------------------------------------------------------------------
-   test_td-1_1         /usr/bin/entrypoint.sh taosd     Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
+test-docker_td-1_1   /tini -- /usr/bin/entrypoi ...   Up
-   test_td-2_1         /usr/bin/entrypoint.sh taosd     Up      6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp
+test-docker_td-2_1   /tini -- /usr/bin/entrypoi ...   Up
-   ```
+test-docker_td-3_1   /tini -- /usr/bin/entrypoi ...   Up
+```
 
 4. 用 taos shell 查看 dnodes
 
-   ```shell
+```shell
-   $ docker-compose exec td-1 taos -s "show dnodes"
 
-   taos> show dnodes
+$ docker-compose exec td-1 taos -s "show dnodes"
-      id   |           end_point            | vnodes | cores  |   status   | role  |       create_time       |      offline reason      |
+
-   ======================================================================================================================================
+taos> show dnodes
-         1 | td-1:6030                      |      1 |      8 | ready      | any   | 2022-01-18 02:47:42.871 |                          |
+
-         2 | td-2:6030                      |      0 |      8 | ready      | any   | 2022-01-18 02:47:43.518 |                          |
+     id      |            endpoint            | vnodes | support_vnodes |   status   |       create_time       |              note              |
-         0 | arbitrator:6042                |      0 |      0 | ready      | arb   | 2022-01-18 02:47:43.633 | -                        |
+=================================================================================================================================================
-   Query OK, 3 row(s) in set (0.000811s)
+
-   ```
+           1 | td-1:6030                      |      0 |             32 | ready      | 2022-08-19 07:57:29.971 |                                |
+           2 | td-2:6030                      |      0 |             32 | ready      | 2022-08-19 07:57:31.415 |                                |
+           3 | td-3:6030                      |      0 |             32 | ready      | 2022-08-19 07:57:31.417 |                                |
+Query OK, 3 rows in database (0.021262s)
+
+```
 
 ## taosAdapter
 
@@ -362,29 +378,23 @@ password:             taosdata
 
 2. 同时为了部署灵活起见，可以在独立的容器中启动 taosAdapter
 
-   ```docker
+```docker
-   services:
+services:
   # ...
   adapter:
     image: tdengine/tdengine:$VERSION
     command: taosadapter
-   ```
+```
 
-   如果要部署多个 taosAdapter 来提高吞吐量并提供高可用性，推荐配置方式为使用 nginx 等反向代理来提供统一的访问入口。具体配置方法请参考 nginx 的官方文档。如下是示例：
+如果要部署多个 taosAdapter 来提高吞吐量并提供高可用性，推荐配置方式为使用 nginx 等反向代理来提供统一的访问入口。具体配置方法请参考 nginx 的官方文档。如下是示例：
 
-   ```docker
+```yml
-   version: "3"
+version: "3"
 
-   networks:
+networks:
   inter:
-     api:
 
-   services:
+services:
-     arbitrator:
-       image: tdengine/tdengine:$VERSION
-       command: tarbitrator
-       networks:
-         - inter
   td-1:
     image: tdengine/tdengine:$VERSION
     networks:
@@ -392,9 +402,6 @@ password:             taosdata
     environment:
       TAOS_FQDN: "td-1"
       TAOS_FIRST_EP: "td-1"
-         TAOS_NUM_OF_MNODES: "2"
-         TAOS_REPLICA: "2"
-         TAOS_ARBITRATOR: arbitrator:6042
     volumes:
       - taosdata-td1:/var/lib/taos/
       - taoslog-td1:/var/log/taos/
@@ -405,15 +412,12 @@ password:             taosdata
     environment:
       TAOS_FQDN: "td-2"
       TAOS_FIRST_EP: "td-1"
-         TAOS_NUM_OF_MNODES: "2"
-         TAOS_REPLICA: "2"
-         TAOS_ARBITRATOR: arbitrator:6042
     volumes:
       - taosdata-td2:/var/lib/taos/
       - taoslog-td2:/var/log/taos/
   adapter:
     image: tdengine/tdengine:$VERSION
-       command: taosadapter
+    entrypoint: "taosadapter"
     networks:
       - inter
     environment:
@@ -427,7 +431,6 @@ password:             taosdata
       - adapter
     networks:
       - inter
-         - api
     ports:
       - 6041:6041
       - 6044:6044/udp
@@ -443,12 +446,12 @@ password:             taosdata
         >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf;
         nginx -g 'daemon off;'",
       ]
-   volumes:
+volumes:
   taosdata-td1:
   taoslog-td1:
   taosdata-td2:
   taoslog-td2:
-   ```
+```
 
 ## 使用 docker swarm 部署
 
@@ -457,14 +460,12 @@ password:             taosdata
 docker-compose 文件可以参考上节。下面是使用 docker swarm 启动 TDengine 的命令：
 
 ```shell
-$ VERSION=2.4.0 docker stack deploy -c docker-compose.yml taos
+$ VERSION=3.0.0.0 docker stack deploy -c docker-compose.yml taos
 Creating network taos_inter
-Creating network taos_api
+Creating service taos_nginx
-Creating service taos_arbitrator
 Creating service taos_td-1
 Creating service taos_td-2
 Creating service taos_adapter
-Creating service taos_nginx
 ```
 
 查看和管理
@@ -472,24 +473,22 @@ Creating service taos_nginx
 ```shell
 $ docker stack ps taos
 ID             NAME             IMAGE                       NODE      DESIRED STATE   CURRENT STATE                ERROR     PORTS
-79ni8temw59n        taos_nginx.1        nginx:latest              TM1701     Running             Running about a minute ago
+7m3sbf532bqp   taos_adapter.1   tdengine/tdengine:3.0.0.0   vm98      Running         Running about a minute ago             
-3e94u72msiyg        taos_adapter.1      tdengine/tdengine:2.4.0   TM1702     Running             Running 56 seconds ago
+pj403n6ofmmh   taos_adapter.2   tdengine/tdengine:3.0.0.0   vm98      Running         Running about a minute ago             
-100amjkwzsc6        taos_td-2.1         tdengine/tdengine:2.4.0   TM1703     Running             Running about a minute ago
+rxqfwsyk5q1h   taos_adapter.3   tdengine/tdengine:3.0.0.0   vm98      Running         Running about a minute ago             
-pkjehr2vvaaa        taos_td-1.1         tdengine/tdengine:2.4.0   TM1704     Running             Running 2 minutes ago
+qj40lpxr40oc   taos_adapter.4   tdengine/tdengine:3.0.0.0   vm98      Running         Running about a minute ago             
-tpzvgpsr1qkt        taos_arbitrator.1   tdengine/tdengine:2.4.0   TM1705     Running             Running 2 minutes ago
+oe3455ulxpze   taos_nginx.1     nginx:latest                vm98      Running         Running about a minute ago             
-rvss3g5yg6fa        taos_adapter.2      tdengine/tdengine:2.4.0   TM1706     Running             Running 56 seconds ago
+o0tsg70nrrc6   taos_td-1.1      tdengine/tdengine:3.0.0.0   vm98      Running         Running about a minute ago             
-i2augxamfllf        taos_adapter.3      tdengine/tdengine:2.4.0   TM1707     Running             Running 56 seconds ago
+q5m1oxs589cp   taos_td-2.1      tdengine/tdengine:3.0.0.0   vm98      Running         Running about a minute ago
-lmjyhzccpvpg        taos_adapter.4      tdengine/tdengine:2.4.0   TM1708     Running             Running 56 seconds ago
 $ docker service ls
 ID             NAME           MODE         REPLICAS   IMAGE                       PORTS
-561t4lu6nfw6        taos_adapter        replicated          4/4                 tdengine/tdengine:2.4.0
+ozuklorgl8bs   taos_adapter   replicated   4/4        tdengine/tdengine:3.0.0.0   
-3hk5ct3q90sm        taos_arbitrator     replicated          1/1                 tdengine/tdengine:2.4.0
+crmhdjw6vxw0   taos_nginx     replicated   1/1        nginx:latest                *:6041->6041/tcp, *:6044->6044/udp
-d8qr52envqzu        taos_nginx          replicated          1/1                 nginx:latest              *:6041->6041/tcp, *:6044->6044/udp
+o86ngy7csv5n   taos_td-1      replicated   1/1        tdengine/tdengine:3.0.0.0   
-2isssfvjk747        taos_td-1           replicated          1/1                 tdengine/tdengine:2.4.0
+rma040ny4tb0   taos_td-2      replicated   1/1        tdengine/tdengine:3.0.0.0
-9pzw7u02ichv        taos_td-2           replicated          1/1                 tdengine/tdengine:2.4.0
 ```
 
-从上面的输出可以看到有两个 dnode， 和两个 taosAdapter，以及一个 nginx 反向代理服务。
+从上面的输出可以看到有两个 dnode， 和四个 taosAdapter，以及一个 nginx 反向代理服务。
 
 接下来，我们可以减少 taosAdapter 服务的数量
 
@@ -502,5 +501,5 @@ verify: Service converged
 
 $ docker service ls -f name=taos_adapter
 ID             NAME           MODE         REPLICAS   IMAGE                       PORTS
-561t4lu6nfw6        taos_adapter        replicated          1/1                 tdengine/tdengine:2.4.0
+ozuklorgl8bs   taos_adapter   replicated   1/1        tdengine/tdengine:3.0.0.0  
 ```

docs/zh/17-operation/03-tolerance.md

@@ -26,5 +26,3 @@ TDengine 集群中的时序数据的副本数是与数据库关联的，一个
 TDengine 集群的节点数必须大于等于副本数，否则创建表时将报错。
 
 当 TDengine 集群中的节点部署在不同的物理机上，并设置多个副本数时，就实现了系统的高可靠性，无需再使用其他软件或工具。TDengine 企业版还可以将副本部署在不同机房，从而实现异地容灾。
-
-另外一种灾备方式是通过 `taosX` 将一个 TDengine 集群的数据同步复制到物理上位于不同数据中心的另一个 TDengine 集群。其详细使用方法请参考 [taosX 参考手册](../../reference/taosX)

examples/c/stream_demo.c

@@ -98,10 +98,9 @@ int32_t create_stream() {
   /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
   /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
-  pRes =
+  pRes = taos_query(pConn,
-      taos_query(pConn,
+                    "create stream stream1 trigger max_delay 10s watermark 10s into outstb as select _wstart start, "
-                 "create stream stream1 trigger max_delay 10s into outstb as select _wstart, sum(k) from st1 partition "
+                    "count(k) from st1 partition by tbname interval(20s) ");
-                 "by tbname session(ts, 10s) ");
   if (taos_errno(pRes) != 0) {
     printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
     return -1;

include/common/tcommon.h

@@ -60,6 +60,7 @@ enum {
   STREAM_INPUT__DATA_RETRIEVE,
   STREAM_INPUT__GET_RES,
   STREAM_INPUT__CHECKPOINT,
+  STREAM_INPUT__DESTROY,
 };
 
 typedef enum EStreamType {

include/common/tdataformat.h

@@ -38,22 +38,18 @@ typedef struct STagVal       STagVal;
 typedef struct STag          STag;
 
 // bitmap
-#define N1(n)        ((1 << (n)) - 1)
+const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0},
-#define BIT1_SIZE(n) (((n)-1) / 8 + 1)
+                                       {0b00000000, 0b00000100, 0b00001000, 2},
-#define BIT2_SIZE(n) (((n)-1) / 4 + 1)
+                                       {0b00000000, 0b00010000, 0b00100000, 4},
-#define SET_BIT1(p, i, v)                            \
+                                       {0b00000000, 0b01000000, 0b10000000, 6}};
-  do {                                               \
-    (p)[(i) / 8] &= N1((i) % 8);                     \
-    (p)[(i) / 8] |= (((uint8_t)(v)) << (((i) % 8))); \
-  } while (0)
 
-#define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1))
+#define N1(n)             ((((uint8_t)1) << (n)) - 1)
-#define SET_BIT2(p, i, v)                                \
+#define BIT1_SIZE(n)      ((((n)-1) >> 3) + 1)
-  do {                                                   \
+#define BIT2_SIZE(n)      ((((n)-1) >> 2) + 1)
-    p[(i) / 4] &= N1((i) % 4 * 2);                       \
+#define SET_BIT1(p, i, v) ((p)[(i) >> 3] = (p)[(i) >> 3] & N1((i)&7) | (((uint8_t)(v)) << ((i)&7)))
-    (p)[(i) / 4] |= (((uint8_t)(v)) << (((i) % 4) * 2)); \
+#define GET_BIT1(p, i)    (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1))
-  } while (0)
+#define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)])
-#define GET_BIT2(p, i) (((p)[(i) / 4] >> (((i) % 4) * 2)) & ((uint8_t)3))
+#define GET_BIT2(p, i)    (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3))
 
 // STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
@@ -171,7 +167,7 @@ struct SColVal {
 
 #pragma pack(push, 1)
 struct STagVal {
-//  char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta
+  //  char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta
   union {
     int16_t cid;
     char   *pKey;

include/common/tmsg.h

@@ -2664,29 +2664,8 @@ typedef struct {
 } SVgEpSet;
 
 typedef struct {
-  int64_t suid;
+  int32_t padding;
-  int8_t  level;
+} SRSmaExecMsg;
-} SRSmaFetchMsg;
-
-static FORCE_INLINE int32_t tEncodeSRSmaFetchMsg(SEncoder* pCoder, const SRSmaFetchMsg* pReq) {
-  if (tStartEncode(pCoder) < 0) return -1;
-
-  if (tEncodeI64(pCoder, pReq->suid) < 0) return -1;
-  if (tEncodeI8(pCoder, pReq->level) < 0) return -1;
-
-  tEndEncode(pCoder);
-  return 0;
-}
-
-static FORCE_INLINE int32_t tDecodeSRSmaFetchMsg(SDecoder* pCoder, SRSmaFetchMsg* pReq) {
-  if (tStartDecode(pCoder) < 0) return -1;
-
-  if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1;
-  if (tDecodeI8(pCoder, &pReq->level) < 0) return -1;
-
-  tEndDecode(pCoder);
-  return 0;
-}
 
 typedef struct {
   int8_t         version;       // for compatibility(default 0)

include/common/tmsgdef.h

@@ -201,7 +201,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_FETCH_RSMA, "vnode-fetch-rsma", SRSmaFetchMsg, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_FETCH_RSMA, "vnode-fetch-rsma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)

include/libs/nodes/nodes.h

@@ -105,7 +105,7 @@ typedef enum ENodeType {
   QUERY_NODE_COLUMN_REF,
 
   // Statement nodes are used in parser and planner module.
-  QUERY_NODE_SET_OPERATOR,
+  QUERY_NODE_SET_OPERATOR = 100,
   QUERY_NODE_SELECT_STMT,
   QUERY_NODE_VNODE_MODIF_STMT,
   QUERY_NODE_CREATE_DATABASE_STMT,
@@ -198,7 +198,7 @@ typedef enum ENodeType {
   QUERY_NODE_QUERY,
 
   // logic plan node
-  QUERY_NODE_LOGIC_PLAN_SCAN,
+  QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
   QUERY_NODE_LOGIC_PLAN_JOIN,
   QUERY_NODE_LOGIC_PLAN_AGG,
   QUERY_NODE_LOGIC_PLAN_PROJECT,
@@ -215,7 +215,7 @@ typedef enum ENodeType {
   QUERY_NODE_LOGIC_PLAN,
 
   // physical plan node
-  QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100,
   QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
   QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN,
   QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN,

include/libs/nodes/querynodes.h

@@ -428,6 +428,9 @@ void    nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal);
 char*   nodesGetFillModeString(EFillMode mode);
 int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc);
 
+const char* operatorTypeStr(EOperatorType type);
+const char* logicConditionTypeStr(ELogicConditionType type);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/stream/tstream.h

@@ -53,6 +53,7 @@ enum {
   TASK_SCHED_STATUS__WAITING,
   TASK_SCHED_STATUS__ACTIVE,
   TASK_SCHED_STATUS__FAILED,
+  TASK_SCHED_STATUS__DROPPING,
 };
 
 enum {
@@ -127,6 +128,10 @@ typedef struct {
   int8_t type;
 } SStreamCheckpoint;
 
+typedef struct {
+  int8_t type;
+} SStreamTaskDestroy;
+
 typedef struct {
   int8_t       type;
   SSDataBlock* pBlock;
@@ -211,7 +216,6 @@ typedef struct {
   void*     vnode;
   FTbSink*  tbSinkFunc;
   STSchema* pTSchema;
-  SHashObj* pHash;  // groupId to tbuid
 } STaskSinkTb;
 
 typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data);

include/libs/sync/sync.h

@@ -33,7 +33,8 @@ extern bool gRaftDetailLog;
 #define SYNC_MAX_READ_RANGE          2
 #define SYNC_MAX_PROGRESS_WAIT_MS    4000
 #define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20)
-#define SYNC_MAX_RECV_TIME_RANGE_MS  1000
+#define SYNC_MAX_RECV_TIME_RANGE_MS  1200
+#define SYNC_ADD_QUORUM_COUNT        3
 
 #define SYNC_MAX_BATCH_SIZE 1
 #define SYNC_INDEX_BEGIN    0

include/util/taoserror.h

@@ -291,6 +291,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_STREAM_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03F1)
 #define TSDB_CODE_MND_INVALID_STREAM_OPTION     TAOS_DEF_ERROR_CODE(0, 0x03F2)
 #define TSDB_CODE_MND_STREAM_MUST_BE_DELETED    TAOS_DEF_ERROR_CODE(0, 0x03F3)
+#define TSDB_CODE_MND_STREAM_TASK_DROPPED       TAOS_DEF_ERROR_CODE(0, 0x03F4)
 
 // mnode-sma
 #define TSDB_CODE_MND_SMA_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0480)
@@ -614,6 +615,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_RSMA_REMOVE_EXISTS             TAOS_DEF_ERROR_CODE(0, 0x3154)
 #define TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP        TAOS_DEF_ERROR_CODE(0, 0x3155)
 #define TSDB_CODE_RSMA_EMPTY_INFO                TAOS_DEF_ERROR_CODE(0, 0x3156)
+#define TSDB_CODE_RSMA_INVALID_SCHEMA            TAOS_DEF_ERROR_CODE(0, 0x3157)
 
 //index
 #define TSDB_CODE_INDEX_REBUILDING               TAOS_DEF_ERROR_CODE(0, 0x3200)

include/util/tdef.h

@@ -132,15 +132,14 @@ typedef enum EOperatorType {
   OP_TYPE_DIV,
   OP_TYPE_REM,
   // unary arithmetic operator
-  OP_TYPE_MINUS,
+  OP_TYPE_MINUS = 20,
-  OP_TYPE_ASSIGN,
 
   // bitwise operator
-  OP_TYPE_BIT_AND,
+  OP_TYPE_BIT_AND = 30,
   OP_TYPE_BIT_OR,
 
   // binary comparison operator
-  OP_TYPE_GREATER_THAN,
+  OP_TYPE_GREATER_THAN = 40,
   OP_TYPE_GREATER_EQUAL,
   OP_TYPE_LOWER_THAN,
   OP_TYPE_LOWER_EQUAL,
@@ -153,7 +152,7 @@ typedef enum EOperatorType {
   OP_TYPE_MATCH,
   OP_TYPE_NMATCH,
   // unary comparison operator
-  OP_TYPE_IS_NULL,
+  OP_TYPE_IS_NULL = 100,
   OP_TYPE_IS_NOT_NULL,
   OP_TYPE_IS_TRUE,
   OP_TYPE_IS_FALSE,
@@ -163,8 +162,11 @@ typedef enum EOperatorType {
   OP_TYPE_IS_NOT_UNKNOWN,
 
   // json operator
-  OP_TYPE_JSON_GET_VALUE,
+  OP_TYPE_JSON_GET_VALUE = 150,
-  OP_TYPE_JSON_CONTAINS
+  OP_TYPE_JSON_CONTAINS,
+
+  // internal operator
+  OP_TYPE_ASSIGN = 200
 } EOperatorType;
 
 #define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR

include/util/tqueue.h

@@ -76,6 +76,7 @@ void       taosFreeQall(STaosQall *qall);
 int32_t    taosReadAllQitems(STaosQueue *queue, STaosQall *qall);
 int32_t    taosGetQitem(STaosQall *qall, void **ppItem);
 void       taosResetQitems(STaosQall *qall);
+int32_t    taosQallItemSize(STaosQall *qall);
 
 STaosQset *taosOpenQset();
 void       taosCloseQset(STaosQset *qset);

packaging/tools/make_install.bat

@@ -1,7 +1,57 @@
 @echo off
 goto %1
 :needAdmin
+
+if exist C:\\TDengine\\data\\dnode\\dnodeCfg.json (
+  echo The default data directory C:/TDengine/data contains old data of tdengine 2.x, please clear it before installing!
+)
+set source_dir=%2
+set source_dir=%source_dir:/=\\%
+set binary_dir=%3
+set binary_dir=%binary_dir:/=\\%
+set osType=%4
+set verNumber=%5
+set tagert_dir=C:\\TDengine
+
+if not exist %tagert_dir% (
+    mkdir %tagert_dir%
+)
+if not exist %tagert_dir%\\cfg (
+    mkdir %tagert_dir%\\cfg
+)
+if not exist %tagert_dir%\\include (
+    mkdir %tagert_dir%\\include
+)
+if not exist %tagert_dir%\\driver (
+    mkdir %tagert_dir%\\driver
+)
+if not exist C:\\TDengine\\cfg\\taos.cfg (
+    copy %source_dir%\\packaging\\cfg\\taos.cfg %tagert_dir%\\cfg\\taos.cfg > nul
+)
+
+if exist %binary_dir%\\test\\cfg\\taosadapter.toml (
+    if not exist %tagert_dir%\\cfg\\taosadapter.toml (
+        copy %binary_dir%\\test\\cfg\\taosadapter.toml %tagert_dir%\\cfg\\taosadapter.toml > nul
+    )
+)
+
+copy %source_dir%\\include\\client\\taos.h %tagert_dir%\\include > nul
+copy %source_dir%\\include\\util\\taoserror.h %tagert_dir%\\include > nul
+copy %source_dir%\\include\\libs\\function\\taosudf.h %tagert_dir%\\include > nul
+copy %binary_dir%\\build\\lib\\taos.lib %tagert_dir%\\driver > nul
+copy %binary_dir%\\build\\lib\\taos_static.lib %tagert_dir%\\driver > nul
+copy %binary_dir%\\build\\lib\\taos.dll %tagert_dir%\\driver > nul
+copy %binary_dir%\\build\\bin\\taos.exe %tagert_dir% > nul
+copy %binary_dir%\\build\\bin\\taosd.exe %tagert_dir% > nul
+copy %binary_dir%\\build\\bin\\udfd.exe %tagert_dir% > nul
+if exist %binary_dir%\\build\\bin\\taosBenchmark.exe (
+    copy %binary_dir%\\build\\bin\\taosBenchmark.exe %tagert_dir% > nul
+)
+if exist %binary_dir%\\build\\bin\\taosadapter.exe (
+    copy %binary_dir%\\build\\bin\\taosadapter.exe %tagert_dir% > nul
+)
+
 mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&& echo To start/stop TDengine with administrator privileges: sc start/stop taosd &goto :eof
 :hasAdmin
-cp -f C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32
+copy /y C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 > nul
 sc query "taosd" >nul || sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND

packaging/tools/make_install.sh

@@ -664,7 +664,9 @@ function install_TDengine() {
 ## ==============================Main program starts from here============================
 echo source directory: $1
 echo binary directory: $2
-if [ "$osType" != "Darwin" ]; then
+if [ -x ${data_dir}/dnode/dnodeCfg.json ]; then
+  echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of tdengine 2.x, please clear it before installing!\033[0m"
+elif [ "$osType" != "Darwin" ]; then
   if [ -x ${bin_dir}/${clientName} ]; then
     update_TDengine
   else

source/client/test/clientTests.cpp

@@ -123,7 +123,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
   }
   taos_free_result(pRes);
 
-  for(int32_t i = 0; i < 100000; i += 20) {
+  for(int32_t i = 0; i < 3280; i += 20) {
     char sql[1024] = {0};
     sprintf(sql,
             "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
@@ -679,30 +679,28 @@ TEST(testCase, projection_query_tables) {
   TAOS_RES* pRes = taos_query(pConn, "use abc1");
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "explain verbose true select _wstart,count(*),a from st1 partition by a interval(1s)");
+  pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
-  printResult(pRes);
+  if (taos_errno(pRes) != 0) {
-//  pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
+    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
-//  if (taos_errno(pRes) != 0) {
+  }
-//    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
+  taos_free_result(pRes);
-//  }
+
-//  taos_free_result(pRes);
+  pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)");
-//
+  if (taos_errno(pRes) != 0) {
-//  pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)");
+    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
-//  if (taos_errno(pRes) != 0) {
+  }
-//    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
+  taos_free_result(pRes);
-//  }
+
-//  taos_free_result(pRes);
+  pRes = taos_query(pConn, "create table tu using st1 tags(1)");
-//
+  if (taos_errno(pRes) != 0) {
-//  pRes = taos_query(pConn, "create table tu using st1 tags(1)");
+    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
-//  if (taos_errno(pRes) != 0) {
+  }
-//    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
+  taos_free_result(pRes);
-//  }
+
-//  taos_free_result(pRes);
+  for(int32_t i = 0; i < 2; ++i) {
-//
+    printf("create table :%d\n", i);
-//  for(int32_t i = 0; i < 1; ++i) {
+    createNewTable(pConn, i);
-//    printf("create table :%d\n", i);
+  }
-//    createNewTable(pConn, i);
-//  }
 //
 //  pRes = taos_query(pConn, "select * from tu");
 //  if (taos_errno(pRes) != 0) {

source/common/src/systable.c

@@ -88,7 +88,7 @@ static const SSysDbTableSchema userDBSchema[] = {
     {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "retention", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "retentions", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL},
     {.name = "cachemodel", .bytes = TSDB_CACHE_MODEL_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "cachesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT},

source/common/src/tglobal.c

@@ -75,7 +75,7 @@ int32_t  tsMonitorMaxLogs = 100;
 bool     tsMonitorComp = false;
 
 // telem
-bool     tsEnableTelem = false;
+bool     tsEnableTelem = true;
 int32_t  tsTelemInterval = 86400;
 char     tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com";
 uint16_t tsTelemPort = 80;

source/common/src/ttypes.c

@@ -392,10 +392,10 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
      getStatics_i64},
     {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f},
     {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d},
-    {TSDB_DATA_TYPE_VARCHAR, 6, 0, "VARCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_bin},
+    {TSDB_DATA_TYPE_VARCHAR, 6, 1, "VARCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_bin},
     {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp,
      tsDecompressTimestamp, getStatics_i64},
-    {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
+    {TSDB_DATA_TYPE_NCHAR, 5, 1, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
     {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint,
      getStatics_u8},
     {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint,

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -338,6 +338,7 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_RSMA, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_EXEC_RSMA, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_FETCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mnode/impl/inc/mndDef.h

@@ -636,6 +636,7 @@ typedef struct {
 
 int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj);
 int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj);
+void    tFreeStreamObj(SStreamObj* pObj);
 
 typedef struct {
   char    streamName[TSDB_STREAM_FNAME_LEN];

source/dnode/mnode/impl/inc/mndStb.h

@@ -34,6 +34,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate);
 SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName);
 int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb);
 int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb);
+void    mndFreeStb(SStbObj *pStb);
 
 void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst);
 void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize);

source/dnode/mnode/impl/src/mndDef.c

@@ -116,6 +116,25 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
   return 0;
 }
 
+void tFreeStreamObj(SStreamObj *pStream) {
+  taosMemoryFree(pStream->sql);
+  taosMemoryFree(pStream->ast);
+  taosMemoryFree(pStream->physicalPlan);
+  if (pStream->outputSchema.nCols) taosMemoryFree(pStream->outputSchema.pSchema);
+
+  int32_t sz = taosArrayGetSize(pStream->tasks);
+  for (int32_t i = 0; i < sz; i++) {
+    SArray *pLevel = taosArrayGetP(pStream->tasks, i);
+    int32_t taskSz = taosArrayGetSize(pLevel);
+    for (int32_t j = 0; j < taskSz; j++) {
+      SStreamTask *pTask = taosArrayGetP(pLevel, j);
+      tFreeSStreamTask(pTask);
+    }
+    taosArrayDestroy(pLevel);
+  }
+  taosArrayDestroy(pStream->tasks);
+}
+
 SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
   SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
   if (pVgEpNew == NULL) return NULL;

source/dnode/mnode/impl/src/mndScheduler.c

@@ -424,6 +424,8 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
       }
       mndAddTaskToTaskSet(taskSourceLevel, pTask);
 
+      pTask->triggerParam = 0;
+
       // source
       pTask->taskLevel = TASK_LEVEL__SOURCE;
 

source/dnode/mnode/impl/src/mndSma.c

@@ -489,7 +489,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
   ASSERT(smaObj.uid != 0);
   char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0};
-  snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "%s_td_tsma_rst_tb",pCreate->name);
+  snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "%s_td_tsma_rst_tb", pCreate->name);
   memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN);
   smaObj.dstTbUid = mndGenerateUid(smaObj.dstTbName, TSDB_TABLE_FNAME_LEN);
   smaObj.stbUid = pStb->uid;
@@ -530,7 +530,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   streamObj.sourceDbUid = pDb->uid;
   streamObj.targetDbUid = pDb->uid;
   streamObj.version = 1;
-  streamObj.sql = pCreate->sql;
+  streamObj.sql = strdup(pCreate->sql);
   streamObj.smaId = smaObj.uid;
   streamObj.watermark = pCreate->watermark;
   streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE;
@@ -585,6 +585,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
     return -1;
   }
   if (pAst != NULL) nodesDestroyNode(pAst);
+  nodesDestroyNode((SNode *)pPlan);
 
   int32_t code = -1;
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq);
@@ -609,6 +610,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   code = 0;
 
 _OVER:
+  tFreeStreamObj(&streamObj);
   mndDestroySmaObj(&smaObj);
   mndTransDrop(pTrans);
   return code;

source/dnode/mnode/impl/src/mndStb.c

@@ -266,6 +266,15 @@ _OVER:
   return pRow;
 }
 
+void mndFreeStb(SStbObj *pStb) {
+  taosArrayDestroy(pStb->pFuncs);
+  taosMemoryFreeClear(pStb->pColumns);
+  taosMemoryFreeClear(pStb->pTags);
+  taosMemoryFreeClear(pStb->comment);
+  taosMemoryFreeClear(pStb->pAst1);
+  taosMemoryFreeClear(pStb->pAst2);
+}
+
 static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) {
   mTrace("stb:%s, perform insert action, row:%p", pStb->name, pStb);
   return 0;
@@ -273,12 +282,7 @@ static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) {
 
 static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) {
   mTrace("stb:%s, perform delete action, row:%p", pStb->name, pStb);
-  taosArrayDestroy(pStb->pFuncs);
+  mndFreeStb(pStb);
-  taosMemoryFreeClear(pStb->pColumns);
-  taosMemoryFreeClear(pStb->pTags);
-  taosMemoryFreeClear(pStb->comment);
-  taosMemoryFreeClear(pStb->pAst1);
-  taosMemoryFreeClear(pStb->pAst2);
   return 0;
 }
 
@@ -438,6 +442,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   if (req.rollup) {
     req.rsmaParam.maxdelay[0] = pStb->maxdelay[0];
     req.rsmaParam.maxdelay[1] = pStb->maxdelay[1];
+    req.rsmaParam.watermark[0] = pStb->watermark[0];
+    req.rsmaParam.watermark[1] = pStb->watermark[1];
     if (pStb->ast1Len > 0) {
       if (mndConvertRsmaTask(&req.rsmaParam.qmsg[0], &req.rsmaParam.qmsgLen[0], pStb->pAst1, pStb->uid,
                              STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0]) < 0) {
@@ -2021,8 +2027,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName,
     FOREACH(pNode, pNodeList) {
       SColumnNode *pCol = (SColumnNode *)pNode;
 
-      if (pCol->tableId != suid) {
+      if (pCol->tableId == suid) {
-        mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId);
         sdbRelease(pSdb, pTopic);
         nodesDestroyNode(pAst);
         return -1;
@@ -2045,6 +2050,16 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
     if (pIter == NULL) break;
 
+    if (pStream->smaId != 0) {
+      sdbRelease(pSdb, pStream);
+      continue;
+    }
+
+    if (pStream->targetStbUid == suid) {
+      sdbRelease(pSdb, pStream);
+      return -1;
+    }
+
     SNode *pAst = NULL;
     if (nodesStringToNode(pStream->ast, &pAst) != 0) {
       ASSERT(0);
@@ -2057,8 +2072,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
     FOREACH(pNode, pNodeList) {
       SColumnNode *pCol = (SColumnNode *)pNode;
 
-      if (pCol->tableId != suid) {
+      if (pCol->tableId == suid) {
-        mDebug("stream:%s, check colId:%d passed", pStream->name, pCol->colId);
         sdbRelease(pSdb, pStream);
         nodesDestroyNode(pAst);
         return -1;

source/dnode/mnode/impl/src/mndStream.c

@@ -167,6 +167,9 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream) {
 
 static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) {
   mTrace("stream:%s, perform delete action", pStream->name);
+  taosWLockLatch(&pStream->lock);
+  tFreeStreamObj(pStream);
+  taosWUnLockLatch(&pStream->lock);
   return 0;
 }
 
@@ -493,10 +496,17 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
 
   stbObj.uid = pStream->targetStbUid;
 
-  if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER;
+  if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) {
+    mndFreeStb(&stbObj);
+    goto _OVER;
+  }
+
+  tFreeSMCreateStbReq(&createReq);
+  mndFreeStb(&stbObj);
 
   return 0;
 _OVER:
+  tFreeSMCreateStbReq(&createReq);
   mndReleaseStb(pMnode, pStb);
   mndReleaseDb(pMnode, pDb);
   return -1;
@@ -715,6 +725,7 @@ _OVER:
   mndReleaseDb(pMnode, pDb);
 
   tFreeSCMCreateStreamReq(&createStreamReq);
+  tFreeStreamObj(&streamObj);
   return code;
 }
 

source/dnode/mnode/impl/src/mndVgroup.c

@@ -509,6 +509,7 @@ int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) {
   pVgroup->replica = 1;
 
   if (mndGetAvailableDnode(pMnode, pDb, pVgroup, pArray) != 0) return -1;
+  taosArrayDestroy(pArray);
 
   mInfo("db:%s, sma vgId:%d is alloced", pDb->name, pVgroup->vgId);
   return 0;

source/dnode/vnode/inc/vnode.h

@@ -63,6 +63,7 @@ void    vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
 int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list);
+int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray* list);
 void   *vnodeGetIdx(SVnode *pVnode);
 void   *vnodeGetIvtIdx(SVnode *pVnode);
 
@@ -95,6 +96,7 @@ int32_t     metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHash
 int32_t     metaReadNext(SMetaReader *pReader);
 const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
 int         metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
+bool        metaIsTableExist(SMeta  *pMeta, tb_uid_t uid);
 
 typedef struct SMetaFltParam {
   tb_uid_t suid;

source/dnode/vnode/src/inc/meta.h

@@ -66,7 +66,6 @@ int32_t metaCacheOpen(SMeta* pMeta);
 void    metaCacheClose(SMeta* pMeta);
 int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo);
 int32_t metaCacheDrop(SMeta* pMeta, int64_t uid);
-int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo);
 
 struct SMeta {
   TdThreadRwlock lock;

source/dnode/vnode/src/inc/sma.h

@@ -32,7 +32,8 @@ extern "C" {
 #define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
-#define RSMA_TASK_INFO_HASH_SLOT 8
+#define RSMA_TASK_INFO_HASH_SLOT (8)
+#define RSMA_EXECUTOR_MAX        (1)
 
 typedef struct SSmaEnv       SSmaEnv;
 typedef struct SSmaStat      SSmaStat;
@@ -60,6 +61,7 @@ typedef struct {
 #define SMA_ENV_LOCK(env)  (&(env)->lock)
 #define SMA_ENV_TYPE(env)  ((env)->type)
 #define SMA_ENV_STAT(env)  ((env)->pStat)
+#define SMA_RSMA_STAT(sma) ((SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)(sma)->pRSmaEnv))
 
 struct STSmaStat {
   int8_t    state;  // ETsdbSmaStat
@@ -89,12 +91,14 @@ struct SRSmaStat {
   SSma            *pSma;
   int64_t          commitAppliedVer;  // vnode applied version for async commit
   int64_t          refId;             // shared by fetch tasks
+  volatile int64_t nBufItems;         // number of items in queue buffer
   SRWLatch         lock;              // r/w lock for rsma fs(e.g. qtaskinfo)
+  volatile int8_t  nExecutor;         // [1, max(half of query threads, 4)]
   int8_t           triggerStat;       // shared by fetch tasks
   int8_t           commitStat;        // 0 not in committing, 1 in committing
   SArray          *aTaskFile;         // qTaskFiles committed recently(for recovery/snapshot r/w)
-  SHashObj *rsmaInfoHash;      // key: stbUid, value: SRSmaInfo;
+  SHashObj        *infoHash;          // key: suid, value: SRSmaInfo
-  SHashObj *iRsmaInfoHash;     // key: stbUid, value: SRSmaInfo; immutable rsmaInfoHash
+  tsem_t           notEmpty;          // has items in queue buffer
 };
 
 struct SSmaStat {
@@ -105,19 +109,20 @@ struct SSmaStat {
   T_REF_DECLARE()
 };
 
-#define SMA_TSMA_STAT(s)      (&(s)->tsmaStat)
+#define SMA_STAT_TSMA(s)     (&(s)->tsmaStat)
-#define SMA_RSMA_STAT(s)      (&(s)->rsmaStat)
+#define SMA_STAT_RSMA(s)     (&(s)->rsmaStat)
-#define RSMA_INFO_HASH(r)     ((r)->rsmaInfoHash)
+#define RSMA_INFO_HASH(r)    ((r)->infoHash)
-#define RSMA_IMU_INFO_HASH(r) ((r)->iRsmaInfoHash)
 #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
 #define RSMA_COMMIT_STAT(r)  (&(r)->commitStat)
 #define RSMA_REF_ID(r)       ((r)->refId)
 #define RSMA_FS_LOCK(r)      (&(r)->lock)
 
 struct SRSmaInfoItem {
-  int8_t  level;
+  int8_t   level : 4;
+  int8_t   fetchLevel : 4;
   int8_t   triggerStat;
-  int32_t maxDelay;
+  uint16_t nSkipped;
+  int32_t  maxDelay;  // ms
   tmr_h    tmrId;
 };
 
@@ -125,14 +130,21 @@ struct SRSmaInfo {
   STSchema *pTSchema;
   int64_t   suid;
   int64_t   refId;     // refId of SRSmaStat
+  int64_t   lastRecv;  // ms
+  int8_t    assigned;  // 0 idle, 1 assgined for exec
   int8_t    delFlag;
+  int16_t   padding;
   T_REF_DECLARE()
   SRSmaInfoItem items[TSDB_RETENTION_L2];
   void         *taskInfo[TSDB_RETENTION_L2];   // qTaskInfo_t
-  void         *iTaskInfo[TSDB_RETENTION_L2];  // immutable
+  STaosQueue   *queue;                         // buffer queue of SubmitReq
+  STaosQall    *qall;                          // buffer qall of SubmitReq
+  void         *iTaskInfo[TSDB_RETENTION_L2];  // immutable qTaskInfo_t
+  STaosQueue   *iQueue;                        // immutable buffer queue of SubmitReq
+  STaosQall    *iQall;                         // immutable buffer qall of SubmitReq
 };
 
-#define RSMA_INFO_HEAD_LEN     32
+#define RSMA_INFO_HEAD_LEN     offsetof(SRSmaInfo, items)
 #define RSMA_INFO_IS_DEL(r)    ((r)->delFlag == 1)
 #define RSMA_INFO_SET_DEL(r)   ((r)->delFlag = 1)
 #define RSMA_INFO_QTASK(r, i)  ((r)->taskInfo[i])
@@ -161,6 +173,12 @@ enum {
   RSMA_RESTORE_SYNC = 2,
 };
 
+typedef enum {
+  RSMA_EXEC_OVERFLOW = 1,  // triggered by queue buf overflow
+  RSMA_EXEC_TIMEOUT = 2,   // triggered by timer
+  RSMA_EXEC_COMMIT = 3,    // triggered by commit
+} ERsmaExecType;
+
 void  tdDestroySmaEnv(SSmaEnv *pSmaEnv);
 void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
 
@@ -228,12 +246,13 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {
 
 void           tdRSmaQTaskInfoGetFileName(int32_t vid, int64_t version, char *outputName);
 void           tdRSmaQTaskInfoGetFullName(int32_t vid, int64_t version, const char *path, char *outputName);
-int32_t        tdCloneRSmaInfo(SSma *pSma, SRSmaInfo **pDest, SRSmaInfo *pSrc);
+int32_t        tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
 void           tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level);
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree);
 int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
+int32_t        tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type);
 
 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
 int32_t tdProcessRSmaRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer);

source/dnode/vnode/src/inc/tsdb.h

@@ -45,7 +45,7 @@ typedef struct SBlockIdx     SBlockIdx;
 typedef struct SBlock        SBlock;
 typedef struct SBlockL       SBlockL;
 typedef struct SColData      SColData;
-typedef struct SBlockDataHdr SBlockDataHdr;
+typedef struct SDiskDataHdr  SDiskDataHdr;
 typedef struct SBlockData    SBlockData;
 typedef struct SDelFile      SDelFile;
 typedef struct SHeadFile     SHeadFile;
@@ -61,7 +61,11 @@ typedef struct SRowIter      SRowIter;
 typedef struct STsdbFS       STsdbFS;
 typedef struct SRowMerger    SRowMerger;
 typedef struct STsdbReadSnap STsdbReadSnap;
+typedef struct SBlockInfo    SBlockInfo;
+typedef struct SSmaInfo      SSmaInfo;
+typedef struct SBlockCol     SBlockCol;
 
+#define TSDB_FILE_DLMT     ((uint32_t)0xF00AFA0F)
 #define TSDB_MAX_SUBBLOCKS 8
 #define TSDB_FHDR_SIZE     512
 
@@ -113,10 +117,14 @@ int32_t tPutBlock(uint8_t *p, void *ph);
 int32_t tGetBlock(uint8_t *p, void *ph);
 int32_t tBlockCmprFn(const void *p1, const void *p2);
 bool    tBlockHasSma(SBlock *pBlock);
+// SBlockL
+int32_t tPutBlockL(uint8_t *p, void *ph);
+int32_t tGetBlockL(uint8_t *p, void *ph);
 // SBlockIdx
 int32_t tPutBlockIdx(uint8_t *p, void *ph);
 int32_t tGetBlockIdx(uint8_t *p, void *ph);
 int32_t tCmprBlockIdx(void const *lhs, void const *rhs);
+int32_t tCmprBlockL(void const *lhs, void const *rhs);
 // SColdata
 void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
 void    tColDataReset(SColData *pColData);
@@ -131,20 +139,25 @@ int32_t tGetColData(uint8_t *p, SColData *pColData);
 #define tBlockDataLastRow(PBLOCKDATA)  tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
 #define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA))
 #define tBlockDataLastKey(PBLOCKDATA)  TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA))
-int32_t   tBlockDataInit(SBlockData *pBlockData);
+
+int32_t   tBlockDataCreate(SBlockData *pBlockData);
+void      tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear);
+int32_t   tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema);
+int32_t   tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom);
 void      tBlockDataReset(SBlockData *pBlockData);
-int32_t   tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema);
+int32_t   tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
-int32_t   tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom);
+void      tBlockDataClear(SBlockData *pBlockData);
-void      tBlockDataClearData(SBlockData *pBlockData);
-void      tBlockDataClear(SBlockData *pBlockData, int8_t deepClear);
-int32_t   tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData);
-int32_t   tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema);
-int32_t   tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
-int32_t   tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest);
 SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx);
 void      tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
-int32_t   tPutBlockData(uint8_t *p, SBlockData *pBlockData);
+int32_t   tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest);
-int32_t   tGetBlockData(uint8_t *p, SBlockData *pBlockData);
+int32_t   tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
+int32_t   tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData);
+int32_t   tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[],
+                         int32_t aBufN[]);
+int32_t   tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]);
+// SDiskDataHdr
+int32_t tPutDiskDataHdr(uint8_t *p, void *ph);
+int32_t tGetDiskDataHdr(uint8_t *p, void *ph);
 // SDelIdx
 int32_t tPutDelIdx(uint8_t *p, void *ph);
 int32_t tGetDelIdx(uint8_t *p, void *ph);
@@ -168,13 +181,25 @@ void    tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *m
 int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now);
 int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline);
 void    tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg);
+int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg);
+int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg);
+int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut,
+                     int32_t *szOut, uint8_t **ppBuf);
+int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut,
+                       uint8_t **ppBuf);
+int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int32_t nOut,
+                        uint8_t **ppBuf);
+int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData,
+                          uint8_t **ppBuf);
+int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck);
 // tsdbMemTable ==============================================================================================
 // SMemTable
 int32_t  tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
 void     tsdbMemTableDestroy(SMemTable *pMemTable);
-void    tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
+STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid);
 void     tsdbRefMemTable(SMemTable *pMemTable);
 void     tsdbUnrefMemTable(SMemTable *pMemTable);
+SArray  *tsdbMemTableGetTbDataArray(SMemTable *pMemTable);
 // STbDataIter
 int32_t  tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
 void    *tsdbTbDataIterDestroy(STbDataIter *pIter);
@@ -223,33 +248,33 @@ int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile);
 int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet);
 int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync);
 int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter);
-int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf);
+int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx);
-int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx);
+int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx);
-int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2,
+int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL);
-                           SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg);
+int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
+                           int8_t cmprAlg, int8_t toLast);
 
 int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
 // SDataFReader
 int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet);
 int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
-int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf);
+int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx);
-int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData, uint8_t **ppBuf);
+int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData);
-int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol,
+int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL);
-                        SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2);
+int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg);
-int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData,
+int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData);
-                          uint8_t **ppBuf1, uint8_t **ppBuf2);
+int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData);
-int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf);
 // SDelFWriter
 int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb);
 int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync);
-int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf, SDelIdx *pDelIdx);
+int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx);
-int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx, uint8_t **ppBuf);
+int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx);
 int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter);
 // SDelFReader
-int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb, uint8_t **ppBuf);
+int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb);
 int32_t tsdbDelFReaderClose(SDelFReader **ppReader);
-int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf);
+int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData);
-int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf);
+int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx);
 // tsdbRead.c ==============================================================================================
 int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap);
 void    tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap);
@@ -260,7 +285,7 @@ void    tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap);
 
 // tsdbCache
 int32_t tsdbOpenCache(STsdb *pTsdb);
-void    tsdbCloseCache(SLRUCache *pCache);
+void    tsdbCloseCache(STsdb *pTsdb);
 int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb);
 int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup);
 int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
@@ -277,13 +302,6 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode);
 int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
 
 // structs =======================
-typedef struct {
-  int   minFid;
-  int   midFid;
-  int   maxFid;
-  TSKEY minKey;
-} SRtn;
-
 struct STsdbFS {
   SDelFile *pDelFile;
   SArray   *aDFileSet;  // SArray<SDFileSet>
@@ -298,6 +316,7 @@ struct STsdb {
   SMemTable     *imem;
   STsdbFS        fs;
   SLRUCache     *lruCache;
+  TdThreadMutex  lruMutex;
 };
 
 struct TSDBKEY {
@@ -311,30 +330,23 @@ struct SMemSkipListNode {
   SMemSkipListNode *forwards[0];
 };
 typedef struct SMemSkipList {
-  uint32_t          seed;
   int64_t           size;
+  uint32_t          seed;
   int8_t            maxLevel;
   int8_t            level;
   SMemSkipListNode *pHead;
   SMemSkipListNode *pTail;
 } SMemSkipList;
 
-struct SDelDataInfo {
-  tb_uid_t suid;
-  tb_uid_t uid;
-};
-
 struct STbData {
   tb_uid_t     suid;
   tb_uid_t     uid;
   TSKEY        minKey;
   TSKEY        maxKey;
-  int64_t      minVersion;
-  int64_t      maxVersion;
-  int32_t      maxSkmVer;
   SDelData    *pHead;
   SDelData    *pTail;
   SMemSkipList sl;
+  STbData     *next;
 };
 
 struct SMemTable {
@@ -344,11 +356,13 @@ struct SMemTable {
   volatile int32_t nRef;
   TSKEY            minKey;
   TSKEY            maxKey;
-  int64_t          minVersion;
-  int64_t          maxVersion;
   int64_t          nRow;
   int64_t          nDel;
-  SArray          *aTbData;  // SArray<STbData*>
+  struct {
+    int32_t   nTbData;
+    int32_t   nBucket;
+    STbData **aBucket;
+  };
 };
 
 struct TSDBROW {
@@ -379,63 +393,51 @@ struct SMapData {
   uint8_t *pData;
 };
 
-typedef struct {
+struct SBlockCol {
   int16_t cid;
   int8_t  type;
   int8_t  smaOn;
   int8_t  flag;      // HAS_NONE|HAS_NULL|HAS_VALUE
-  int32_t offset;
-  int32_t szBitmap;  // bitmap size
-  int32_t szOffset;  // size of offset, only for variant-length data type
-  int32_t szValue;   // compressed column value size
   int32_t szOrigin;  // original column value size (only save for variant data type)
-} SBlockCol;
+  int32_t szBitmap;  // bitmap size, 0 only for flag == HAS_VAL
+  int32_t szOffset;  // offset size, 0 only for non-variant-length type
+  int32_t szValue;   // value size, 0 when flag == (HAS_NULL | HAS_NONE)
+  int32_t offset;
+};
 
-typedef struct {
+struct SBlockInfo {
-  int32_t nRow;
-  int8_t  cmprAlg;
   int64_t offset;  // block data offset
-  int32_t szBlockCol;  // SBlockCol size
+  int32_t szBlock;
-  int32_t szVersion;   // VERSION size
+  int32_t szKey;
-  int32_t szTSKEY;     // TSKEY size
+};
-  int32_t szBlock;     // total block size
+
-  int64_t sOffset;     // sma offset
+struct SSmaInfo {
-  int32_t nSma;        // sma size
+  int64_t offset;
-} SSubBlock;
+  int32_t size;
+};
 
 struct SBlock {
   TSDBKEY    minKey;
   TSDBKEY    maxKey;
-  int64_t   minVersion;
-  int64_t   maxVersion;
-  int32_t   nRow;
-  int8_t    last;
-  int8_t    hasDup;
-  int8_t    nSubBlock;
-  SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS];
-};
-
-struct SBlockL {
-  struct {
-    int64_t uid;
-    int64_t version;
-    TSKEY   ts;
-  } minKey;
-  struct {
-    int64_t uid;
-    int64_t version;
-    TSKEY   ts;
-  } maxKey;
   int64_t    minVer;
   int64_t    maxVer;
   int32_t    nRow;
-  int8_t  cmprAlg;
+  int8_t     hasDup;
-  int64_t offset;
+  int8_t     nSubBlock;
-  int32_t szBlock;
+  SBlockInfo aSubBlock[TSDB_MAX_SUBBLOCKS];
-  int32_t szBlockCol;
+  SSmaInfo   smaInfo;
-  int32_t szUid;
+};
-  int32_t szVer;
+
-  int32_t szTSKEY;
+struct SBlockL {
+  int64_t    suid;
+  int64_t    minUid;
+  int64_t    maxUid;
+  TSKEY      minKey;
+  TSKEY      maxKey;
+  int64_t    minVer;
+  int64_t    maxVer;
+  int32_t    nRow;
+  SBlockInfo bInfo;
 };
 
 struct SColData {
@@ -450,10 +452,17 @@ struct SColData {
   uint8_t *pData;
 };
 
+// (SBlockData){.suid = 0, .uid = 0}: block data not initialized
+// (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file
+// (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file
+// (SBlockData){.suid = 0, .uid = uid}: block data for 1 normal table int .last/.data file
 struct SBlockData {
-  int32_t  nRow;
+  int64_t  suid;      // 0 means normal table block data, otherwise child table block data
-  int64_t *aVersion;
+  int64_t  uid;       // 0 means block data in .last file, otherwise in .data file
-  TSKEY   *aTSKEY;
+  int32_t  nRow;      // number of rows
+  int64_t *aUid;      // uids of each row, only exist in block data in .last file (uid == 0)
+  int64_t *aVersion;  // versions of each row
+  TSKEY   *aTSKEY;    // timestamp of each row
   SArray  *aIdx;      // SArray<int32_t>
   SArray  *aColData;  // SArray<SColData>
 };
@@ -491,13 +500,18 @@ struct SDelIdx {
   int64_t  size;
 };
 
-#pragma pack(push, 1)
+struct SDiskDataHdr {
-struct SBlockDataHdr {
   uint32_t delimiter;
+  uint32_t fmtVer;
   int64_t  suid;
   int64_t  uid;
+  int32_t  szUid;
+  int32_t  szVer;
+  int32_t  szKey;
+  int32_t  szBlkCol;
+  int32_t  nRow;
+  int8_t   cmprAlg;
 };
-#pragma pack(pop)
 
 struct SDelFile {
   volatile int32_t nRef;
@@ -527,6 +541,7 @@ struct SLastFile {
 
   int64_t commitID;
   int64_t size;
+  int64_t offset;
 };
 
 struct SSmaFile {
@@ -561,6 +576,8 @@ struct SDelFWriter {
   STsdb    *pTsdb;
   SDelFile  fDel;
   TdFilePtr pWriteH;
+
+  uint8_t *aBuf[1];
 };
 
 struct SDataFWriter {
@@ -576,6 +593,8 @@ struct SDataFWriter {
   SDataFile fData;
   SLastFile fLast;
   SSmaFile  fSma;
+
+  uint8_t *aBuf[4];
 };
 
 struct STsdbReadSnap {

source/dnode/vnode/src/inc/vnd.h

@@ -65,6 +65,7 @@ struct SVBufPool {
   SVBufPool*       next;
   SVnode*          pVnode;
   volatile int32_t nRef;
+  TdThreadSpinlock lock;
   int64_t          size;
   uint8_t*         ptr;
   SVBufPoolNode*   pTail;

source/dnode/vnode/src/inc/vnodeInt.h

@@ -189,6 +189,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
 int32_t smaInit();
 void    smaCleanUp();
 int32_t smaOpen(SVnode* pVnode);
+int32_t smaPreClose(SSma* pSma);
 int32_t smaClose(SSma* pSma);
 int32_t smaBegin(SSma* pSma);
 int32_t smaSyncPreCommit(SSma* pSma);
@@ -198,7 +199,7 @@ int32_t smaAsyncPreCommit(SSma* pSma);
 int32_t smaAsyncCommit(SSma* pSma);
 int32_t smaAsyncPostCommit(SSma* pSma);
 int32_t smaDoRetention(SSma* pSma, int64_t now);
-int32_t smaProcessFetch(SSma* pSma, void* pMsg);
+int32_t smaProcessExec(SSma* pSma, void* pMsg);
 
 int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
@@ -322,6 +323,7 @@ struct SVnode {
   TdThreadMutex lock;
   bool          blocked;
   bool          restored;
+  bool          inClose;
   tsem_t        syncSem;
   SQHandle*     pQuery;
 };
@@ -368,6 +370,7 @@ struct SSma {
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
 enum {
+  SNAP_DATA_CFG = 0,
   SNAP_DATA_META = 1,
   SNAP_DATA_TSDB = 2,
   SNAP_DATA_DEL = 3,

source/dnode/vnode/src/meta/metaQuery.c

@@ -127,6 +127,15 @@ _err:
 //   return 0;
 // }
 
+bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) {
+  // query uid.idx
+  if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) {
+    return false;
+  }
+
+  return true;
+}
+
 int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
   SMeta  *pMeta = pReader->pMeta;
   int64_t version;
@@ -504,17 +513,64 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
 
 int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema **ppTSchema) {
   int32_t code = 0;
-  STSchema *pTSchema = NULL;
+
-  SSkmDbKey skmDbKey = {.uid = suid ? suid : uid, .sver = sver};
   void     *pData = NULL;
   int       nData = 0;
+  SSkmDbKey skmDbKey;
+  if (sver <= 0) {
+    SMetaInfo info;
+    if (metaGetInfo(pMeta, suid ? suid : uid, &info) == 0) {
+      sver = info.skmVer;
+    } else {
+      TBC *pSkmDbC = NULL;
+      int  c;
 
-  // query
+      skmDbKey.uid = suid ? suid : uid;
+      skmDbKey.sver = INT32_MAX;
+
+      tdbTbcOpen(pMeta->pSkmDb, &pSkmDbC, NULL);
       metaRLock(pMeta);
-  if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), &pData, &nData) < 0) {
+
-    code = TSDB_CODE_NOT_FOUND;
+      if (tdbTbcMoveTo(pSkmDbC, &skmDbKey, sizeof(skmDbKey), &c) < 0) {
         metaULock(pMeta);
-    goto _err;
+        tdbTbcClose(pSkmDbC);
+        code = TSDB_CODE_NOT_FOUND;
+        goto _exit;
+      }
+
+      ASSERT(c);
+
+      if (c < 0) {
+        tdbTbcMoveToPrev(pSkmDbC);
+      }
+
+      const void *pKey = NULL;
+      int32_t     nKey = 0;
+      tdbTbcGet(pSkmDbC, &pKey, &nKey, NULL, NULL);
+
+      if (((SSkmDbKey *)pKey)->uid != skmDbKey.uid) {
+        metaULock(pMeta);
+        tdbTbcClose(pSkmDbC);
+        code = TSDB_CODE_NOT_FOUND;
+        goto _exit;
+      }
+
+      sver = ((SSkmDbKey *)pKey)->sver;
+
+      metaULock(pMeta);
+      tdbTbcClose(pSkmDbC);
+    }
+  }
+
+  ASSERT(sver > 0);
+
+  skmDbKey.uid = suid ? suid : uid;
+  skmDbKey.sver = sver;
+  metaRLock(pMeta);
+  if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(SSkmDbKey), &pData, &nData) < 0) {
+    metaULock(pMeta);
+    code = TSDB_CODE_NOT_FOUND;
+    goto _exit;
   }
   metaULock(pMeta);
 
@@ -536,15 +592,13 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
     SSchema *pSchema = pSchemaWrapper->pSchema + i;
     tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes);
   }
-  pTSchema = tdGetSchemaFromBuilder(&sb);
+  STSchema *pTSchema = tdGetSchemaFromBuilder(&sb);
   tdDestroyTSchemaBuilder(&sb);
 
   *ppTSchema = pTSchema;
   taosMemoryFree(pSchemaWrapper->pSchema);
-  return code;
 
-_err:
+_exit:
-  *ppTSchema = NULL;
   return code;
 }
 
@@ -997,6 +1051,8 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t metaCacheGet(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo);
+
 int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo) {
   int32_t code = 0;
   void   *pData = NULL;

source/dnode/vnode/src/meta/metaTable.c

@@ -357,10 +357,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   metaSaveToTbDb(pMeta, &nStbEntry);
 
   // update uid index
-  SMetaInfo info;
+  metaUpdateUidIdx(pMeta, &nStbEntry);
-  metaGetEntryInfo(&nStbEntry, &info);
-  tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t),
-               &(SUidIdxVal){.suid = info.suid, .version = info.version, .skmVer = info.skmVer}, sizeof(SUidIdxVal), 0);
 
   if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
   metaULock(pMeta);
@@ -884,7 +881,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   }
 
   SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid};
-  tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag*)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn);
+  tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags,
+              ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn);
 
   tDecoderClear(&dc1);
   tDecoderClear(&dc2);
@@ -1091,7 +1089,8 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
 static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
 
-  return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, ((STag*)(pME->ctbEntry.pTags))->len, &pMeta->txn);
+  return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags,
+                     ((STag *)(pME->ctbEntry.pTags))->len, &pMeta->txn);
 }
 
 int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid,

source/dnode/vnode/src/sma/smaCommit.c

@@ -83,8 +83,7 @@ int32_t smaBegin(SSma *pSma) {
     return TSDB_CODE_SUCCESS;
   }
 
-  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv);
-  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
 
   int8_t rsmaTriggerStat =
       atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED, TASK_TRIGGER_STAT_ACTIVE);
@@ -110,7 +109,7 @@ int32_t smaBegin(SSma *pSma) {
 /**
  * @brief pre-commit for rollup sma(sync commit).
  *  1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED.
- *  2) wait all triggered fetch tasks finished
+ *  2) wait for all triggered fetch tasks to finish
  *  3) perform persist task for qTaskInfo
  *
  * @param pSma
@@ -123,19 +122,19 @@ static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) {
   }
 
   SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
-  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+  SRSmaStat *pRSmaStat = SMA_STAT_RSMA(pStat);
 
   // step 1: set rsma stat paused
   atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
 
-  // step 2: wait all triggered fetch tasks finished
+  // step 2: wait for all triggered fetch tasks to finish
   int32_t nLoops = 0;
   while (1) {
     if (T_REF_VAL_GET(pStat) == 0) {
-      smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma));
+      smaDebug("vgId:%d, rsma fetch tasks are all finished", SMA_VID(pSma));
       break;
     } else {
-      smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma));
+      smaDebug("vgId:%d, rsma fetch tasks are not all finished yet", SMA_VID(pSma));
     }
     ++nLoops;
     if (nLoops > 1000) {
@@ -289,8 +288,7 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) {
     return TSDB_CODE_SUCCESS;
   }
 
-  SSmaEnv   *pSmaEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma);
-  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(SMA_ENV_STAT(pSmaEnv));
 
   // cleanup outdated qtaskinfo files
   tdCleanupQTaskInfoFiles(pSma, pRSmaStat);
@@ -299,10 +297,9 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) {
 }
 
 /**
- * @brief Rsma async commit implementation
+ * @brief Rsma async commit implementation(only do some necessary light weighted task)
  *  1) set rsma stat TASK_TRIGGER_STAT_PAUSED
  *  2) Wait all running fetch task finish to fetch and put submitMsg into level 2/3 wQueue(blocking level 1 write)
- *  3)
  *
  * @param pSma
  * @return int32_t
@@ -314,20 +311,22 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
   }
 
   SSmaStat  *pStat = SMA_ENV_STAT(pEnv);
-  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+  SRSmaStat *pRSmaStat = SMA_STAT_RSMA(pStat);
 
   // step 1: set rsma stat
   atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
   atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 1);
+  pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied;
+  ASSERT(pRSmaStat->commitAppliedVer > 0);
 
-  // step 2: wait all triggered fetch tasks finished
+  // step 2: wait for all triggered fetch tasks to finish
   int32_t nLoops = 0;
   while (1) {
     if (T_REF_VAL_GET(pStat) == 0) {
-      smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma));
+      smaDebug("vgId:%d, rsma commit, fetch tasks are all finished", SMA_VID(pSma));
       break;
     } else {
-      smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma));
+      smaDebug("vgId:%d, rsma commit, fetch tasks are not all finished yet", SMA_VID(pSma));
     }
     ++nLoops;
     if (nLoops > 1000) {
@@ -336,29 +335,51 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
     }
   }
 
-  // step 3:  swap rsmaInfoHash and iRsmaInfoHash
+  /**
-  // lock
+   * @brief step 3: consume the SubmitReq in buffer
-  taosWLockLatch(SMA_ENV_LOCK(pEnv));
+   *  1) This is high cost task and should not put in asyncPreCommit originally.
-
+   *  2) But, if put in asyncCommit, would trigger taskInfo cloning frequently.
-  ASSERT(RSMA_INFO_HASH(pRSmaStat));
+   */
-  ASSERT(!RSMA_IMU_INFO_HASH(pRSmaStat));
+  if (tdRSmaProcessExecImpl(pSma, RSMA_EXEC_COMMIT) < 0) {
-
-  RSMA_IMU_INFO_HASH(pRSmaStat) = RSMA_INFO_HASH(pRSmaStat);
-  RSMA_INFO_HASH(pRSmaStat) =
-      taosHashInit(RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
-
-  if (!RSMA_INFO_HASH(pRSmaStat)) {
-    // unlock
-    taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
-    smaError("vgId:%d, rsma async commit failed since %s", SMA_VID(pSma), terrstr());
     return TSDB_CODE_FAILED;
   }
 
-  // unlock
+  smaInfo("vgId:%d, rsma commit, wait for all items to be consumed, TID:%p", SMA_VID(pSma), (void*)taosGetSelfPthreadId());
-  taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
+  nLoops = 0;
+  while (atomic_load_64(&pRSmaStat->nBufItems) > 0) {
+    ++nLoops;
+    if (nLoops > 1000) {
+      sched_yield();
+      nLoops = 0;
+    }
+  }
+  smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
+  if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) {
+    return TSDB_CODE_FAILED;
+  }
+  smaInfo("vgId:%d, rsma commit, operator state commited, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
 
-  // step 4: others
+#if 0 // consuming task of qTaskInfo clone 
-  pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied;
+  // step 4:  swap queue/qall and iQueue/iQall
+  // lock
+  // taosWLockLatch(SMA_ENV_LOCK(pEnv));
+
+  ASSERT(RSMA_INFO_HASH(pRSmaStat));
+
+  void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
+
+  while (pIter) {
+    SRSmaInfo *pInfo = *(SRSmaInfo **)pIter;
+    TSWAP(pInfo->iQall, pInfo->qall);
+    TSWAP(pInfo->iQueue, pInfo->queue);
+    TSWAP(pInfo->iTaskInfo[0], pInfo->taskInfo[0]);
+    TSWAP(pInfo->iTaskInfo[1], pInfo->taskInfo[1]);
+    pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter);
+  }
+
+  // unlock
+  // taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
+#endif
 
   return TSDB_CODE_SUCCESS;
 }
@@ -374,18 +395,20 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) {
   if (!pSmaEnv) {
     return TSDB_CODE_SUCCESS;
   }
+#if 0
+  SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv);
 
-  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  // perform persist task for qTaskInfo operator
-  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+  if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) {
-
+    return TSDB_CODE_FAILED;
-  // perform persist task for qTaskInfo
+  }
-  tdRSmaPersistExecImpl(pRSmaStat, RSMA_IMU_INFO_HASH(pRSmaStat));
+#endif
 
   return TSDB_CODE_SUCCESS;
 }
 
 /**
- * @brief Migrate rsmaInfo from iRsmaInfo to rsmaInfo if rsmaInfoHash not empty.
+ * @brief Migrate rsmaInfo from iRsmaInfo to rsmaInfo if rsma infoHash not empty.
  *
  * @param pSma
  * @return int32_t
@@ -396,37 +419,25 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
     return TSDB_CODE_SUCCESS;
   }
 
-  SSmaStat  *pStat = SMA_ENV_STAT(pEnv);
+  SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
-  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+  SArray    *rsmaDeleted = NULL;
 
-  // step 1: merge rsmaInfoHash and iRsmaInfoHash
+  // step 1: merge qTaskInfo and iQTaskInfo
   // lock
-  taosWLockLatch(SMA_ENV_LOCK(pEnv));
+  // taosWLockLatch(SMA_ENV_LOCK(pEnv));
-#if 0
-  if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) {
-    // just switch the hash pointer if rsmaInfoHash is empty
-    if (taosHashGetSize(RSMA_IMU_INFO_HASH(pRSmaStat)) > 0) {
-      SHashObj *infoHash = RSMA_INFO_HASH(pRSmaStat);
-      RSMA_INFO_HASH(pRSmaStat) = RSMA_IMU_INFO_HASH(pRSmaStat);
-      RSMA_IMU_INFO_HASH(pRSmaStat) = infoHash;
-    }
-  } else {
-#endif
-#if 1
-  void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL);
-  while (pIter) {
-    tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
 
-    if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) {
+  void *pIter = NULL;
+  while ((pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter))) {
+    tb_uid_t  *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
     SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter;
     if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
       int32_t refVal = T_REF_VAL_GET(pRSmaInfo);
       if (refVal == 0) {
-          tdFreeRSmaInfo(pSma, pRSmaInfo, true);
+        if (!rsmaDeleted) {
-          smaDebug(
+          if ((rsmaDeleted = taosArrayInit(1, sizeof(tb_uid_t)))) {
-              "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for "
+            taosArrayPush(rsmaDeleted, pSuid);
-              "table:%" PRIi64,
+          }
-              SMA_VID(pSma), *pSuid);
+        }
       } else {
         smaDebug(
             "vgId:%d, rsma async post commit, not free rsma info since ref is %d although already deleted for "
@@ -434,30 +445,40 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
             SMA_VID(pSma), refVal, *pSuid);
       }
 
-        pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter);
       continue;
     }
-      taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter));
+#if 0
-      smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma),
+    if (pRSmaInfo->taskInfo[0]) {
-               *pSuid);
+      if (pRSmaInfo->iTaskInfo[0]) {
-    } else {
+        SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pRSmaInfo->iTaskInfo[0];
-      // free the resources
-      SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter;
         tdFreeRSmaInfo(pSma, pRSmaInfo, false);
-      smaDebug("vgId:%d, rsma async post commit, free rsma info since already COW for table:%" PRIi64, SMA_VID(pSma),
+        pRSmaInfo->iTaskInfo[0] = NULL;
-               *pSuid);
+      }
+    } else {
+      TSWAP(pRSmaInfo->taskInfo[0], pRSmaInfo->iTaskInfo[0]);
     }
 
-    pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter);
+    taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter));
-  }
+    smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid);
 #endif
-  // }
+  }
 
-  taosHashCleanup(RSMA_IMU_INFO_HASH(pRSmaStat));
+  for (int32_t i = 0; i < taosArrayGetSize(rsmaDeleted); ++i) {
-  RSMA_IMU_INFO_HASH(pRSmaStat) = NULL;
+    tb_uid_t *pSuid = taosArrayGet(rsmaDeleted, i);
+    void     *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t));
+    if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+      tdFreeRSmaInfo(pSma, pRSmaInfo, true);
+      smaDebug(
+          "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for "
+          "table:%" PRIi64,
+          SMA_VID(pSma), *pSuid);
+    }
+    taosHashRemove(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t));
+  }
+  taosArrayDestroy(rsmaDeleted);
 
   // unlock
-  taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
+  // taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
 
   // step 2: cleanup outdated qtaskinfo files
   tdCleanupQTaskInfoFiles(pSma, pRSmaStat);

source/dnode/vnode/src/sma/smaEnv.c

@@ -209,6 +209,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
       SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat);
       pRSmaStat->pSma = (SSma *)pSma;
       atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT);
+      tsem_init(&pRSmaStat->notEmpty, 0, 0);
 
       // init smaMgmt
       smaInit();
@@ -228,7 +229,6 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
       RSMA_INFO_HASH(pRSmaStat) = taosHashInit(
           RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
       if (!RSMA_INFO_HASH(pRSmaStat)) {
-        taosMemoryFreeClear(*pSmaStat);
         return TSDB_CODE_FAILED;
       }
     } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
@@ -262,10 +262,9 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
     smaDebug("vgId:%d, destroy rsma stat %p", SMA_VID(pSma), pRSmaStat);
     // step 1: set rsma trigger stat cancelled
     atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
+    tsem_destroy(&(pStat->notEmpty));
 
     // step 2: destroy the rsma info and associated fetch tasks
-    // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
-#if 1
     if (taosHashGetSize(RSMA_INFO_HASH(pStat)) > 0) {
       void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
       while (infoHash) {
@@ -274,17 +273,16 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
         infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
       }
     }
-#endif
     taosHashCleanup(RSMA_INFO_HASH(pStat));
 
-    // step 3: wait all triggered fetch tasks finished
+    // step 3: wait for all triggered fetch tasks to finish
     int32_t nLoops = 0;
     while (1) {
       if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) {
-        smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma));
+        smaDebug("vgId:%d, rsma fetch tasks are all finished", SMA_VID(pSma));
         break;
       } else {
-        smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma));
+        smaDebug("vgId:%d, rsma fetch tasks are not all finished yet", SMA_VID(pSma));
       }
       ++nLoops;
       if (nLoops > 1000) {
@@ -293,7 +291,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
       }
     }
 
-    // step 4: free pStat
+    // step 5: free pStat
     taosMemoryFreeClear(pStat);
   }
 }
@@ -318,9 +316,9 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
 int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
   if (pSmaStat) {
     if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
-      tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat));
+      tdDestroyTSmaStat(SMA_STAT_TSMA(pSmaStat));
     } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
-      SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat);
+      SRSmaStat *pRSmaStat = &pSmaStat->rsmaStat;
       int32_t    vid = SMA_VID(pRSmaStat->pSma);
       int64_t    refId = RSMA_REF_ID(pRSmaStat);
       if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) {

source/dnode/vnode/src/sma/smaOpen.c

@@ -146,6 +146,20 @@ int32_t smaClose(SSma *pSma) {
   return 0;
 }
 
+int32_t smaPreClose(SSma *pSma) {
+  if (pSma && VND_IS_RSMA(pSma->pVnode)) {
+    SSmaEnv   *pEnv = NULL;
+    SRSmaStat *pStat = NULL;
+    if (!(pEnv = SMA_RSMA_ENV(pSma)) || !(pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv))) {
+      return 0;
+    }
+    for (int32_t i = 0; i < RSMA_EXECUTOR_MAX; ++i) {
+      tsem_post(&(pStat->notEmpty));
+    }
+  }
+  return 0;
+}
+
 /**
  * @brief rsma env restore
  * 

source/dnode/vnode/src/sma/smaRollup.c

@@ -15,8 +15,12 @@
 
 #include "sma.h"
 
-#define RSMA_QTASKINFO_BUFSIZE  32768
+#define RSMA_QTASKINFO_BUFSIZE     (32768)                                               // size
 #define RSMA_QTASKINFO_HEAD_LEN    (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t))  // len + type + suid
+#define RSMA_QTASKEXEC_SMOOTH_SIZE (100)                                                 // cnt
+#define RSMA_SUBMIT_BATCH_SIZE     (1024)                                                // cnt
+#define RSMA_FETCH_DELAY_MAX       (900000)                                              // ms
+#define RSMA_FETCH_ACTIVE_MAX      (1800)                                                // ms
 
 SSmaMgmt smaMgmt = {
     .inited = 0,
@@ -27,19 +31,21 @@ SSmaMgmt smaMgmt = {
 #define TD_RSMAINFO_DEL_FILE      "rsmainfo.del"
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
+typedef struct SRSmaExecQItem     SRSmaExecQItem;
 
 static int32_t    tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
 static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
 static int32_t    tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
                                           int8_t idx);
-static int32_t    tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid,
+static int32_t    tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo,
-                                    int8_t level);
+                                    ERsmaExecType type, int8_t level);
 static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid);
 static void       tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
-static int32_t    tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema,
+static void       tdFreeRSmaSubmitItems(SArray *pItems);
-                                             int64_t suid, int8_t blkType);
+static int32_t    tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr);
+static int32_t    tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema,
+                                            int64_t suid);
 static void       tdRSmaFetchTrigger(void *param, void *tmrId);
-static int32_t    tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level);
 static int32_t    tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
 static int32_t    tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
 static int32_t    tdRSmaQTaskInfoRestore(SSma *pSma, int8_t type, SRSmaQTaskInfoIter *pIter);
@@ -76,6 +82,11 @@ struct SRSmaQTaskInfoIter {
   int32_t nBufPos;
 };
 
+struct SRSmaExecQItem {
+  void *pRSmaInfo;
+  void *qall;
+};
+
 void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName) {
   tdGetVndFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName);
 }
@@ -139,6 +150,18 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
     if (isDeepFree) {
       taosMemoryFreeClear(pInfo->pTSchema);
     }
+
+    if (isDeepFree) {
+      if (pInfo->queue) taosCloseQueue(pInfo->queue);
+      if (pInfo->qall) taosFreeQall(pInfo->qall);
+      if (pInfo->iQueue) taosCloseQueue(pInfo->iQueue);
+      if (pInfo->iQall) taosFreeQall(pInfo->iQall);
+      pInfo->queue = NULL;
+      pInfo->qall = NULL;
+      pInfo->iQueue = NULL;
+      pInfo->iQall = NULL;
+    }
+
     taosMemoryFree(pInfo);
   }
 
@@ -179,7 +202,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids)
 
   for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
     if (pRSmaInfo->taskInfo[i]) {
-      if ((qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, true) < 0)) {
+      if (((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, true)) < 0)) {
         tdReleaseRSmaInfo(pSma, pRSmaInfo);
         smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i,
                  terrstr());
@@ -351,6 +374,19 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
     goto _err;
   }
   pRSmaInfo->pTSchema = pTSchema;
+  if (!(pRSmaInfo->queue = taosOpenQueue())) {
+    goto _err;
+  }
+
+  if (!(pRSmaInfo->qall = taosAllocateQall())) {
+    goto _err;
+  }
+  if (!(pRSmaInfo->iQueue = taosOpenQueue())) {
+    goto _err;
+  }
+  if (!(pRSmaInfo->iQall = taosAllocateQall())) {
+    goto _err;
+  }
   pRSmaInfo->suid = suid;
   pRSmaInfo->refId = RSMA_REF_ID(pStat);
   T_REF_INIT_VAL(pRSmaInfo, 1);
@@ -419,8 +455,7 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) {
     return TSDB_CODE_SUCCESS;
   }
 
-  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv);
-  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
 
   SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pReq->suid);
 
@@ -528,6 +563,14 @@ void *tdUidStoreFree(STbUidStore *pStore) {
   return NULL;
 }
 
+/**
+ * @brief The SubmitReq for rsma L2/L3 is inserted by tsdbInsertData method directly while not by WriteQ, as the queue
+ * would be freed when close Vnode, thus lock should be used if with race condition.
+ * @param pTsdb
+ * @param version
+ * @param pReq
+ * @return int32_t
+ */
 static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
   if (!pReq) {
     terrno = TSDB_CODE_INVALID_PTR;
@@ -535,7 +578,7 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
   }
 
   SSubmitReq *pSubmitReq = (SSubmitReq *)pReq;
-
+  // TODO: spin lock for race conditiond
   if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) {
     return TSDB_CODE_FAILED;
   }
@@ -569,17 +612,6 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
   return 0;
 }
 
-static void tdDestroySDataBlockArray(SArray *pArray) {
-  // TODO
-#if 0
-  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
-    SSDataBlock *pDataBlock = taosArrayGet(pArray, i);
-    blockDestroyInner(pDataBlock);
-  }
-#endif
-  taosArrayDestroy(pArray);
-}
-
 /**
  * @brief retention of rsma1/rsma2
  *
@@ -604,8 +636,8 @@ _end:
   return code;
 }
 
-static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema,
+static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema,
-                                          int64_t suid, int8_t blkType) {
+                                         int64_t suid) {
   SArray *pResList = taosArrayInit(1, POINTER_BYTES);
   if (pResList == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -637,8 +669,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm
     } else {
       smaDebug("vgId:%d, rsma %" PRIi8 " data fetched", SMA_VID(pSma), pItem->level);
     }
-
+#if 0
-#if 1
     char flag[10] = {0};
     snprintf(flag, 10, "level %" PRIi8, pItem->level);
     blockDebugShowDataBlocks(pResList, flag);
@@ -665,7 +696,6 @@ static int32_t tdRSmaFetchAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSm
 
       smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " version:%" PRIi64,
                SMA_VID(pSma), suid, pItem->level, output->info.version);
-
     }
   }
 
@@ -677,34 +707,115 @@ _err:
   return TSDB_CODE_FAILED;
 }
 
-static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid,
+/**
-                                 int8_t level) {
+ * @brief Copy msg to rsmaQueueBuffer for batch process
+ *
+ * @param pSma
+ * @param pMsg
+ * @param inputType
+ * @param pInfo
+ * @param suid
+ * @return int32_t
+ */
+static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo,
+                                      tb_uid_t suid) {
+  const SSubmitReq *pReq = (const SSubmitReq *)pMsg;
+
+  void *qItem = taosAllocateQitem(pReq->header.contLen, DEF_QITEM);
+  if (!qItem) {
+    return TSDB_CODE_FAILED;
+  }
+
+  memcpy(qItem, pMsg, pReq->header.contLen);
+
+  taosWriteQitem(pInfo->queue, qItem);
+
+  pInfo->lastRecv = taosGetTimestampMs();
+
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma);
+
+  tsem_post(&(pRSmaStat->notEmpty));
+
+  int64_t nItems = atomic_fetch_add_64(&pRSmaStat->nBufItems, 1);
+
+  // smoothing consume
+  int32_t n = nItems / RSMA_QTASKEXEC_SMOOTH_SIZE;
+  if (n > 1) {
+    if (n > 10) {
+      n = 10;
+    }
+    taosMsleep(n << 3);
+    if (n > 5) {
+      smaWarn("vgId:%d, pInfo->queue itemSize:%d, memSize:%" PRIi64 ", sleep %d ms", SMA_VID(pSma),
+              taosQueueItemSize(pInfo->queue), taosQueueMemorySize(pInfo->queue), n << 3);
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) {
+  SSubmitMsgIter msgIter = {0};
+  SSubmitBlkIter blkIter = {0};
+  STSRow        *row = NULL;
+  if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1;
+  while (true) {
+    SSubmitBlk *pBlock = NULL;
+    if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+    if (pBlock == NULL) break;
+    tInitSubmitBlkIter(&msgIter, pBlock, &blkIter);
+    while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) {
+      smaDebug("vgId:%d, numOfRows:%d, suid:%" PRIi64 ", uid:%" PRIi64 ", version:%" PRIi64 ", ts:%" PRIi64,
+               SMA_VID(pSma), msgIter.numOfRows, msgIter.suid, msgIter.uid, pReq->version, row->ts);
+    }
+  }
+  return 0;
+}
+
+/**
+ * @brief sync mode
+ *
+ * @param pSma
+ * @param pMsg
+ * @param msgSize
+ * @param inputType
+ * @param pInfo
+ * @param type
+ * @param level
+ * @return int32_t
+ */
+static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo,
+                                 ERsmaExecType type, int8_t level) {
   int32_t idx = level - 1;
-  if (!pInfo || !RSMA_INFO_QTASK(pInfo, idx)) {
+
-    smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
+  void *qTaskInfo = (type == RSMA_EXEC_COMMIT) ? RSMA_INFO_IQTASK(pInfo, idx) : RSMA_INFO_QTASK(pInfo, idx);
+  if (!qTaskInfo) {
+    smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level,
+             pInfo->suid);
     return TSDB_CODE_SUCCESS;
   }
   if (!pInfo->pTSchema) {
-    smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
+    smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, pInfo->suid);
     return TSDB_CODE_FAILED;
   }
 
   smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level,
-           RSMA_INFO_QTASK(pInfo, idx), suid);
+           RSMA_INFO_QTASK(pInfo, idx), pInfo->suid);
 
-  if (qSetMultiStreamInput(RSMA_INFO_QTASK(pInfo, idx), pMsg, 1, inputType) < 0) {  // INPUT__DATA_SUBMIT
+#if 0
+  for (int32_t i = 0; i < msgSize; ++i) {
+    SSubmitReq *pReq = *(SSubmitReq **)((char *)pMsg + i * sizeof(void *));
+    smaDebug("vgId:%d, [%d][%d] version %" PRIi64, SMA_VID(pSma), msgSize, i, pReq->version);
+    tdRsmaPrintSubmitReq(pSma, pReq);
+  }
+#endif
+  if (qSetMultiStreamInput(qTaskInfo, pMsg, msgSize, inputType) < 0) {
     smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno));
     return TSDB_CODE_FAILED;
   }
 
   SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx);
-  tdRSmaFetchAndSubmitResult(pSma, RSMA_INFO_QTASK(pInfo, idx), pItem, pInfo->pTSchema, suid,
+  tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo->pTSchema, pInfo->suid);
-                             STREAM_INPUT__DATA_SUBMIT);
-  atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
-
-  if (smaMgmt.tmrHandle) {
-    taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
-  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -732,58 +843,27 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) {
     return NULL;
   }
 
-  taosRLockLatch(SMA_ENV_LOCK(pEnv));
+  // taosRLockLatch(SMA_ENV_LOCK(pEnv));
   pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
   if (pRSmaInfo && (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
     if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
-      taosRUnLockLatch(SMA_ENV_LOCK(pEnv));
+      // taosRUnLockLatch(SMA_ENV_LOCK(pEnv));
       return NULL;
     }
+    if (!pRSmaInfo->taskInfo[0]) {
+      if (tdCloneRSmaInfo(pSma, pRSmaInfo) < 0) {
+        // taosRUnLockLatch(SMA_ENV_LOCK(pEnv));
+        return NULL;
+      }
+    }
     tdRefRSmaInfo(pSma, pRSmaInfo);
-    taosRUnLockLatch(SMA_ENV_LOCK(pEnv));
+    // taosRUnLockLatch(SMA_ENV_LOCK(pEnv));
+    ASSERT(pRSmaInfo->suid == suid);
     return pRSmaInfo;
   }
-  taosRUnLockLatch(SMA_ENV_LOCK(pEnv));
+  // taosRUnLockLatch(SMA_ENV_LOCK(pEnv));
 
-  if (RSMA_COMMIT_STAT(pStat) == 0) {  // return NULL if not in committing stat
   return NULL;
-  }
-
-  // clone the SRSmaInfo from iRsmaInfoHash to rsmaInfoHash if in committing stat
-  SRSmaInfo *pCowRSmaInfo = NULL;
-  // lock
-  taosWLockLatch(SMA_ENV_LOCK(pEnv));
-  if (!(pCowRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)))) {  // 2-phase lock
-    void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
-    if (iRSmaInfo) {
-      SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo;
-      if (pIRSmaInfo && !RSMA_INFO_IS_DEL(pIRSmaInfo)) {
-        if (tdCloneRSmaInfo(pSma, &pCowRSmaInfo, pIRSmaInfo) < 0) {
-          // unlock
-          taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
-          smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr());
-          return NULL;
-        }
-        smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid);
-        if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) {
-          // unlock
-          taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
-          smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr());
-          return NULL;
-        }
-      }
-    }
-  } else {
-    pCowRSmaInfo = *(SRSmaInfo **)pCowRSmaInfo;
-    ASSERT(!pCowRSmaInfo);
-  }
-
-  if (pCowRSmaInfo) {
-    tdRefRSmaInfo(pSma, pCowRSmaInfo);
-  }
-  // unlock
-  taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
-  return pCowRSmaInfo;
 }
 
 static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
@@ -792,29 +872,85 @@ static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
   }
 }
 
-static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
+/**
+ * @brief async mode
+ *
+ * @param pSma
+ * @param pMsg
+ * @param inputType
+ * @param suid
+ * @return int32_t
+ */
+static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
   SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid);
   if (!pRSmaInfo) {
-    smaError("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
+    smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
     return TSDB_CODE_SUCCESS;
   }
 
   if (inputType == STREAM_INPUT__DATA_SUBMIT) {
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L1);
+    if (tdExecuteRSmaImplAsync(pSma, pMsg, inputType, pRSmaInfo, suid) < 0) {
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L2);
+      tdReleaseRSmaInfo(pSma, pRSmaInfo);
+      return TSDB_CODE_FAILED;
+    }
+    if (smaMgmt.tmrHandle) {
+      SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, 0);
+      if (pItem->level > 0) {
+        atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
+      }
+      pItem = RSMA_INFO_ITEM(pRSmaInfo, 1);
+      if (pItem->level > 0) {
+        atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
+      }
+    }
+  } else {
+    ASSERT(0);
   }
 
   tdReleaseRSmaInfo(pSma, pRSmaInfo);
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t tdRSmaExecCheck(SSma *pSma) {
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma);
+
+  if (atomic_load_8(&pRSmaStat->nExecutor) >= TMIN(RSMA_EXECUTOR_MAX, tsNumOfVnodeQueryThreads / 2)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SRSmaExecMsg fetchMsg;
+  int32_t      contLen = sizeof(SMsgHead);
+  void        *pBuf = rpcMallocCont(0 + contLen);
+
+  ((SMsgHead *)pBuf)->vgId = SMA_VID(pSma);
+  ((SMsgHead *)pBuf)->contLen = sizeof(SMsgHead);
+
+  SRpcMsg rpcMsg = {
+      .code = 0,
+      .msgType = TDMT_VND_EXEC_RSMA,
+      .pCont = pBuf,
+      .contLen = contLen,
+  };
+
+  if ((terrno = tmsgPutToQueue(&pSma->pVnode->msgCb, QUERY_QUEUE, &rpcMsg)) != 0) {
+    smaError("vgId:%d, failed to put rsma exec msg into query-queue since %s", SMA_VID(pSma), terrstr());
+    goto _err;
+  }
+
+  smaDebug("vgId:%d, success to put rsma fetch msg into query-queue", SMA_VID(pSma));
+
+  return TSDB_CODE_SUCCESS;
+_err:
+  return TSDB_CODE_FAILED;
+}
+
 int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
   SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
   if (!pEnv) {
     // only applicable when rsma env exists
     return TSDB_CODE_SUCCESS;
   }
-
+  STbUidStore uidStore = {0};
   SRetention *pRetention = SMA_RETENTION(pSma);
   if (!RETENTION_VALID(pRetention + 1)) {
     // return directly if retention level 1 is invalid
@@ -822,49 +958,75 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
   }
 
   if (inputType == STREAM_INPUT__DATA_SUBMIT) {
-    STbUidStore uidStore = {0};
+    if (tdFetchSubmitReqSuids(pMsg, &uidStore) < 0) {
-    tdFetchSubmitReqSuids(pMsg, &uidStore);
+      goto _err;
+    }
 
     if (uidStore.suid != 0) {
-      tdExecuteRSma(pSma, pMsg, inputType, uidStore.suid);
+      if (tdExecuteRSmaAsync(pSma, pMsg, inputType, uidStore.suid) < 0) {
+        goto _err;
+      }
 
-      void *pIter = taosHashIterate(uidStore.uidHash, NULL);
+      void *pIter = NULL;
-      while (pIter) {
+      while ((pIter = taosHashIterate(uidStore.uidHash, pIter))) {
         tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
-        tdExecuteRSma(pSma, pMsg, inputType, *pTbSuid);
+        if (tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid) < 0) {
-        pIter = taosHashIterate(uidStore.uidHash, pIter);
+          goto _err;
+        }
       }
 
+      if (tdRSmaExecCheck(pSma) < 0) {
+        goto _err;
+      }
+    }
+  }
   tdUidStoreDestory(&uidStore);
-    }
-  }
   return TSDB_CODE_SUCCESS;
+_err:
+  tdUidStoreDestory(&uidStore);
+  smaError("vgId:%d, failed to process rsma submit since: %s", SMA_VID(pSma), terrstr());
+  return TSDB_CODE_FAILED;
 }
 
+/**
+ * @brief retrieve rsma meta and init
+ *
+ * @param pSma
+ * @param nTables number of tables of rsma
+ * @return int32_t
+ */
 static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
   SVnode     *pVnode = pSma->pVnode;
+  SArray     *suidList = NULL;
+  STbUidStore uidStore = {0};
+  SMetaReader mr = {0};
 
-  SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
+  if (!(suidList = taosArrayInit(1, sizeof(tb_uid_t)))) {
-  if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosArrayDestroy(suidList);
+    goto _err;
+  }
+
+  if (vnodeGetStbIdList(pSma->pVnode, 0, suidList) < 0) {
     smaError("vgId:%d, failed to restore rsma env since get stb id list error: %s", TD_VID(pVnode), terrstr());
-    return TSDB_CODE_FAILED;
+    goto _err;
   }
 
   int64_t arrSize = taosArrayGetSize(suidList);
 
-  if (nTables) {
-    *nTables = arrSize;
-  }
-
   if (arrSize == 0) {
+    if (nTables) {
+      *nTables = 0;
+    }
     taosArrayDestroy(suidList);
     smaDebug("vgId:%d, no need to restore rsma env since empty stb id list", TD_VID(pVnode));
     return TSDB_CODE_SUCCESS;
   }
 
-  SMetaReader mr = {0};
+  int64_t nRsmaTables = 0;
   metaReaderInit(&mr, SMA_META(pSma), 0);
+  if (!(uidStore.tbUids = taosArrayInit(1024, sizeof(tb_uid_t)))) {
+    goto _err;
+  }
   for (int64_t i = 0; i < arrSize; ++i) {
     tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
     smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid);
@@ -877,6 +1039,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
     ASSERT(mr.me.type == TSDB_SUPER_TABLE);
     ASSERT(mr.me.uid == suid);
     if (TABLE_IS_ROLLUP(mr.me.flags)) {
+      ++nRsmaTables;
       SRSmaParam *param = &mr.me.stbEntry.rsmaParam;
       for (int i = 0; i < TSDB_RETENTION_L2; ++i) {
         smaDebug("vgId:%d, rsma restore, table:%" PRIi64 " level:%d, maxdelay:%" PRIi64 " watermark:%" PRIi64
@@ -887,17 +1050,40 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
         smaError("vgId:%d, rsma restore env failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr());
         goto _err;
       }
+
+      // reload all ctbUids for suid
+      uidStore.suid = suid;
+      if (vnodeGetCtbIdList(pVnode, suid, uidStore.tbUids) < 0) {
+        smaError("vgId:%d, rsma restore, get ctb idlist failed for %" PRIi64 " since %s", TD_VID(pVnode), suid,
+                 terrstr());
+        goto _err;
+      }
+
+      if (tdUpdateTbUidList(pVnode->pSma, &uidStore) < 0) {
+        smaError("vgId:%d, rsma restore, update tb uid list failed for %" PRIi64 " since %s", TD_VID(pVnode), suid,
+                 terrstr());
+        goto _err;
+      }
+
+      taosArrayClear(uidStore.tbUids);
+
       smaDebug("vgId:%d, rsma restore env success for %" PRIi64, TD_VID(pVnode), suid);
     }
   }
 
   metaReaderClear(&mr);
   taosArrayDestroy(suidList);
+  tdUidStoreDestory(&uidStore);
+
+  if (nTables) {
+    *nTables = nRsmaTables;
+  }
 
   return TSDB_CODE_SUCCESS;
 _err:
   metaReaderClear(&mr);
   taosArrayDestroy(suidList);
+  tdUidStoreDestory(&uidStore);
 
   return TSDB_CODE_FAILED;
 }
@@ -1230,6 +1416,9 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
     }
 
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+#if 0
+      qTaskInfo_t taskInfo = RSMA_INFO_IQTASK(pRSmaInfo, i);
+#endif
       qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pRSmaInfo, i);
       if (!taskInfo) {
         smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1);
@@ -1367,7 +1556,16 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       smaDebug("vgId:%d, rsma fetch task started for level:%" PRIi8 " suid:%" PRIi64 " since stat is active",
                SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
       // async process
-      tdRSmaFetchSend(pSma, pRSmaInfo, pItem->level);
+      pItem->fetchLevel = pItem->level;
+#if 0
+      SRSmaInfo     *qInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaInfo->suid);
+      SRSmaInfoItem *qItem = RSMA_INFO_ITEM(qInfo, pItem->level - 1);
+      ASSERT(qItem->level == pItem->level);
+      ASSERT(qItem->fetchLevel == pItem->fetchLevel);
+#endif
+      tsem_post(&(pStat->notEmpty));
+      smaInfo("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64, SMA_VID(pSma), pItem->level,
+              pRSmaInfo->suid);
     } break;
     case TASK_TRIGGER_STAT_PAUSED: {
       smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is paused",
@@ -1382,127 +1580,268 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
                SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
     } break;
     default: {
-      smaWarn("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is unknown",
+      smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is unknown",
                SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
     } break;
   }
 
 _end:
-  // taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
+  taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
   tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId);
 }
 
+static void tdFreeRSmaSubmitItems(SArray *pItems) {
+  for (int32_t i = 0; i < taosArrayGetSize(pItems); ++i) {
+    taosFreeQitem(*(void **)taosArrayGet(pItems, i));
+  }
+}
+
 /**
- * @brief put rsma fetch msg to fetch queue
+ * @brief fetch rsma result(consider the efficiency and functionality)
  *
  * @param pSma
  * @param pInfo
- * @param level
+ * @param pSubmitArr
  * @return int32_t
  */
-int32_t tdRSmaFetchSend(SSma *pSma, SRSmaInfo *pInfo, int8_t level) {
+static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr) {
-  SRSmaFetchMsg fetchMsg = {.suid = pInfo->suid, .level = level};
+  SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
-  int32_t       ret = 0;
+  for (int8_t i = 1; i <= TSDB_RETENTION_L2; ++i) {
-  int32_t       contLen = 0;
+    SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1);
-  SEncoder      encoder = {0};
+    if (pItem->fetchLevel) {
-  tEncodeSize(tEncodeSRSmaFetchMsg, &fetchMsg, contLen, ret);
+      pItem->fetchLevel = 0;
-  if (ret < 0) {
+      qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pInfo, i - 1);
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
+      if (!taskInfo) {
-    tEncoderClear(&encoder);
+        continue;
+      }
+
+      int64_t curMs = taosGetTimestampMs();
+      if ((pItem->nSkipped * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) {
+        smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch executed",
+                SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay);
+      } else if (((curMs - pInfo->lastRecv) < RSMA_FETCH_ACTIVE_MAX)) {
+        ++pItem->nSkipped;
+        smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ",
+                 SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv);
+        continue;
+      } else {
+        smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ",
+                SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv);
+      }
+
+      pItem->nSkipped = 0;
+
+      if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) {
+        goto _err;
+      }
+      if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) {
+        tdCleanupStreamInputDataBlock(taskInfo);
         goto _err;
       }
 
-  void *pBuf = rpcMallocCont(contLen + sizeof(SMsgHead));
+      tdCleanupStreamInputDataBlock(taskInfo);
-  tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), contLen);
+      smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch finished",
-  if (tEncodeSRSmaFetchMsg(&encoder, &fetchMsg) < 0) {
+              SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay);
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    } else {
-    tEncoderClear(&encoder);
+      smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8
+               " maxDelay:%d, fetch not executed as fetch level is %" PRIi8,
+               SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay, pItem->fetchLevel);
     }
-  tEncoderClear(&encoder);
-
-  ((SMsgHead *)pBuf)->vgId = SMA_VID(pSma);
-  ((SMsgHead *)pBuf)->contLen = contLen + sizeof(SMsgHead);
-
-  SRpcMsg rpcMsg = {
-      .code = 0,
-      .msgType = TDMT_VND_FETCH_RSMA,
-      .pCont = pBuf,
-      .contLen = contLen,
-  };
-
-  if ((terrno = tmsgPutToQueue(&pSma->pVnode->msgCb, QUERY_QUEUE, &rpcMsg)) != 0) {
-    smaError("vgId:%d, failed to put rsma fetch msg into fetch-queue for suid:%" PRIi64 " level:%" PRIi8 " since %s",
-             SMA_VID(pSma), pInfo->suid, level, terrstr());
-    goto _err;
   }
 
-  smaDebug("vgId:%d, success to put rsma fetch msg into fetch-queue for suid:%" PRIi64 " level:%" PRIi8, SMA_VID(pSma),
+_end:
-           pInfo->suid, level);
+  tdReleaseRSmaInfo(pSma, pInfo);
-
   return TSDB_CODE_SUCCESS;
 _err:
+  tdReleaseRSmaInfo(pSma, pInfo);
+  return TSDB_CODE_FAILED;
+}
+
+static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SArray *pSubmitArr, ERsmaExecType type) {
+  taosArrayClear(pSubmitArr);
+  while (1) {
+    void *msg = NULL;
+    taosGetQitem(qall, (void **)&msg);
+    if (msg) {
+      if (taosArrayPush(pSubmitArr, &msg) < 0) {
+        tdFreeRSmaSubmitItems(pSubmitArr);
+        goto _err;
+      }
+    } else {
+      break;
+    }
+  }
+
+  int32_t size = taosArrayGetSize(pSubmitArr);
+  if (size > 0) {
+    for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) {
+      if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, type, i) < 0) {
+        tdFreeRSmaSubmitItems(pSubmitArr);
+        goto _err;
+      }
+    }
+    tdFreeRSmaSubmitItems(pSubmitArr);
+  }
+  return TSDB_CODE_SUCCESS;
+_err:
+  while (1) {
+    void *msg = NULL;
+    taosGetQitem(qall, (void **)&msg);
+    if (msg) {
+      taosFreeQitem(msg);
+    } else {
+      break;
+    }
+  }
   return TSDB_CODE_FAILED;
 }
 
 /**
- * @brief fetch rsma data of level 2/3 and submit
+ * @brief
+ *
+ * @param pSma
+ * @param type
+ * @return int32_t
+ */
+int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) {
+  SVnode    *pVnode = pSma->pVnode;
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+  SHashObj  *infoHash = NULL;
+  SArray    *pSubmitArr = NULL;
+  bool       isFetchAll = false;
+
+  if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) {
+    terrno = TSDB_CODE_RSMA_INVALID_STAT;
+    goto _err;
+  }
+
+  if (!(pSubmitArr =
+            taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), POINTER_BYTES))) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  bool isBusy = false;
+  while (true) {
+    isBusy = false;
+    // step 1: rsma exec - consume data in buffer queue for all suids
+    if (type == RSMA_EXEC_OVERFLOW || type == RSMA_EXEC_COMMIT) {
+      void *pIter = taosHashIterate(infoHash, NULL);  // infoHash has r/w lock
+      while (pIter) {
+        SRSmaInfo *pInfo = *(SRSmaInfo **)pIter;
+        int64_t    itemSize = 0;
+        if ((itemSize = taosQueueItemSize(pInfo->queue)) || RSMA_INFO_ITEM(pInfo, 0)->fetchLevel ||
+            RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) {
+          smaDebug("vgId:%d, queueItemSize is %" PRIi64 " execType:%" PRIi8, SMA_VID(pSma), itemSize, type);
+          if (atomic_val_compare_exchange_8(&pInfo->assigned, 0, 1) == 0) {
+            taosReadAllQitems(pInfo->queue, pInfo->qall);  // queue has mutex lock
+            int32_t qallItemSize = taosQallItemSize(pInfo->qall);
+            if (qallItemSize > 0) {
+              tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type);
+            }
+
+            if (type == RSMA_EXEC_OVERFLOW) {
+              tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr);
+            }
+
+            if (qallItemSize > 0) {
+              // subtract the item size after the task finished, commit should wait for all items be consumed
+              atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize);
+              isBusy = true;
+            }
+            ASSERT(1 == atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0));
+          }
+        }
+        pIter = taosHashIterate(infoHash, pIter);
+      }
+      if (type == RSMA_EXEC_COMMIT) {
+        break;
+      }
+    }
+#if 0
+    else if (type == RSMA_EXEC_COMMIT) {
+      while (pIter) {
+        SRSmaInfo *pInfo = *(SRSmaInfo **)pIter;
+        if (taosQueueItemSize(pInfo->iQueue)) {
+          if (atomic_val_compare_exchange_8(&pInfo->assigned, 0, 1) == 0) {
+            taosReadAllQitems(pInfo->iQueue, pInfo->iQall);  // queue has mutex lock
+            int32_t qallItemSize = taosQallItemSize(pInfo->iQall);
+            if (qallItemSize > 0) {
+              atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize);
+              nIdle = 0;
+
+              // batch exec
+              tdRSmaBatchExec(pSma, pInfo, pInfo->qall, pSubmitArr, type);
+            }
+
+            // tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr);
+            ASSERT(1 == atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0));
+          }
+        }
+        ASSERT(taosQueueItemSize(pInfo->iQueue) == 0);
+        pIter = taosHashIterate(infoHash, pIter);
+      }
+      break;
+    }
+#endif
+    else {
+      ASSERT(0);
+    }
+
+    if (atomic_load_64(&pRSmaStat->nBufItems) <= 0) {
+      if (pVnode->inClose) {
+        break;
+      }
+      tsem_wait(&pRSmaStat->notEmpty);
+      if (pVnode->inClose && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) {
+        smaInfo("vgId:%d, exec task end, inClose:%d, nBufItems:%" PRIi64, SMA_VID(pSma), pVnode->inClose,
+                atomic_load_64(&pRSmaStat->nBufItems));
+        break;
+      }
+    }
+  }  // end of while(true)
+
+_end:
+  taosArrayDestroy(pSubmitArr);
+  return TSDB_CODE_SUCCESS;
+_err:
+  taosArrayDestroy(pSubmitArr);
+  return TSDB_CODE_FAILED;
+}
+
+/**
+ * @brief exec rsma level 1data, fetch result of level 2/3 and submit
  *
  * @param pSma
  * @param pMsg
  * @return int32_t
  */
-int32_t smaProcessFetch(SSma *pSma, void *pMsg) {
+int32_t smaProcessExec(SSma *pSma, void *pMsg) {
   SRpcMsg   *pRpcMsg = (SRpcMsg *)pMsg;
-  SRSmaFetchMsg  req = {0};
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma);
-  SDecoder       decoder = {0};
-  void          *pBuf = NULL;
-  SRSmaInfo     *pInfo = NULL;
-  SRSmaInfoItem *pItem = NULL;
 
   if (!pRpcMsg || pRpcMsg->contLen < sizeof(SMsgHead)) {
     terrno = TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP;
     goto _err;
   }
+  smaDebug("vgId:%d, begin to process rsma exec msg by TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
 
-  pBuf = POINTER_SHIFT(pRpcMsg->pCont, sizeof(SMsgHead));
+  int8_t nOld = atomic_fetch_add_8(&pRSmaStat->nExecutor, 1);
 
-  tDecoderInit(&decoder, pBuf, pRpcMsg->contLen);
+  if (nOld < TMIN(RSMA_EXECUTOR_MAX, tsNumOfVnodeQueryThreads / 2)) {
-  if (tDecodeSRSmaFetchMsg(&decoder, &req) < 0) {
+    if (tdRSmaProcessExecImpl(pSma, RSMA_EXEC_OVERFLOW) < 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
       goto _err;
     }
-
+  } else {
-  pInfo = tdAcquireRSmaInfoBySuid(pSma, req.suid);
+    atomic_fetch_sub_8(&pRSmaStat->nExecutor, 1);
-  if (!pInfo) {
-    if (terrno == TSDB_CODE_SUCCESS) {
-      terrno = TSDB_CODE_RSMA_EMPTY_INFO;
-    }
-    smaWarn("vgId:%d, failed to process rsma fetch msg for suid:%" PRIi64 " level:%" PRIi8 " since %s", SMA_VID(pSma),
-            req.suid, req.level, terrstr());
-    goto _err;
   }
 
-  pItem = RSMA_INFO_ITEM(pInfo, req.level - 1);
+  smaDebug("vgId:%d, success to process rsma exec msg by TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
-
-  SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
-  qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pInfo, req.level - 1);
-  if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) {
-    goto _err;
-  }
-  if (tdRSmaFetchAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid, STREAM_INPUT__DATA_BLOCK) < 0) {
-    goto _err;
-  }
-
-  tdCleanupStreamInputDataBlock(taskInfo);
-
-  tdReleaseRSmaInfo(pSma, pInfo);
-  tDecoderClear(&decoder);
-  smaDebug("vgId:%d, success to process rsma fetch msg for suid:%" PRIi64 " level:%" PRIi8, SMA_VID(pSma), req.suid,
-           req.level);
   return TSDB_CODE_SUCCESS;
 _err:
-  tdReleaseRSmaInfo(pSma, pInfo);
+  atomic_fetch_sub_8(&pRSmaStat->nExecutor, 1);
-  tDecoderClear(&decoder);
+  smaError("vgId:%d, failed to process rsma exec msg by TID:%p since %s", SMA_VID(pSma), (void *)taosGetSelfPthreadId(),
-  smaError("vgId:%d, failed to process rsma fetch msg since %s", SMA_VID(pSma), terrstr());
+           terrstr());
   return TSDB_CODE_FAILED;
 }

source/dnode/vnode/src/sma/smaSnapshot.c

@@ -139,7 +139,6 @@ static int32_t rsmaSnapReadQTaskInfo(SRsmaSnapReader* pReader, uint8_t** ppBuf)
 
   smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo, size:%" PRIi64, SMA_VID(pSma), size);
 
-
   SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppBuf);
   pHdr->type = SNAP_DATA_QTASK;
   pHdr->size = size;
@@ -279,7 +278,8 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapWrit
   TdFilePtr qTaskF = taosCreateFile(qTaskInfoFullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (!qTaskF) {
     code = TAOS_SYSTEM_ERROR(errno);
-    smaError("vgId:%d, rsma snapshot writer open %s failed since %s", TD_VID(pSma->pVnode), qTaskInfoFullName, tstrerror(code));
+    smaError("vgId:%d, rsma snapshot writer open %s failed since %s", TD_VID(pSma->pVnode), qTaskInfoFullName,
+             tstrerror(code));
     goto _err;
   }
   qWriter->pWriteH = qTaskF;
@@ -309,7 +309,7 @@ int32_t rsmaSnapWriterClose(SRsmaSnapWriter** ppWriter, int8_t rollback) {
   if (rollback) {
     // TODO: rsma1/rsma2
     // qtaskinfo
-    if(pWriter->pQTaskFWriter) {
+    if (pWriter->pQTaskFWriter) {
       taosRemoveFile(pWriter->pQTaskFWriter->fname);
     }
   } else {

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -175,7 +175,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
   }
 
   tdRefSmaStat(pSma, pStat);
-  pTsmaStat = SMA_TSMA_STAT(pStat);
+  pTsmaStat = SMA_STAT_TSMA(pStat);
 
   if (!pTsmaStat->pTSma) {
     STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid);

source/dnode/vnode/src/sma/smaUtil.c

@@ -350,49 +350,48 @@ _err:
 }
 
 /**
- * @brief pTSchema is shared
+ * @brief Clone qTaskInfo of SRSmaInfo
  *
  * @param pSma
- * @param pDest
+ * @param pInfo
- * @param pSrc
  * @return int32_t
  */
-int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo **pDest, SRSmaInfo *pSrc) {
+int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
-  SVnode     *pVnode = pSma->pVnode;
   SRSmaParam *param = NULL;
-  if (!pSrc) {
+  if (!pInfo) {
-    *pDest = NULL;
     return TSDB_CODE_SUCCESS;
   }
 
   SMetaReader mr = {0};
   metaReaderInit(&mr, SMA_META(pSma), 0);
-  smaDebug("vgId:%d, rsma clone, suid is %" PRIi64, TD_VID(pVnode), pSrc->suid);
+  smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid);
-  if (metaGetTableEntryByUid(&mr, pSrc->suid) < 0) {
+  if (metaGetTableEntryByUid(&mr, pInfo->suid) < 0) {
-    smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), pSrc->suid,
+    smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid,
              terrstr());
     goto _err;
   }
   ASSERT(mr.me.type == TSDB_SUPER_TABLE);
-  ASSERT(mr.me.uid == pSrc->suid);
+  ASSERT(mr.me.uid == pInfo->suid);
   if (TABLE_IS_ROLLUP(mr.me.flags)) {
     param = &mr.me.stbEntry.rsmaParam;
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
-      if (tdCloneQTaskInfo(pSma, pSrc->iTaskInfo[i], pSrc->taskInfo[i], param, pSrc->suid, i) < 0) {
+      if (!pInfo->iTaskInfo[i]) {
+        continue;
+      }
+      if (tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i) < 0) {
         goto _err;
       }
     }
-    smaDebug("vgId:%d, rsma clone env success for %" PRIi64, TD_VID(pVnode), pSrc->suid);
+    smaDebug("vgId:%d, rsma clone env success for %" PRIi64, SMA_VID(pSma), pInfo->suid);
+  } else {
+    terrno = TSDB_CODE_RSMA_INVALID_SCHEMA;
+    goto _err;
   }
 
   metaReaderClear(&mr);
-
-  *pDest = pSrc;  // pointer copy
-
   return TSDB_CODE_SUCCESS;
 _err:
-  *pDest = NULL;
   metaReaderClear(&mr);
-  smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", TD_VID(pVnode), pSrc->suid, terrstr());
+  smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, terrstr());
   return TSDB_CODE_FAILED;
 }

source/dnode/vnode/src/tq/tq.c

@@ -628,8 +628,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe
 }
 
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) {
-  int32_t code = 0;
-
   if (pTask->taskLevel == TASK_LEVEL__AGG) {
     ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0);
   }
@@ -640,8 +638,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) {
   pTask->outputQueue = streamQueueOpen();
 
   if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) {
-    code = -1;
+    return -1;
-    goto FAIL;
   }
 
   pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
@@ -686,14 +683,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) {
 
   streamSetupTrigger(pTask);
 
-  tqInfo("deploy stream task on vg %d, task id %d, child id %d", TD_VID(pTq->pVnode), pTask->taskId,
+  tqInfo("expand stream task on vg %d, task id %d, child id %d", TD_VID(pTq->pVnode), pTask->taskId,
          pTask->selfChildId);
-
+  return 0;
-FAIL:
-  if (pTask->inputQueue) streamQueueClose(pTask->inputQueue);
-  if (pTask->outputQueue) streamQueueClose(pTask->outputQueue);
-  // TODO free executor
-  return code;
 }
 
 int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {

source/dnode/vnode/src/tq/tqRead.c

@@ -341,7 +341,7 @@ FAIL:
   return -1;
 }
 
-void tqReaderSetColIdList(STqReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
+void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; }
 
 int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) {
   if (pReader->tbIdHash) {

source/dnode/vnode/src/tq/tqSink.c

@@ -231,11 +231,12 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
 
   ASSERT(pTask->tbSink.pTSchema);
   deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq));
-  SSubmitReq* pReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid,
+  SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid,
                                           pTask->tbSink.stbFullName, &deleteReq);
 
   tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId);
 
+  if (taosArrayGetSize(deleteReq.deleteReqs) != 0) {
     int32_t code;
     int32_t len;
     tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code);
@@ -244,21 +245,21 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
       ASSERT(0);
     }
     SEncoder encoder;
-  void*    buf = rpcMallocCont(len + sizeof(SMsgHead));
+    void*    serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
-  void*    abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+    void*    abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));
     tEncoderInit(&encoder, abuf, len);
     tEncodeSBatchDeleteReq(&encoder, &deleteReq);
     tEncoderClear(&encoder);
 
-  ((SMsgHead*)buf)->vgId = pVnode->config.vgId;
+    ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId;
 
-  if (taosArrayGetSize(deleteReq.deleteReqs) != 0) {
     SRpcMsg msg = {
         .msgType = TDMT_VND_BATCH_DEL,
-        .pCont = buf,
+        .pCont = serializedDeleteReq,
         .contLen = len + sizeof(SMsgHead),
     };
     if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+      rpcFreeCont(serializedDeleteReq);
       tqDebug("failed to put into write-queue since %s", terrstr());
     }
   }
@@ -268,11 +269,12 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
   // build write msg
   SRpcMsg msg = {
       .msgType = TDMT_VND_SUBMIT,
-      .pCont = pReq,
+      .pCont = submitReq,
-      .contLen = ntohl(pReq->length),
+      .contLen = ntohl(submitReq->length),
   };
 
   if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
+    rpcFreeCont(submitReq);
     tqDebug("failed to put into write-queue since %s", terrstr());
   }
 }

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -33,16 +33,21 @@ int32_t tsdbOpenCache(STsdb *pTsdb) {
 
   taosLRUCacheSetStrictCapacity(pCache, true);
 
+  taosThreadMutexInit(&pTsdb->lruMutex, NULL);
+
 _err:
   pTsdb->lruCache = pCache;
   return code;
 }
 
-void tsdbCloseCache(SLRUCache *pCache) {
+void tsdbCloseCache(STsdb *pTsdb) {
+  SLRUCache *pCache = pTsdb->lruCache;
   if (pCache) {
     taosLRUCacheEraseUnrefEntries(pCache);
 
     taosLRUCacheCleanup(pCache);
+
+    taosThreadMutexDestroy(&pTsdb->lruMutex);
   }
 }
 
@@ -261,14 +266,14 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
     }
 
     for (++iCol; iCol < nCol; ++iCol) {
-      SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
+      SLastCol *tTsVal1 = (SLastCol *)taosArrayGet(pLast, iCol);
-      if (keyTs >= tTsVal->ts) {
+      if (keyTs >= tTsVal1->ts) {
-        SColVal *tColVal = &tTsVal->colVal;
+        SColVal *tColVal = &tTsVal1->colVal;
 
         SColVal colVal = {0};
         tTSRowGetVal(row, pTSchema, iCol, &colVal);
         if (colVal.isNone || colVal.isNull) {
-          if (keyTs == tTsVal->ts && !tColVal->isNone && !tColVal->isNull) {
+          if (keyTs == tTsVal1->ts && !tColVal->isNone && !tColVal->isNull) {
             invalidate = true;
 
             break;
@@ -279,6 +284,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
       }
     }
 
+  _invalidate:
     taosMemoryFreeClear(pTSchema);
 
     taosLRUCacheRelease(pCache, h, invalidate);
@@ -317,7 +323,7 @@ static int32_t getTableDelDataFromDelIdx(SDelFReader *pDelReader, SDelIdx *pDelI
   int32_t code = 0;
 
   if (pDelIdx) {
-    code = tsdbReadDelData(pDelReader, pDelIdx, aDelData, NULL);
+    code = tsdbReadDelData(pDelReader, pDelIdx, aDelData);
   }
 
   return code;
@@ -388,8 +394,7 @@ static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t
   SDelIdx idx = {.suid = suid, .uid = uid};
 
   // tMapDataReset(&delIdxMap);
-  //  code = tsdbReadDelIdx(pDelFReader, &delIdxMap, NULL);
+  code = tsdbReadDelIdx(pDelFReader, pDelIdxArray);
-  code = tsdbReadDelIdx(pDelFReader, pDelIdxArray, NULL);
   if (code) goto _err;
 
   // code = tMapDataSearch(&delIdxMap, &idx, tGetDelIdx, tCmprDelIdx, pDelIdx);
@@ -405,6 +410,178 @@ _err:
   return code;
 }
 
+typedef enum {
+  SFSLASTNEXTROW_FS,
+  SFSLASTNEXTROW_FILESET,
+  SFSLASTNEXTROW_BLOCKDATA,
+  SFSLASTNEXTROW_BLOCKROW
+} SFSLASTNEXTROWSTATES;
+
+typedef struct {
+  SFSLASTNEXTROWSTATES state;         // [input]
+  STsdb               *pTsdb;         // [input]
+  SBlockIdx           *pBlockIdxExp;  // [input]
+  STSchema            *pTSchema;      // [input]
+  int32_t              nFileSet;
+  int32_t              iFileSet;
+  SArray              *aDFileSet;
+  SDataFReader        *pDataFReader;
+  SArray              *aBlockL;
+  SBlockL             *pBlockL;
+  SBlockData          *pBlockDataL;
+  SBlockData           blockDataL;
+  int32_t              nRow;
+  int32_t              iRow;
+  TSDBROW              row;
+  /*
+  SArray    *aBlockIdx;
+  SBlockIdx *pBlockIdx;
+  SMapData   blockMap;
+  int32_t    nBlock;
+  int32_t    iBlock;
+  SBlock     block;
+  */
+} SFSLastNextRowIter;
+
+static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
+  SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter;
+  int32_t             code = 0;
+
+  switch (state->state) {
+    case SFSLASTNEXTROW_FS:
+      // state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet;
+      state->nFileSet = taosArrayGetSize(state->aDFileSet);
+      state->iFileSet = state->nFileSet;
+
+      state->pBlockDataL = NULL;
+
+    case SFSLASTNEXTROW_FILESET: {
+      SDFileSet *pFileSet = NULL;
+    _next_fileset:
+      if (--state->iFileSet >= 0) {
+        pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet);
+      } else {
+        if (state->pBlockDataL) {
+          tBlockDataDestroy(state->pBlockDataL, 1);
+          state->pBlockDataL = NULL;
+        }
+
+        *ppRow = NULL;
+        return code;
+      }
+
+      code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet);
+      if (code) goto _err;
+
+      if (!state->aBlockL) {
+        state->aBlockL = taosArrayInit(0, sizeof(SBlockIdx));
+      } else {
+        taosArrayClear(state->aBlockL);
+      }
+
+      code = tsdbReadBlockL(state->pDataFReader, state->aBlockL);
+      if (code) goto _err;
+
+      // SBlockL *pBlockL = (SBlockL *)taosArrayGet(state->aBlockL, state->iBlockL);
+
+      state->pBlockL = taosArraySearch(state->aBlockL, state->pBlockIdxExp, tCmprBlockL, TD_EQ);
+      if (!state->pBlockL) {
+        goto _next_fileset;
+      }
+
+      int64_t suid = state->pBlockL->suid;
+      int64_t uid = state->pBlockL->maxUid;
+
+      if (!state->pBlockDataL) {
+        state->pBlockDataL = &state->blockDataL;
+      }
+      code = tBlockDataInit(state->pBlockDataL, suid, suid ? 0 : uid, state->pTSchema);
+      if (code) goto _err;
+    }
+    case SFSLASTNEXTROW_BLOCKDATA:
+      code = tsdbReadLastBlock(state->pDataFReader, state->pBlockL, state->pBlockDataL);
+      if (code) goto _err;
+
+      state->nRow = state->blockDataL.nRow;
+      state->iRow = state->nRow - 1;
+
+      if (!state->pBlockDataL->uid) {
+        while (state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) {
+          --state->iRow;
+        }
+      }
+
+      state->state = SFSLASTNEXTROW_BLOCKROW;
+    case SFSLASTNEXTROW_BLOCKROW:
+      if (state->pBlockDataL->uid) {
+        if (state->iRow >= 0) {
+          state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow);
+          *ppRow = &state->row;
+
+          if (--state->iRow < 0) {
+            state->state = SFSLASTNEXTROW_FILESET;
+          }
+        }
+      } else {
+        if (state->iRow >= 0 && state->pBlockIdxExp->uid == state->pBlockDataL->aUid[state->iRow]) {
+          state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow);
+          *ppRow = &state->row;
+
+          if (--state->iRow < 0 || state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) {
+            state->state = SFSLASTNEXTROW_FILESET;
+          }
+        }
+      }
+
+      return code;
+    default:
+      ASSERT(0);
+      break;
+  }
+
+_err:
+  if (state->pDataFReader) {
+    tsdbDataFReaderClose(&state->pDataFReader);
+    state->pDataFReader = NULL;
+  }
+  if (state->aBlockL) {
+    taosArrayDestroy(state->aBlockL);
+    state->aBlockL = NULL;
+  }
+  if (state->pBlockDataL) {
+    tBlockDataDestroy(state->pBlockDataL, 1);
+    state->pBlockDataL = NULL;
+  }
+
+  *ppRow = NULL;
+
+  return code;
+}
+
+int32_t clearNextRowFromFSLast(void *iter) {
+  SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter;
+  int32_t             code = 0;
+
+  if (!state) {
+    return code;
+  }
+
+  if (state->pDataFReader) {
+    tsdbDataFReaderClose(&state->pDataFReader);
+    state->pDataFReader = NULL;
+  }
+  if (state->aBlockL) {
+    taosArrayDestroy(state->aBlockL);
+    state->aBlockL = NULL;
+  }
+  if (state->pBlockDataL) {
+    tBlockDataDestroy(state->pBlockDataL, 1);
+    state->pBlockDataL = NULL;
+  }
+
+  return code;
+}
+
 typedef enum SFSNEXTROWSTATES {
   SFSNEXTROW_FS,
   SFSNEXTROW_FILESET,
@@ -451,9 +628,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
       if (--state->iFileSet >= 0) {
         pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet);
       } else {
-        // tBlockDataClear(&state->blockData, 1);
+        // tBlockDataDestroy(&state->blockData, 1);
         if (state->pBlockData) {
-          tBlockDataClear(state->pBlockData, 1);
+          tBlockDataDestroy(state->pBlockData, 1);
           state->pBlockData = NULL;
         }
 
@@ -465,13 +642,12 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
       if (code) goto _err;
 
       // tMapDataReset(&state->blockIdxMap);
-      // code = tsdbReadBlockIdx(state->pDataFReader, &state->blockIdxMap, NULL);
       if (!state->aBlockIdx) {
         state->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
       } else {
         taosArrayClear(state->aBlockIdx);
       }
-      code = tsdbReadBlockIdx(state->pDataFReader, state->aBlockIdx, NULL);
+      code = tsdbReadBlockIdx(state->pDataFReader, state->aBlockIdx);
       if (code) goto _err;
 
       /* if (state->pBlockIdx) { */
@@ -487,8 +663,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
       }
 
       tMapDataReset(&state->blockMap);
-      code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap, NULL);
+      code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap);
-      /* code = tsdbReadBlock(state->pDataFReader, &state->blockIdx, &state->blockMap, NULL); */
       if (code) goto _err;
 
       state->nBlock = state->blockMap.nItem;
@@ -497,7 +672,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
       if (!state->pBlockData) {
         state->pBlockData = &state->blockData;
 
-        tBlockDataInit(&state->blockData);
+        tBlockDataCreate(&state->blockData);
       }
     }
     case SFSNEXTROW_BLOCKDATA:
@@ -510,7 +685,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
 
         tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock);
         /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */
-        code = tsdbReadBlockData(state->pDataFReader, state->pBlockIdx, &block, state->pBlockData, NULL, NULL);
+        code = tsdbReadDataBlock(state->pDataFReader, &block, state->pBlockData);
         if (code) goto _err;
 
         state->nRow = state->blockData.nRow;
@@ -555,8 +730,8 @@ _err:
     state->aBlockIdx = NULL;
   }
   if (state->pBlockData) {
-    // tBlockDataClear(&state->blockData, 1);
+    // tBlockDataDestroy(&state->blockData, 1);
-    tBlockDataClear(state->pBlockData, 1);
+    tBlockDataDestroy(state->pBlockData, 1);
     state->pBlockData = NULL;
   }
 
@@ -582,8 +757,8 @@ int32_t clearNextRowFromFS(void *iter) {
     state->aBlockIdx = NULL;
   }
   if (state->pBlockData) {
-    // tBlockDataClear(&state->blockData, 1);
+    // tBlockDataDestroy(&state->blockData, 1);
-    tBlockDataClear(state->pBlockData, 1);
+    tBlockDataDestroy(state->pBlockData, 1);
     state->pBlockData = NULL;
   }
 
@@ -728,15 +903,16 @@ typedef struct {
   SBlockIdx          idx;
   SMemNextRowIter    memState;
   SMemNextRowIter    imemState;
+  SFSLastNextRowIter fsLastState;
   SFSNextRowIter     fsState;
-  TSDBROW         memRow, imemRow, fsRow;
+  TSDBROW            memRow, imemRow, fsLastRow, fsRow;
 
-  TsdbNextRowState input[3];
+  TsdbNextRowState input[4];
   STsdbReadSnap   *pReadSnap;
   STsdb           *pTsdb;
 } CacheNextRowIter;
 
-static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb) {
+static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema) {
   int code = 0;
 
   tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
@@ -745,12 +921,12 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
 
   STbData *pMem = NULL;
   if (pIter->pReadSnap->pMem) {
-    tsdbGetTbDataFromMemTable(pIter->pReadSnap->pMem, suid, uid, &pMem);
+    pMem = tsdbGetTbDataFromMemTable(pIter->pReadSnap->pMem, suid, uid);
   }
 
   STbData *pIMem = NULL;
   if (pIter->pReadSnap->pIMem) {
-    tsdbGetTbDataFromMemTable(pIter->pReadSnap->pIMem, suid, uid, &pIMem);
+    pIMem = tsdbGetTbDataFromMemTable(pIter->pReadSnap->pIMem, suid, uid);
   }
 
   pIter->pTsdb = pTsdb;
@@ -763,7 +939,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
   if (pDelFile) {
     SDelFReader *pDelFReader;
 
-    code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL);
+    code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb);
     if (code) goto _err;
 
     code = getTableDelIdx(pDelFReader, suid, uid, &delIdx);
@@ -782,6 +958,12 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
 
   pIter->idx = (SBlockIdx){.suid = suid, .uid = uid};
 
+  pIter->fsLastState.state = (SFSLASTNEXTROWSTATES) SFSNEXTROW_FS;
+  pIter->fsLastState.pTsdb = pTsdb;
+  pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet;
+  pIter->fsLastState.pBlockIdxExp = &pIter->idx;
+  pIter->fsLastState.pTSchema = pTSchema;
+
   pIter->fsState.state = SFSNEXTROW_FS;
   pIter->fsState.pTsdb = pTsdb;
   pIter->fsState.aDFileSet = pIter->pReadSnap->fs.aDFileSet;
@@ -789,7 +971,9 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
 
   pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, &pIter->memState, getNextRowFromMem, NULL};
   pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, &pIter->imemState, getNextRowFromMem, NULL};
-  pIter->input[2] =
+  pIter->input[2] = (TsdbNextRowState){&pIter->fsLastRow,     false, true, &pIter->fsLastState, getNextRowFromFSLast,
+                                       clearNextRowFromFSLast};
+  pIter->input[3] =
       (TsdbNextRowState){&pIter->fsRow, false, true, &pIter->fsState, getNextRowFromFS, clearNextRowFromFS};
 
   if (pMem) {
@@ -814,7 +998,7 @@ _err:
 static int32_t nextRowIterClose(CacheNextRowIter *pIter) {
   int code = 0;
 
-  for (int i = 0; i < 3; ++i) {
+  for (int i = 0; i < 4; ++i) {
     if (pIter->input[i].nextRowClearFn) {
       pIter->input[i].nextRowClearFn(pIter->input[i].iter);
     }
@@ -826,7 +1010,6 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) {
 
   tsdbUntakeReadSnap(pIter->pTsdb, pIter->pReadSnap);
 
-  return code;
 _err:
   return code;
 }
@@ -835,7 +1018,7 @@ _err:
 static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) {
   int code = 0;
 
-  for (int i = 0; i < 3; ++i) {
+  for (int i = 0; i < 4; ++i) {
     if (pIter->input[i].next && !pIter->input[i].stop) {
       code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow);
       if (code) goto _err;
@@ -847,18 +1030,18 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) {
     }
   }
 
-  if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop) {
+  if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop && pIter->input[3].stop) {
     *ppRow = NULL;
     return code;
   }
 
-  // select maxpoint(s) from mem, imem, fs
+  // select maxpoint(s) from mem, imem, fs and last
-  TSDBROW *max[3] = {0};
+  TSDBROW *max[4] = {0};
-  int      iMax[3] = {-1, -1, -1};
+  int      iMax[4] = {-1, -1, -1, -1};
   int      nMax = 0;
   TSKEY    maxKey = TSKEY_MIN;
 
-  for (int i = 0; i < 3; ++i) {
+  for (int i = 0; i < 4; ++i) {
     if (!pIter->input[i].stop && pIter->input[i].pRow != NULL) {
       TSDBKEY key = TSDBROW_KEY(pIter->input[i].pRow);
 
@@ -876,13 +1059,13 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) {
   }
 
   // delete detection
-  TSDBROW *merge[3] = {0};
+  TSDBROW *merge[4] = {0};
-  int      iMerge[3] = {-1, -1, -1};
+  int      iMerge[4] = {-1, -1, -1, -1};
   int      nMerge = 0;
   for (int i = 0; i < nMax; ++i) {
-    TSDBKEY maxKey = TSDBROW_KEY(max[i]);
+    TSDBKEY maxKey1 = TSDBROW_KEY(max[i]);
 
-    bool deleted = tsdbKeyDeleted(&maxKey, pIter->pSkyline, &pIter->iSkyline);
+    bool deleted = tsdbKeyDeleted(&maxKey1, pIter->pSkyline, &pIter->iSkyline);
     if (!deleted) {
       iMerge[nMerge] = iMax[i];
       merge[nMerge++] = max[i];
@@ -918,7 +1101,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
   TSKEY lastRowTs = TSKEY_MAX;
 
   CacheNextRowIter iter = {0};
-  nextRowIterOpen(&iter, uid, pTsdb);
+  nextRowIterOpen(&iter, uid, pTsdb, pTSchema);
 
   do {
     TSDBROW *pRow = NULL;
@@ -1015,7 +1198,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
   TSKEY lastRowTs = TSKEY_MAX;
 
   CacheNextRowIter iter = {0};
-  nextRowIterOpen(&iter, uid, pTsdb);
+  nextRowIterOpen(&iter, uid, pTsdb, pTSchema);
 
   do {
     TSDBROW *pRow = NULL;
@@ -1100,8 +1283,11 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH
   //  getTableCacheKeyS(uid, "lr", key, &keyLen);
   getTableCacheKey(uid, 0, key, &keyLen);
   LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen);
-  if (h) {
+  if (!h) {
-  } else {
+    taosThreadMutexLock(&pTsdb->lruMutex);
+
+    h = taosLRUCacheLookup(pCache, key, keyLen);
+    if (!h) {
       STSRow *pRow = NULL;
       bool    dup = false;  // which is always false for now
       code = mergeLastRow(uid, pTsdb, &dup, &pRow);
@@ -1111,7 +1297,10 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH
           taosMemoryFree(pRow);
         }
 
+        taosThreadMutexUnlock(&pTsdb->lruMutex);
+
         *handle = NULL;
+
         return 0;
       }
 
@@ -1122,7 +1311,12 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH
         code = -1;
       }
 
+      taosThreadMutexUnlock(&pTsdb->lruMutex);
+
       h = taosLRUCacheLookup(pCache, key, keyLen);
+    } else {
+      taosThreadMutexUnlock(&pTsdb->lruMutex);
+    }
   }
 
   *handle = h;

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -576,10 +576,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _exit;
     }
-    fSet.pHeadF->nRef = 0;
+    *fSet.pHeadF = *pSet->pHeadF;
-    fSet.pHeadF->commitID = pSet->pHeadF->commitID;
-    fSet.pHeadF->size = pSet->pHeadF->size;
-    fSet.pHeadF->offset = pSet->pHeadF->offset;
 
     // data
     fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile));
@@ -587,9 +584,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _exit;
     }
-    fSet.pDataF->nRef = 0;
+    *fSet.pDataF = *pSet->pDataF;
-    fSet.pDataF->commitID = pSet->pDataF->commitID;
-    fSet.pDataF->size = pSet->pDataF->size;
 
     // data
     fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
@@ -597,9 +592,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _exit;
     }
-    fSet.pLastF->nRef = 0;
+    *fSet.pLastF = *pSet->pLastF;
-    fSet.pLastF->commitID = pSet->pLastF->commitID;
-    fSet.pLastF->size = pSet->pLastF->size;
 
     // last
     fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
@@ -607,9 +600,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _exit;
     }
-    fSet.pSmaF->nRef = 0;
+    *fSet.pSmaF = *pSet->pSmaF;
-    fSet.pSmaF->commitID = pSet->pSmaF->commitID;
-    fSet.pSmaF->size = pSet->pSmaF->size;
 
     if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -58,6 +58,7 @@ int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) {
 
   n += tPutI64v(p ? p + n : p, pLastFile->commitID);
   n += tPutI64v(p ? p + n : p, pLastFile->size);
+  n += tPutI64v(p ? p + n : p, pLastFile->offset);
 
   return n;
 }
@@ -67,6 +68,7 @@ static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) {
 
   n += tGetI64v(p + n, &pLastFile->commitID);
   n += tGetI64v(p + n, &pLastFile->size);
+  n += tGetI64v(p + n, &pLastFile->offset);
 
   return n;
 }
@@ -186,11 +188,16 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
   n += tPutI32v(p ? p + n : p, pSet->diskId.level);
   n += tPutI32v(p ? p + n : p, pSet->diskId.id);
   n += tPutI32v(p ? p + n : p, pSet->fid);
+
+  // data
   n += tPutHeadFile(p ? p + n : p, pSet->pHeadF);
   n += tPutDataFile(p ? p + n : p, pSet->pDataF);
-  n += tPutLastFile(p ? p + n : p, pSet->pLastF);
   n += tPutSmaFile(p ? p + n : p, pSet->pSmaF);
 
+  // last
+  n += tPutU8(p ? p + n : p, 1);  // for future compatibility
+  n += tPutLastFile(p ? p + n : p, pSet->pLastF);
+
   return n;
 }
 
@@ -200,11 +207,17 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
   n += tGetI32v(p + n, &pSet->diskId.level);
   n += tGetI32v(p + n, &pSet->diskId.id);
   n += tGetI32v(p + n, &pSet->fid);
+
+  // data
   n += tGetHeadFile(p + n, pSet->pHeadF);
   n += tGetDataFile(p + n, pSet->pDataF);
-  n += tGetLastFile(p + n, pSet->pLastF);
   n += tGetSmaFile(p + n, pSet->pSmaF);
 
+  // last
+  uint8_t nLast;
+  n += tGetU8(p + n, &nLast);
+  n += tGetLastFile(p + n, pSet->pLastF);
+
   return n;
 }
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -15,6 +15,7 @@
 
 #include "tsdb.h"
 
+#define MEM_MIN_HASH 1024
 #define SL_MAX_LEVEL 5
 
 #define SL_NODE_SIZE(l)        (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
@@ -45,12 +46,12 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) {
   pMemTable->nRef = 1;
   pMemTable->minKey = TSKEY_MAX;
   pMemTable->maxKey = TSKEY_MIN;
-  pMemTable->minVersion = VERSION_MAX;
-  pMemTable->maxVersion = VERSION_MIN;
   pMemTable->nRow = 0;
   pMemTable->nDel = 0;
-  pMemTable->aTbData = taosArrayInit(128, sizeof(STbData *));
+  pMemTable->nTbData = 0;
-  if (pMemTable->aTbData == NULL) {
+  pMemTable->nBucket = MEM_MIN_HASH;
+  pMemTable->aBucket = (STbData **)taosMemoryCalloc(pMemTable->nBucket, sizeof(STbData *));
+  if (pMemTable->aBucket == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     taosMemoryFree(pMemTable);
     goto _err;
@@ -68,37 +69,30 @@ _err:
 void tsdbMemTableDestroy(SMemTable *pMemTable) {
   if (pMemTable) {
     vnodeBufPoolUnRef(pMemTable->pPool);
-    taosArrayDestroy(pMemTable->aTbData);
+    taosMemoryFree(pMemTable->aBucket);
     taosMemoryFree(pMemTable);
   }
 }
 
-static int32_t tbDataPCmprFn(const void *p1, const void *p2) {
+static FORCE_INLINE STbData *tsdbGetTbDataFromMemTableImpl(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid) {
-  STbData *pTbData1 = *(STbData **)p1;
+  STbData *pTbData = pMemTable->aBucket[TABS(uid) % pMemTable->nBucket];
-  STbData *pTbData2 = *(STbData **)p2;
 
-  if (pTbData1->suid < pTbData2->suid) {
+  while (pTbData) {
-    return -1;
+    if (pTbData->uid == uid) break;
-  } else if (pTbData1->suid > pTbData2->suid) {
+    pTbData = pTbData->next;
-    return 1;
   }
 
-  if (pTbData1->uid < pTbData2->uid) {
+  return pTbData;
-    return -1;
-  } else if (pTbData1->uid > pTbData2->uid) {
-    return 1;
-  }
-
-  return 0;
 }
-void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) {
+
-  STbData *pTbData = &(STbData){.suid = suid, .uid = uid};
+STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid) {
+  STbData *pTbData;
 
   taosRLockLatch(&pMemTable->latch);
-  void *p = taosArraySearch(pMemTable->aTbData, &pTbData, tbDataPCmprFn, TD_EQ);
+  pTbData = tsdbGetTbDataFromMemTableImpl(pMemTable, suid, uid);
   taosRUnLockLatch(&pMemTable->latch);
 
-  *ppTbData = p ? *(STbData **)p : NULL;
+  return pTbData;
 }
 
 int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock,
@@ -184,10 +178,6 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
     pTbData->pTail = pDelData;
   }
 
-  // update the state of pMemTable and other (todo)
-
-  pMemTable->minVersion = TMIN(pMemTable->minVersion, version);
-  pMemTable->maxVersion = TMAX(pMemTable->maxVersion, version);
   pMemTable->nDel++;
 
   if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) {
@@ -320,18 +310,44 @@ _exit:
   return pIter->pRow;
 }
 
+static int32_t tsdbMemTableRehash(SMemTable *pMemTable) {
+  int32_t code = 0;
+
+  int32_t   nBucket = pMemTable->nBucket * 2;
+  STbData **aBucket = (STbData **)taosMemoryCalloc(nBucket, sizeof(STbData *));
+  if (aBucket == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+
+  for (int32_t iBucket = 0; iBucket < pMemTable->nBucket; iBucket++) {
+    STbData *pTbData = pMemTable->aBucket[iBucket];
+
+    while (pTbData) {
+      STbData *pNext = pTbData->next;
+
+      int32_t idx = TABS(pTbData->uid) % nBucket;
+      pTbData->next = aBucket[idx];
+      aBucket[idx] = pTbData;
+
+      pTbData = pNext;
+    }
+  }
+
+  taosMemoryFree(pMemTable->aBucket);
+  pMemTable->nBucket = nBucket;
+  pMemTable->aBucket = aBucket;
+
+_exit:
+  return code;
+}
+
 static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) {
   int32_t code = 0;
-  int32_t  idx = 0;
-  STbData *pTbData = NULL;
-  STbData *pTbDataT = &(STbData){.suid = suid, .uid = uid};
 
   // get
-  idx = taosArraySearchIdx(pMemTable->aTbData, &pTbDataT, tbDataPCmprFn, TD_GE);
+  STbData *pTbData = tsdbGetTbDataFromMemTableImpl(pMemTable, suid, uid);
-  if (idx >= 0) {
+  if (pTbData) goto _exit;
-    pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, idx);
-    if (tbDataPCmprFn(&pTbDataT, &pTbData) == 0) goto _exit;
-  }
 
   // create
   SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
@@ -346,9 +362,6 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
   pTbData->uid = uid;
   pTbData->minKey = TSKEY_MAX;
   pTbData->maxKey = TSKEY_MIN;
-  pTbData->minVersion = VERSION_MAX;
-  pTbData->maxVersion = VERSION_MIN;
-  pTbData->maxSkmVer = -1;
   pTbData->pHead = NULL;
   pTbData->pTail = NULL;
   pTbData->sl.seed = taosRand();
@@ -367,21 +380,23 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
     SL_NODE_FORWARD(pTbData->sl.pTail, iLevel) = NULL;
   }
 
-  void *p;
-  if (idx < 0) {
-    idx = taosArrayGetSize(pMemTable->aTbData);
-  }
-
   taosWLockLatch(&pMemTable->latch);
-  p = taosArrayInsert(pMemTable->aTbData, idx, &pTbData);
+
+  if (pMemTable->nTbData >= pMemTable->nBucket) {
+    code = tsdbMemTableRehash(pMemTable);
+    if (code) {
       taosWUnLockLatch(&pMemTable->latch);
-
-  tsdbDebug("vgId:%d, add table data %p at idx:%d", TD_VID(pMemTable->pTsdb->pVnode), pTbData, idx);
-
-  if (p == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
       goto _err;
     }
+  }
+
+  int32_t idx = TABS(uid) % pMemTable->nBucket;
+  pTbData->next = pMemTable->aBucket[idx];
+  pMemTable->aBucket[idx] = pTbData;
+  pMemTable->nTbData++;
+
+  taosWUnLockLatch(&pMemTable->latch);
+
 _exit:
   *ppTbData = pTbData;
   return code;
@@ -591,15 +606,9 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i
     tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb);
   }
 
-  pTbData->minVersion = TMIN(pTbData->minVersion, version);
-  pTbData->maxVersion = TMAX(pTbData->maxVersion, version);
-  pTbData->maxSkmVer = TMAX(pTbData->maxSkmVer, pMsgIter->sversion);
-
   // SMemTable
   pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey);
   pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey);
-  pMemTable->minVersion = TMIN(pMemTable->minVersion, pTbData->minVersion);
-  pMemTable->maxVersion = TMAX(pMemTable->maxVersion, pTbData->maxVersion);
   pMemTable->nRow += nRow;
 
   pRsp->numOfRows = nRow;
@@ -624,3 +633,41 @@ void tsdbUnrefMemTable(SMemTable *pMemTable) {
     tsdbMemTableDestroy(pMemTable);
   }
 }
+
+static FORCE_INLINE int32_t tbDataPCmprFn(const void *p1, const void *p2) {
+  STbData *pTbData1 = *(STbData **)p1;
+  STbData *pTbData2 = *(STbData **)p2;
+
+  if (pTbData1->suid < pTbData2->suid) {
+    return -1;
+  } else if (pTbData1->suid > pTbData2->suid) {
+    return 1;
+  }
+
+  if (pTbData1->uid < pTbData2->uid) {
+    return -1;
+  } else if (pTbData1->uid > pTbData2->uid) {
+    return 1;
+  }
+
+  return 0;
+}
+
+SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) {
+  SArray *aTbDataP = taosArrayInit(pMemTable->nTbData, sizeof(STbData *));
+  if (aTbDataP == NULL) goto _exit;
+
+  for (int32_t iBucket = 0; iBucket < pMemTable->nBucket; iBucket++) {
+    STbData *pTbData = pMemTable->aBucket[iBucket];
+
+    while (pTbData) {
+      taosArrayPush(aTbDataP, &pTbData);
+      pTbData = pTbData->next;
+    }
+  }
+
+  taosArraySort(aTbDataP, tbDataPCmprFn);
+
+_exit:
+  return aTbDataP;
+}

source/dnode/vnode/src/tsdb/tsdbOpen.c

@@ -86,7 +86,7 @@ int tsdbClose(STsdb **pTsdb) {
   if (*pTsdb) {
     taosThreadRwlockDestroy(&(*pTsdb)->rwLock);
     tsdbFSClose(*pTsdb);
-    tsdbCloseCache((*pTsdb)->lruCache);
+    tsdbCloseCache(*pTsdb);
     taosMemoryFreeClear(*pTsdb);
   }
   return 0;

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -27,8 +27,12 @@ struct STsdbSnapReader {
   int32_t       fid;
   SDataFReader* pDataFReader;
   SArray*       aBlockIdx;  // SArray<SBlockIdx>
-  int32_t       iBlockIdx;
+  SArray*       aBlockL;    // SArray<SBlockL>
   SBlockIdx*    pBlockIdx;
+  SBlockL*      pBlockL;
+
+  int32_t    iBlockIdx;
+  int32_t    iBlockL;
   SMapData   mBlock;  // SMapData<SBlock>
   int32_t    iBlock;
   SBlockData oBlockData;
@@ -47,115 +51,117 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
 
   while (true) {
     if (pReader->pDataFReader == NULL) {
-      SDFileSet* pSet =
+      // next
-          taosArraySearch(pReader->fs.aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSetCmprFn, TD_GT);
+      SDFileSet  dFileSet = {.fid = pReader->fid};
-
+      SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT);
       if (pSet == NULL) goto _exit;
-
       pReader->fid = pSet->fid;
-      code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet);
+
+      // load
+      code = tsdbDataFReaderOpen(&pReader->pDataFReader, pTsdb, pSet);
       if (code) goto _err;
 
-      // SBlockIdx
+      code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx);
-      code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx, NULL);
       if (code) goto _err;
 
+      code = tsdbReadBlockL(pReader->pDataFReader, pReader->aBlockL);
+      if (code) goto _err;
+
+      // init
       pReader->iBlockIdx = 0;
+      if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
+        pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+
+        code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock);
+        if (code) goto _err;
+
+        pReader->iBlock = 0;
+      } else {
         pReader->pBlockIdx = NULL;
+      }
+
+      pReader->iBlockL = 0;
+      while (true) {
+        if (pReader->iBlockL >= taosArrayGetSize(pReader->aBlockL)) {
+          pReader->pBlockL = NULL;
+          break;
+        }
+
+        pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL);
+        if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) {
+          // TODO
+          break;
+        }
+
+        pReader->iBlockL++;
+      }
 
       tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path,
                pReader->fid);
     }
 
     while (true) {
-      if (pReader->pBlockIdx == NULL) {
+      if (pReader->pBlockIdx && pReader->pBlockL) {
-        if (pReader->iBlockIdx >= taosArrayGetSize(pReader->aBlockIdx)) {
+        TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid};
-          tsdbDataFReaderClose(&pReader->pDataFReader);
+
-          break;
+        ASSERT(0);
+
+        // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) {
+        //   // TODO
+        // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) {
+        //   // TODO
+        // } else {
+        //   // TODO
+        // }
+      } else if (pReader->pBlockIdx) {
+        while (pReader->iBlock < pReader->mBlock.nItem) {
+          SBlock block;
+          tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock);
+
+          if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) {
+            // load data (todo)
           }
 
-        pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+          // next
-        pReader->iBlockIdx++;
+          pReader->iBlock++;
+          if (*ppData) break;
+        }
 
-        code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock, NULL);
+        if (pReader->iBlock >= pReader->mBlock.nItem) {
+          pReader->iBlockIdx++;
+          if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
+            pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+
+            code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock);
             if (code) goto _err;
 
             pReader->iBlock = 0;
-      }
+          } else {
-
-      SBlock  block;
-      SBlock* pBlock = &block;
-      while (true) {
-        if (pReader->iBlock >= pReader->mBlock.nItem) {
             pReader->pBlockIdx = NULL;
+          }
+        }
+
+        if (*ppData) goto _exit;
+      } else if (pReader->pBlockL) {
+        while (pReader->pBlockL) {
+          if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) {
+            // load data (todo)
+          }
+
+          // next
+          pReader->iBlockL++;
+          if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) {
+            pReader->pBlockL = (SBlockL*)taosArrayGetSize(pReader->aBlockL);
+          } else {
+            pReader->pBlockL = NULL;
+          }
+
+          if (*ppData) goto _exit;
+        }
+      } else {
+        tsdbDataFReaderClose(&pReader->pDataFReader);
         break;
       }
-
-        tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock);
-        pReader->iBlock++;
-
-        if (pBlock->minVersion > pReader->ever || pBlock->maxVersion < pReader->sver) continue;
-
-        code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->oBlockData, NULL, NULL);
-        if (code) goto _err;
-
-        // filter
-        tBlockDataReset(&pReader->nBlockData);
-        for (int32_t iColData = 0; iColData < taosArrayGetSize(pReader->oBlockData.aIdx); iColData++) {
-          SColData* pColDataO = tBlockDataGetColDataByIdx(&pReader->oBlockData, iColData);
-          SColData* pColDataN = NULL;
-
-          code = tBlockDataAddColData(&pReader->nBlockData, taosArrayGetSize(pReader->nBlockData.aIdx), &pColDataN);
-          if (code) goto _err;
-
-          tColDataInit(pColDataN, pColDataO->cid, pColDataO->type, pColDataO->smaOn);
-        }
-
-        for (int32_t iRow = 0; iRow < pReader->oBlockData.nRow; iRow++) {
-          TSDBROW row = tsdbRowFromBlockData(&pReader->oBlockData, iRow);
-          int64_t version = TSDBROW_VERSION(&row);
-
-          tsdbTrace("vgId:%d, vnode snapshot tsdb read for %s, %" PRId64 "(%" PRId64 " , %" PRId64 ")",
-                    TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path, version, pReader->sver, pReader->ever);
-
-          if (version < pReader->sver || version > pReader->ever) continue;
-
-          code = tBlockDataAppendRow(&pReader->nBlockData, &row, NULL);
-          if (code) goto _err;
-        }
-
-        if (pReader->nBlockData.nRow <= 0) {
-          continue;
-        }
-
-        // org data
-        // compress data (todo)
-        int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData);
-
-        *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size);
-        if (*ppData == NULL) {
-          code = TSDB_CODE_OUT_OF_MEMORY;
-          goto _err;
-        }
-
-        SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
-        pHdr->type = pReader->type;
-        pHdr->size = size;
-
-        TABLEID* pId = (TABLEID*)(&pHdr[1]);
-        pId->suid = pReader->pBlockIdx->suid;
-        pId->uid = pReader->pBlockIdx->uid;
-
-        tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData);
-
-        tsdbInfo("vgId:%d, vnode snapshot read data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64
-                 " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d",
-                 TD_VID(pTsdb->pVnode), pTsdb->path, pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid,
-                 pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow,
-                 size);
-
-        goto _exit;
-      }
     }
   }
 
@@ -179,11 +185,11 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
     }
 
     // open
-    code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb, NULL);
+    code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb);
     if (code) goto _err;
 
     // read index
-    code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx, NULL);
+    code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx);
     if (code) goto _err;
 
     pReader->iDelIdx = 0;
@@ -199,7 +205,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
 
     pReader->iDelIdx++;
 
-    code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL);
+    code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData);
     if (code) goto _err;
 
     int32_t size = 0;
@@ -292,10 +298,15 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
+  pReader->aBlockL = taosArrayInit(0, sizeof(SBlockL));
+  if (pReader->aBlockL == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
   pReader->mBlock = tMapDataInit();
-  code = tBlockDataInit(&pReader->oBlockData);
+  code = tBlockDataCreate(&pReader->oBlockData);
   if (code) goto _err;
-  code = tBlockDataInit(&pReader->nBlockData);
+  code = tBlockDataCreate(&pReader->nBlockData);
   if (code) goto _err;
 
   pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
@@ -327,10 +338,11 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
   if (pReader->pDataFReader) {
     tsdbDataFReaderClose(&pReader->pDataFReader);
   }
+  taosArrayDestroy(pReader->aBlockL);
   taosArrayDestroy(pReader->aBlockIdx);
   tMapDataClear(&pReader->mBlock);
-  tBlockDataClear(&pReader->oBlockData, 1);
+  tBlockDataDestroy(&pReader->oBlockData, 1);
-  tBlockDataClear(&pReader->nBlockData, 1);
+  tBlockDataDestroy(&pReader->nBlockData, 1);
 
   if (pReader->pDelFReader) {
     tsdbDelFReaderClose(&pReader->pDelFReader);
@@ -405,6 +417,7 @@ struct STsdbSnapWriter {
   int8_t  cmprAlg;
   int64_t commitID;
 
+  uint8_t* aBuf[5];
   // for data file
   SBlockData bData;
 
@@ -418,6 +431,9 @@ struct STsdbSnapWriter {
   SBlockData*   pBlockData;
   int32_t       iRow;
   SBlockData    bDataR;
+  SArray*       aBlockL;  // SArray<SBlockL>
+  int32_t       iBlockL;
+  SBlockData    lDataR;
 
   SDataFWriter* pDataFWriter;
   SBlockIdx*    pBlockIdxW;  // NULL when no committing table
@@ -427,6 +443,7 @@ struct STsdbSnapWriter {
 
   SMapData mBlockW;     // SMapData<SBlock>
   SArray*  aBlockIdxW;  // SArray<SBlockIdx>
+  SArray*  aBlockLW;    // SArray<SBlockL>
 
   // for del file
   SDelFReader* pDelFReader;
@@ -437,25 +454,6 @@ struct STsdbSnapWriter {
   SArray*      aDelIdxW;
 };
 
-static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
-  int32_t     code = 0;
-  int32_t     iRow = 0;           // todo
-  int32_t     nRow = 0;           // todo
-  SBlockData* pBlockData = NULL;  // todo
-
-  while (iRow < nRow) {
-    code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
-    if (code) goto _err;
-  }
-
-  return code;
-
-_err:
-  tsdbError("vgId:%d, tsdb snapshot write append data for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
-            pWriter->pTsdb->path, tstrerror(code));
-  return code;
-}
-
 static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
   int32_t code = 0;
 
@@ -467,20 +465,21 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
   if (pWriter->pBlockData) {
     ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
     while (pWriter->iRow < pWriter->pBlockData->nRow) {
-      code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL);
+      code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL,
+                                 0);  // todo
       if (code) goto _err;
 
       if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
-        pWriter->blockW.last = 0;
+        // pWriter->blockW.last = 0;
-        code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+        // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
-                                  &pWriter->blockW, pWriter->cmprAlg);
+        //                           &pWriter->blockW, pWriter->cmprAlg);
         if (code) goto _err;
 
         code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
         if (code) goto _err;
 
         tBlockReset(&pWriter->blockW);
-        tBlockDataClearData(&pWriter->bDataW);
+        tBlockDataClear(&pWriter->bDataW);
       }
 
       pWriter->iRow++;
@@ -489,16 +488,16 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
 
   // write remain data if has
   if (pWriter->bDataW.nRow > 0) {
-    pWriter->blockW.last = 0;
+    // pWriter->blockW.last = 0;
     if (pWriter->bDataW.nRow < pWriter->minRow) {
       if (pWriter->iBlock > pWriter->mBlock.nItem) {
-        pWriter->blockW.last = 1;
+        // pWriter->blockW.last = 1;
       }
     }
 
-    code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+    // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
-                              &pWriter->blockW, pWriter->cmprAlg);
+    //                           &pWriter->blockW, pWriter->cmprAlg);
-    if (code) goto _err;
+    // if (code) goto _err;
 
     code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
     if (code) goto _err;
@@ -510,16 +509,16 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
     SBlock block;
     tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
 
-    if (block.last) {
+    // if (block.last) {
-      code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
+    //   code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
-      if (code) goto _err;
+    //   if (code) goto _err;
 
-      tBlockReset(&block);
+    //   tBlockReset(&block);
-      block.last = 1;
+    //   block.last = 1;
-      code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block,
+    //   code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block,
-                                pWriter->cmprAlg);
+    //                             pWriter->cmprAlg);
-      if (code) goto _err;
+    //   if (code) goto _err;
-    }
+    // }
 
     code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
     if (code) goto _err;
@@ -528,8 +527,8 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
   }
 
   // SBlock
-  code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW);
+  // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW);
-  if (code) goto _err;
+  // if (code) goto _err;
 
   // SBlockIdx
   if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) {
@@ -550,7 +549,7 @@ _err:
 static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) {
   int32_t code = 0;
 
-  code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL);
+  code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock);
   if (code) goto _err;
 
   // SBlockData
@@ -559,16 +558,17 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p
   for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) {
     tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock);
 
-    if (block.last) {
+    // if (block.last) {
-      code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
+    //   code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
-      if (code) goto _err;
+    //   if (code) goto _err;
 
-      tBlockReset(&block);
+    //   tBlockReset(&block);
-      block.last = 1;
+    //   block.last = 1;
-      code =
+    //   code =
-          tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, pWriter->cmprAlg);
+    //       tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block,
-      if (code) goto _err;
+    //       pWriter->cmprAlg);
-    }
+    //   if (code) goto _err;
+    // }
 
     code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
     if (code) goto _err;
@@ -576,7 +576,7 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p
 
   // SBlock
   SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
-  code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx);
+  code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx);
   if (code) goto _err;
 
   // SBlockIdx
@@ -601,9 +601,9 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
   TSDBROW     row;
   TSDBROW*    pRow = &row;
 
-  // correct schema
+  // // correct schema
-  code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData);
+  // code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData);
-  if (code) goto _err;
+  // if (code) goto _err;
 
   // loop to merge
   *pRow = tsdbRowFromBlockData(pBlockData, iRow);
@@ -618,8 +618,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
       ASSERT(c);
 
       if (c < 0) {
-        code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
+        // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
-        if (code) goto _err;
+        // if (code) goto _err;
 
         iRow++;
         if (iRow < pWriter->pBlockData->nRow) {
@@ -628,8 +628,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
           pRow = NULL;
         }
       } else if (c > 0) {
-        code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL);
+        // code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow),
-        if (code) goto _err;
+        // NULL); if (code) goto _err;
 
         pWriter->iRow++;
         if (pWriter->iRow >= pWriter->pBlockData->nRow) {
@@ -647,16 +647,15 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
 
         tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
 
-        if (block.last) {
+        // if (block.last) {
-          pWriter->pBlockData = &pWriter->bDataR;
+        //   pWriter->pBlockData = &pWriter->bDataR;
 
-          code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL);
+        //   code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL,
-          if (code) goto _err;
+        //   NULL); if (code) goto _err; pWriter->iRow = 0;
-          pWriter->iRow = 0;
 
-          pWriter->iBlock++;
+        //   pWriter->iBlock++;
-          break;
+        //   break;
-        }
+        // }
 
         c = tsdbKeyCmprFn(&block.maxKey, &key);
 
@@ -664,16 +663,16 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
 
         if (c < 0) {
           if (pWriter->bDataW.nRow) {
-            pWriter->blockW.last = 0;
+            // pWriter->blockW.last = 0;
-            code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+            // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
-                                      &pWriter->blockW, pWriter->cmprAlg);
+            //                           &pWriter->blockW, pWriter->cmprAlg);
-            if (code) goto _err;
+            // if (code) goto _err;
 
             code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
             if (code) goto _err;
 
             tBlockReset(&pWriter->blockW);
-            tBlockDataClearData(&pWriter->bDataW);
+            tBlockDataClear(&pWriter->bDataW);
           }
 
           code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
@@ -687,9 +686,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
 
           if (c > 0) {
             pWriter->pBlockData = &pWriter->bDataR;
-            code =
+            // code =
-                tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL);
+            //     tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL,
-            if (code) goto _err;
+            //     NULL);
+            // if (code) goto _err;
             pWriter->iRow = 0;
 
             pWriter->iBlock++;
@@ -700,8 +700,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
 
       if (pWriter->pBlockData) continue;
 
-      code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
+      // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
-      if (code) goto _err;
+      // if (code) goto _err;
 
       iRow++;
       if (iRow < pBlockData->nRow) {
@@ -715,15 +715,15 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
     if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue;
 
   _write_block:
-    code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+    // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
-                              &pWriter->blockW, pWriter->cmprAlg);
+    //                           &pWriter->blockW, pWriter->cmprAlg);
-    if (code) goto _err;
+    // if (code) goto _err;
 
     code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
     if (code) goto _err;
 
     tBlockReset(&pWriter->blockW);
-    tBlockDataClearData(&pWriter->bDataW);
+    tBlockDataClear(&pWriter->bDataW);
   }
 
   return code;
@@ -789,7 +789,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) {
     }
 
     if (pWriter->pBlockIdx) {
-      code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL);
+      code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock);
       if (code) goto _err;
     } else {
       tMapDataReset(&pWriter->mBlock);
@@ -831,9 +831,11 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
 
   if (pWriter->pDataFWriter == NULL) goto _exit;
 
+  // finish current table
   code = tsdbSnapWriteTableDataEnd(pWriter);
   if (code) goto _err;
 
+  // move remain table
   while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
     code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx));
     if (code) goto _err;
@@ -841,8 +843,16 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
     pWriter->iBlockIdx++;
   }
 
-  code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL);
+  // write remain stuff
+  if (taosArrayGetSize(pWriter->aBlockLW) > 0) {
+    code = tsdbWriteBlockL(pWriter->pDataFWriter, pWriter->aBlockIdxW);
     if (code) goto _err;
+  }
+
+  if (taosArrayGetSize(pWriter->aBlockIdx) > 0) {
+    code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW);
+    if (code) goto _err;
+  }
 
   code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet);
   if (code) goto _err;
@@ -868,17 +878,20 @@ _err:
 static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t       code = 0;
   STsdb*        pTsdb = pWriter->pTsdb;
+  SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
   TABLEID       id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
   int64_t       n;
 
   // decode
   SBlockData* pBlockData = &pWriter->bData;
-  n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData);
+  code = tDecmprBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pHdr->size - sizeof(TABLEID), pBlockData,
-  ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData);
+                          pWriter->aBuf);
+  if (code) goto _err;
 
   // open file
-  TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData);
+  TSDBKEY keyFirst = {.version = pBlockData->aVersion[0], .ts = pBlockData->aTSKEY[0]};
-  TSDBKEY keyLast = tBlockDataLastKey(pBlockData);
+  TSDBKEY keyLast = {.version = pBlockData->aVersion[pBlockData->nRow - 1],
+                     .ts = pBlockData->aTSKEY[pBlockData->nRow - 1]};
 
   int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision);
   ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision));
@@ -895,11 +908,15 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
       code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
       if (code) goto _err;
 
-      code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL);
+      code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx);
+      if (code) goto _err;
+
+      code = tsdbReadBlockL(pWriter->pDataFReader, pWriter->aBlockL);
       if (code) goto _err;
     } else {
       ASSERT(pWriter->pDataFReader == NULL);
       taosArrayClear(pWriter->aBlockIdx);
+      taosArrayClear(pWriter->aBlockL);
     }
     pWriter->iBlockIdx = 0;
     pWriter->pBlockIdx = NULL;
@@ -907,7 +924,9 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     pWriter->iBlock = 0;
     pWriter->pBlockData = NULL;
     pWriter->iRow = 0;
+    pWriter->iBlockL = 0;
     tBlockDataReset(&pWriter->bDataR);
+    tBlockDataReset(&pWriter->lDataR);
 
     // write
     SHeadFile fHead;
@@ -928,7 +947,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
       wSet.fid = fid;
       fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
       fData = (SDataFile){.commitID = pWriter->commitID, .size = 0};
-      fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0};
+      fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0, .offset = 0};
       fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0};
     }
 
@@ -936,6 +955,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     if (code) goto _err;
 
     taosArrayClear(pWriter->aBlockIdxW);
+    taosArrayClear(pWriter->aBlockLW);
     tMapDataReset(&pWriter->mBlockW);
     pWriter->pBlockIdxW = NULL;
     tBlockDataReset(&pWriter->bDataW);
@@ -963,10 +983,10 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
 
     // reader
     if (pDelFile) {
-      code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL);
+      code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb);
       if (code) goto _err;
 
-      code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR, NULL);
+      code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR);
       if (code) goto _err;
     }
 
@@ -980,52 +1000,16 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
   TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
 
   while (true) {
-    SDelIdx* pDelIdx = NULL;
+    if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break;
-    int64_t  n = sizeof(SSnapDataHdr) + sizeof(TABLEID);
+    if (tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) >= 0) break;
-    SDelData delData;
-    SDelIdx  delIdx;
-    int8_t   toBreak = 0;
 
-    if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR)) {
+    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
-      pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
-    }
 
-    if (pDelIdx) {
+    code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
-      int32_t c = tTABLEIDCmprFn(&id, pDelIdx);
-      if (c < 0) {
-        goto _new_del;
-      } else {
-        code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL);
     if (code) goto _err;
 
-        pWriter->iDelIdx++;
+    SDelIdx delIdx = *pDelIdx;
-        if (c == 0) {
+    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
-          toBreak = 1;
-          delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
-          goto _merge_del;
-        } else {
-          delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid};
-          goto _write_del;
-        }
-      }
-    }
-
-  _new_del:
-    toBreak = 1;
-    delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
-    taosArrayClear(pWriter->aDelData);
-
-  _merge_del:
-    while (n < nData) {
-      n += tGetDelData(pData + n, &delData);
-      if (taosArrayPush(pWriter->aDelData, &delData) == NULL) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _err;
-      }
-    }
-
-  _write_del:
-    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
     if (code) goto _err;
 
     if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) {
@@ -1033,7 +1017,40 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
       goto _err;
     }
 
-    if (toBreak) break;
+    pWriter->iDelIdx++;
+  }
+
+  if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR) &&
+      tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) == 0) {
+    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
+
+    code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
+    if (code) goto _err;
+
+    pWriter->iDelIdx++;
+  } else {
+    taosArrayClear(pWriter->aDelData);
+  }
+
+  int64_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID);
+  while (n < nData) {
+    SDelData delData;
+
+    n += tGetDelData(pData + n, &delData);
+
+    if (taosArrayPush(pWriter->aDelData, &delData) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+  }
+
+  SDelIdx delIdx = {.suid = id.suid, .uid = id.uid};
+  code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
+  if (code) goto _err;
+
+  if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
   }
 
 _exit:
@@ -1054,11 +1071,11 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
   for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) {
     SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
 
-    code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL);
+    code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
     if (code) goto _err;
 
-    SDelIdx delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid};
+    SDelIdx delIdx = *pDelIdx;
-    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
+    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
     if (code) goto _err;
 
     if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) {
@@ -1117,7 +1134,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
   pWriter->commitID = pTsdb->pVnode->state.commitID;
 
   // for data file
-  code = tBlockDataInit(&pWriter->bData);
+  code = tBlockDataCreate(&pWriter->bData);
 
   if (code) goto _err;
   pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
@@ -1125,17 +1142,29 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
-  code = tBlockDataInit(&pWriter->bDataR);
+  code = tBlockDataCreate(&pWriter->bDataR);
   if (code) goto _err;
 
+  pWriter->aBlockL = taosArrayInit(0, sizeof(SBlockL));
+  if (pWriter->aBlockL == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
   pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx));
   if (pWriter->aBlockIdxW == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
-  code = tBlockDataInit(&pWriter->bDataW);
+  code = tBlockDataCreate(&pWriter->bDataW);
   if (code) goto _err;
 
+  pWriter->aBlockLW = taosArrayInit(0, sizeof(SBlockL));
+  if (pWriter->aBlockLW == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
   // for del file
   pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx));
   if (pWriter->aDelIdxR == NULL) {
@@ -1186,6 +1215,10 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
     if (code) goto _err;
   }
 
+  for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) {
+    tFree(pWriter->aBuf[iBuf]);
+  }
+
   tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
   taosMemoryFree(pWriter);
   *ppWriter = NULL;
@@ -1224,6 +1257,7 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
 
 _exit:
   tsdbDebug("vgId:%d, tsdb snapshot write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
+
   return code;
 
 _err:

source/dnode/vnode/src/vnd/vnodeBufPool.c

@@ -78,7 +78,7 @@ void vnodeBufPoolReset(SVBufPool *pPool) {
 void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
   SVBufPoolNode *pNode;
   void          *p;
-
+  taosThreadSpinLock(&pPool->lock);
   if (pPool->node.size >= pPool->ptr - pPool->node.data + size) {
     // allocate from the anchor node
     p = pPool->ptr;
@@ -89,6 +89,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
     pNode = taosMemoryMalloc(sizeof(*pNode) + size);
     if (pNode == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
+      taosThreadSpinUnlock(&pPool->lock);
       return NULL;
     }
 
@@ -101,7 +102,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
 
     pPool->size = pPool->size + sizeof(*pNode) + size;
   }
-
+  taosThreadSpinUnlock(&pPool->lock);
   return p;
 }
 
@@ -129,6 +130,12 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool)
     return -1;
   }
 
+  if (taosThreadSpinInit(&pPool->lock, 0) != 0) {
+    taosMemoryFree(pPool);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
   pPool->next = NULL;
   pPool->pVnode = pVnode;
   pPool->nRef = 0;
@@ -145,6 +152,7 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool)
 
 static int vnodeBufPoolDestroy(SVBufPool *pPool) {
   vnodeBufPoolReset(pPool);
+  taosThreadSpinDestroy(&pPool->lock);
   taosMemoryFree(pPool);
   return 0;
 }

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -220,9 +220,6 @@ int vnodeCommit(SVnode *pVnode) {
   vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID,
         pVnode->state.applied);
 
-  vnodeBufPoolUnRef(pVnode->inUse);
-  pVnode->inUse = NULL;
-
   pVnode->state.commitTerm = pVnode->state.applyTerm;
 
   // save info
@@ -239,7 +236,13 @@ int vnodeCommit(SVnode *pVnode) {
 
   // preCommit
   // smaSyncPreCommit(pVnode->pSma);
-  smaAsyncPreCommit(pVnode->pSma);
+  if(smaAsyncPreCommit(pVnode->pSma) < 0){
+    ASSERT(0);
+    return -1;
+  }
+
+  vnodeBufPoolUnRef(pVnode->inUse);
+  pVnode->inUse = NULL;
 
   // commit each sub-system
   if (metaCommit(pVnode->pMeta) < 0) {
@@ -248,7 +251,10 @@ int vnodeCommit(SVnode *pVnode) {
   }
 
   if (VND_IS_RSMA(pVnode)) {
-    smaAsyncCommit(pVnode->pSma);
+    if (smaAsyncCommit(pVnode->pSma) < 0) {
+      ASSERT(0);
+      return -1;
+    }
 
     if (tsdbCommit(VND_RSMA0(pVnode)) < 0) {
       ASSERT(0);
@@ -285,7 +291,10 @@ int vnodeCommit(SVnode *pVnode) {
 
   // postCommit
   // smaSyncPostCommit(pVnode->pSma);
-  smaAsyncPostCommit(pVnode->pSma);
+  if (smaAsyncPostCommit(pVnode->pSma) < 0) {
+    ASSERT(0);
+    return -1;
+  }
 
   // apply the commit (TODO)
   walEndSnapshot(pVnode->pWal);

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -87,6 +87,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   pVnode->msgCb = msgCb;
   taosThreadMutexInit(&pVnode->lock, NULL);
   pVnode->blocked = false;
+  pVnode->inClose = false;
 
   tsem_init(&pVnode->syncSem, 0, 0);
   tsem_init(&(pVnode->canCommit), 0, 1);
@@ -181,6 +182,8 @@ _err:
 void vnodePreClose(SVnode *pVnode) {
   if (pVnode) {
     syncLeaderTransfer(pVnode->sync);
+    pVnode->inClose = true;
+    smaPreClose(pVnode->pSma);
   }
 }
 

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -424,6 +424,25 @@ int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t vnodeGetStbIdList(SVnode* pVnode, int64_t suid, SArray* list) {
+  SMStbCursor* pCur = metaOpenStbCursor(pVnode->pMeta, suid);
+  if (!pCur) {
+    return TSDB_CODE_FAILED;
+  }
+
+  while (1) {
+    tb_uid_t id = metaStbCursorNext(pCur);
+    if (id == 0) {
+      break;
+    }
+
+    taosArrayPush(list, &id);
+  }
+
+  metaCloseStbCursor(pCur);
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
   SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid);
   if (!pCur) {

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -301,8 +301,8 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
       return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg, 0);
     case TDMT_SCH_QUERY_CONTINUE:
       return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg, 0);
-    case TDMT_VND_FETCH_RSMA:
+    case TDMT_VND_EXEC_RSMA:
-      return smaProcessFetch(pVnode->pSma, pMsg);
+      return smaProcessExec(pVnode->pSma, pMsg);
     default:
       vError("unknown msg type:%d in query queue", pMsg->msgType);
       return TSDB_CODE_VND_APP_ERROR;
@@ -530,7 +530,9 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR
   }
 
   tqUpdateTbUidList(pVnode->pTq, tbUids, true);
-  tdUpdateTbUidList(pVnode->pSma, pStore);
+  if (tdUpdateTbUidList(pVnode->pSma, pStore) < 0) {
+    goto _exit;
+  }
   tdUidStoreFree(pStore);
 
   // prepare rsp

source/libs/executor/inc/executorimpl.h

@@ -205,7 +205,7 @@ typedef struct SExprSupp {
 } SExprSupp;
 
 typedef struct SOperatorInfo {
-  uint8_t                operatorType;
+  uint16_t               operatorType;
   bool                   blocking;  // block operator or not
   uint8_t                status;    // denote if current operator is completed
   char*                  name;      // name, for debug purpose
@@ -434,7 +434,7 @@ typedef struct SStreamAggSupporter {
 typedef struct SessionWindowSupporter {
   SStreamAggSupporter* pStreamAggSup;
   int64_t              gap;
-  uint8_t              parentType;
+  uint16_t             parentType;
   SAggSupporter*       pIntervalAggSup;
 } SessionWindowSupporter;
 
@@ -924,9 +924,6 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
 
 SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                      SExecTaskInfo* pTaskInfo, int32_t numOfChild);
-SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
-                                                SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
-                                                STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
                                             SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,

source/libs/executor/inc/tsimplehash.h

@@ -17,7 +17,6 @@
 #define TDENGINE_TSIMPLEHASH_H
 
 #include "tarray.h"
-#include "tlockfree.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -27,6 +26,10 @@ typedef uint32_t (*_hash_fn_t)(const char *, uint32_t);
 typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
 typedef void (*_hash_free_fn_t)(void *);
 
+/**
+ * @brief single thread hash
+ * 
+ */
 typedef struct SSHashObj SSHashObj;
 
 /**
@@ -36,7 +39,7 @@ typedef struct SSHashObj SSHashObj;
  * @param fn          hash function to generate the hash value
  * @return
  */
-SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen);
+SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn);
 
 /**
  * return the size of hash table
@@ -48,22 +51,26 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj);
 int32_t tSimpleHashPrint(const SSHashObj *pHashObj);
 
 /**
- * put element into hash table, if the element with the same key exists, update it
+ * @brief put element into hash table, if the element with the same key exists, update it
+ * 
  * @param pHashObj 
  * @param key 
+ * @param keyLen 
  * @param data 
- * @return
+ * @param dataLen 
+ * @return int32_t 
  */
-int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data);
+int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t dataLen);
 
 /**
  * return the payload data with the specified key
  *
  * @param pHashObj
  * @param key
+ * @param keyLen
  * @return
  */
-void *tSimpleHashGet(SSHashObj *pHashObj, const void *key);
+void *tSimpleHashGet(SSHashObj *pHashObj, const void *key, size_t keyLen);
 
 /**
  * remove item with the specified key
@@ -71,7 +78,7 @@ void *tSimpleHashGet(SSHashObj *pHashObj, const void *key);
  * @param key
  * @param keyLen
  */
-int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key);
+int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen);
 
 /**
  * Clear the hash table.
@@ -98,7 +105,7 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj);
  * @param keyLen
  * @return
  */
-void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen);
+void *tSimpleHashGetKey(void *data, size_t* keyLen);
 
 /**
  * Create the hash table iterator
@@ -109,6 +116,17 @@ void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen);
  */
 void *tSimpleHashIterate(const SSHashObj *pHashObj, void *data, int32_t *iter);
 
+/**
+ * Create the hash table iterator
+ * 
+ * @param pHashObj 
+ * @param data 
+ * @param key 
+ * @param iter 
+ * @return void* 
+ */
+void *tSimpleHashIterateKV(const SSHashObj *pHashObj, void *data, void **key, int32_t *iter);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/executor/src/executil.c

@@ -429,7 +429,9 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray
   for (int32_t i = 0; i < rows; i++) {
     int64_t* uid = taosArrayGet(uidList, i);
     void* tag = taosHashGet(tags, uid, sizeof(int64_t));
+    if (suid != 0) {
       ASSERT(tag);
+    }
     for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){
       SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j);
 
@@ -533,8 +535,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
       vnodeGetCtbIdList(pVnode, pScanNode->suid, res);
     }
   } else {  // Create one table group.
+    if(metaIsTableExist(metaHandle, tableUid)){
       taosArrayPush(res, &tableUid);
     }
+  }
 
   if (pTagCond) {
     SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond);
@@ -599,7 +603,10 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) {
 int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) {
   SMetaReader mr = {0};
   metaReaderInit(&mr, pMeta, 0);
-  metaGetTableEntryByUid(&mr, uid);
+  if(metaGetTableEntryByUid(&mr, uid) != 0){    // table not exist
+    metaReaderClear(&mr);
+    return TSDB_CODE_PAR_TABLE_NOT_EXIST;
+  }
 
   SNodeList* groupNew = nodesCloneList(pGroupNode);
 

source/libs/executor/src/executor.c

@@ -55,7 +55,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
     taosArrayClear(pInfo->pBlockLists);
 
     if (type == STREAM_INPUT__MERGED_SUBMIT) {
-      ASSERT(numOfBlocks > 1);
+      // ASSERT(numOfBlocks > 1);
       for (int32_t i = 0; i < numOfBlocks; i++) {
         SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*));
         taosArrayPush(pInfo->pBlockLists, &pReq);
@@ -348,7 +348,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
   taosThreadOnce(&initPoolOnce, initRefPool);
   atexit(cleanupRefPool);
 
-  qDebug("start to create subplan task, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId);
+  qDebug("start to create subplan task, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId);
 
   int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model);
   if (code != TSDB_CODE_SUCCESS) {
@@ -374,7 +374,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
     }
   }
 
-  qDebug("subplan task create completed, TID:0x%"PRIx64 " QID:0x%"PRIx64, taskId, pSubplan->id.queryId);
+  qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId);
 
 _error:
   // if failed to add ref for all tables in this query, abort current query
@@ -427,7 +427,7 @@ int waitMoment(SQInfo* pQInfo) {
 #endif
 
 static void freeBlock(void* param) {
-  SSDataBlock* pBlock = *(SSDataBlock**) param;
+  SSDataBlock* pBlock = *(SSDataBlock**)param;
   blockDataDestroy(pBlock);
 }
 
@@ -472,7 +472,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) {
 
   int64_t st = taosGetTimestampUs();
 
-  while((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) {
+  while ((pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot)) != NULL) {
     SSDataBlock* p = createOneDataBlock(pRes, true);
     current += p->info.rows;
     ASSERT(p->info.rows > 0);
@@ -494,7 +494,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) {
   uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows;
 
   qDebug("%s task suspended, %d rows in %d blocks returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",
-         GET_TASKID(pTaskInfo), current, (int32_t) taosArrayGetSize(pResList), total, 0, el / 1000.0);
+         GET_TASKID(pTaskInfo), current, (int32_t)taosArrayGetSize(pResList), total, 0, el / 1000.0);
 
   atomic_store_64(&pTaskInfo->owner, 0);
   return pTaskInfo->code;
@@ -632,7 +632,7 @@ int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {
   SOperatorInfo* pOperator = pTaskInfo->pRoot;
 
   while (1) {
-    uint8_t type = pOperator->operatorType;
+    uint16_t type = pOperator->operatorType;
     if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
       *scanner = pOperator->info;
       return 0;
@@ -691,7 +691,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
   pTaskInfo->streamInfo.prepareStatus = *pOffset;
   if (!tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) {
     while (1) {
-      uint8_t type = pOperator->operatorType;
+      uint16_t type = pOperator->operatorType;
       pOperator->status = OP_OPENED;
       // TODO add more check
       if (type != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

source/libs/executor/src/executorimpl.c

@@ -76,6 +76,12 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) {
 #define realloc u_realloc
 #endif
 
+#define T_LONG_JMP(_obj, _c) \
+  do {                       \
+    assert((_c) != -1);      \
+    longjmp((_obj), (_c));   \
+  } while (0);
+
 #define CLEAR_QUERY_STATUS(q, st)   ((q)->status &= (~(st)))
 #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0)
 
@@ -3131,6 +3137,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) {
 
     initResultRow(resultRow);
     pInfo->resultRowInfo.cur = (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset};
+    // releaseBufPage(pSup->pResultBuf, getBufPage(pSup->pResultBuf, pageId));
   }
 
   if (offset != length) {
@@ -3217,8 +3224,8 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa
 }
 
 static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
-static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo *pOperator, SFillOperatorInfo* pInfo, SResultInfo* pResultInfo,
+static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
-                                               SExecTaskInfo* pTaskInfo) {
+                                               SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) {
   pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;
   SSDataBlock* pResBlock = pInfo->pFinalRes;
 
@@ -3242,8 +3249,8 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo *pOperator, SFillOp
   pInfo->existNewGroupBlock = NULL;
 }
 
-static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, SResultInfo* pResultInfo,
+static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
-                                            SExecTaskInfo* pTaskInfo) {
+                                            SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) {
   if (taosFillHasMoreResults(pInfo->pFillInfo)) {
     int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows;
     taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows);
@@ -3270,7 +3277,7 @@ static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlo
   SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, pInfo->primarySrcSlotId);
   colDataAssign(pDst, pSrc, pInfo->pRes->info.rows, &pResBlock->info);
 
-  for(int32_t i = 0; i < pInfo->numOfNotFillExpr; ++i) {
+  for (int32_t i = 0; i < pInfo->numOfNotFillExpr; ++i) {
     SFillColInfo* pCol = &pInfo->pFillInfo->pFillCol[i + pInfo->numOfExpr];
     ASSERT(pCol->notFillCol);
 
@@ -3664,7 +3671,7 @@ void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) {
   taosRemoveRef(exchangeObjRefPool, pExInfo->self);
 }
 
-void freeSourceDataInfo(void *p) {
+void freeSourceDataInfo(void* p) {
   SSourceDataInfo* pInfo = (SSourceDataInfo*)p;
   taosMemoryFreeClear(pInfo->pRsp);
 }
@@ -3694,8 +3701,8 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t
   STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, win.skey);
   w = getFirstQualifiedTimeWindow(win.skey, &w, pInterval, TSDB_ORDER_ASC);
 
-  pInfo->pFillInfo =
+  pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo,
-      taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id);
+                                        pInfo->primaryTsCol, order, id);
 
   pInfo->win = win;
   pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES);
@@ -3741,9 +3748,9 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
   SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc,
                                                  &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
 
-  int32_t code =
+  int32_t code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr,
-      initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, (SNodeListNode*)pPhyFillNode->pValues,
+                              (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity,
-                   pPhyFillNode->timeRange, pResultInfo->capacity, pTaskInfo->id.str, pInterval, type, order);
+                              pTaskInfo->id.str, pInterval, type, order);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }

source/libs/executor/src/scanoperator.c

@@ -139,7 +139,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
       }
 
       assert(w.ekey > pBlockInfo->window.ekey);
-      if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) {
+      if (TMAX(w.skey, pBlockInfo->window.skey) <= pBlockInfo->window.ekey) {
         return true;
       }
     }
@@ -147,7 +147,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
     w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey);
     assert(w.skey <= pBlockInfo->window.ekey);
 
-    if (w.skey > pBlockInfo->window.skey) {
+    if (TMAX(w.skey, pBlockInfo->window.skey) <= TMIN(w.ekey, pBlockInfo->window.ekey)) {
       return true;
     }
 
@@ -158,7 +158,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
       }
 
       assert(w.skey < pBlockInfo->window.skey);
-      if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) {
+      if (pBlockInfo->window.skey <= TMIN(w.ekey, pBlockInfo->window.ekey)) {
         return true;
       }
     }
@@ -1649,6 +1649,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     }
     taosArrayDestroy(tableIdList);
     memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->cond, sizeof(SQueryTableDataCond));
+  } else {
+    taosArrayDestroy(pColIds);
   }
 
   // create the pseduo columns info
@@ -2038,9 +2040,33 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
     metaReaderClear(&smr);
 
     if (numOfRows >= pOperator->resultInfo.capacity) {
+      p->info.rows = numOfRows;
+      pInfo->pRes->info.rows = numOfRows;
+
+      relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
+      doFilterResult(pInfo);
+
+      blockDataCleanup(p);
+      numOfRows = 0;
+
+      if (pInfo->pRes->info.rows > 0) {
         break;
       }
     }
+  }
+
+  if (numOfRows > 0) {
+    p->info.rows = numOfRows;
+    pInfo->pRes->info.rows = numOfRows;
+
+    relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
+    doFilterResult(pInfo);
+
+    blockDataCleanup(p);
+    numOfRows = 0;
+  }
+
+  blockDataDestroy(p);
 
   // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found
   if (ret != 0) {
@@ -2049,14 +2075,6 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
     doSetOperatorCompleted(pOperator);
   }
 
-  p->info.rows = numOfRows;
-  pInfo->pRes->info.rows = numOfRows;
-
-  relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
-  doFilterResult(pInfo);
-
-  blockDataDestroy(p);
-
   pInfo->loadInfo.totalRows += pInfo->pRes->info.rows;
   return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes;
 }
@@ -2213,9 +2231,33 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
       colDataAppend(pColInfoData, numOfRows, n, false);
 
       if (++numOfRows >= pOperator->resultInfo.capacity) {
+        p->info.rows = numOfRows;
+        pInfo->pRes->info.rows = numOfRows;
+
+        relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
+        doFilterResult(pInfo);
+
+        blockDataCleanup(p);
+        numOfRows = 0;
+
+        if (pInfo->pRes->info.rows > 0) {
           break;
         }
       }
+    }
+
+    if (numOfRows > 0) {
+      p->info.rows = numOfRows;
+      pInfo->pRes->info.rows = numOfRows;
+
+      relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
+      doFilterResult(pInfo);
+
+      blockDataCleanup(p);
+      numOfRows = 0;
+    }
+
+    blockDataDestroy(p);
 
     // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found
     if (ret != 0) {
@@ -2224,14 +2266,6 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
       doSetOperatorCompleted(pOperator);
     }
 
-    p->info.rows = numOfRows;
-    pInfo->pRes->info.rows = numOfRows;
-
-    relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
-    doFilterResult(pInfo);
-
-    blockDataDestroy(p);
-
     pInfo->loadInfo.totalRows += pInfo->pRes->info.rows;
     return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes;
   }

source/libs/executor/src/timewindowoperator.c

@@ -871,7 +871,6 @@ static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_
 
 static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) {
   return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap);
-  ;
 }
 
 static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
@@ -910,10 +909,10 @@ int32_t compareWinRes(void* pKey, void* data, int32_t index) {
 }
 
 static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) {
-  if (!pUpdatedMap || taosHashGetSize(pUpdatedMap) == 0) {
+  int32_t delSize = taosArrayGetSize(pDelWins);
+  if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) {
     return;
   }
-  int32_t delSize = taosArrayGetSize(pDelWins);
   void* pIte = NULL;
   while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
     SResKeyPos* pResKey = (SResKeyPos*)pIte;
@@ -1595,7 +1594,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   SOperatorInfo* downstream = pOperator->pDownstream[0];
 
   SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
-  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);
+  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
   while (1) {
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
@@ -1706,14 +1705,19 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
   blockDataDestroy(pInfo->pPullDataRes);
   taosArrayDestroy(pInfo->pRecycledPages);
   blockDataDestroy(pInfo->pUpdateRes);
+  taosArrayDestroy(pInfo->pDelWins);
+  blockDataDestroy(pInfo->pDelRes);
 
   if (pInfo->pChildren) {
     int32_t size = taosArrayGetSize(pInfo->pChildren);
     for (int32_t i = 0; i < size; i++) {
       SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i);
       destroyStreamFinalIntervalOperatorInfo(pChildOp->info, numOfOutput);
+      taosMemoryFree(pChildOp->pDownstream);
+      cleanupExprSupp(&pChildOp->exprSupp);
       taosMemoryFreeClear(pChildOp);
     }
+    taosArrayDestroy(pInfo->pChildren);
   }
   nodesDestroyNode((SNode*)pInfo->pPhyNode);
   colDataDestroy(&pInfo->twAggSup.timeWindowData);
@@ -1780,7 +1784,7 @@ void increaseTs(SqlFunctionCtx* pCtx) {
   }
 }
 
-void initIntervalDownStream(SOperatorInfo* downstream, uint8_t type, SAggSupporter* pSup) {
+void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup) {
   if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     // Todo(liuyao) support partition by column
     return;
@@ -1881,62 +1885,6 @@ _error:
   return NULL;
 }
 
-SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
-                                                SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
-                                                STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) {
-  SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo));
-  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
-  if (pInfo == NULL || pOperator == NULL) {
-    goto _error;
-  }
-
-  pOperator->pTaskInfo = pTaskInfo;
-  pInfo->inputOrder = TSDB_ORDER_ASC;
-  pInfo->interval = *pInterval;
-  pInfo->execModel = OPTR_EXEC_MODEL_STREAM;
-  pInfo->win = pTaskInfo->window;
-  pInfo->twAggSup = *pTwAggSupp;
-  pInfo->primaryTsIndex = primaryTsSlotId;
-
-  int32_t numOfRows = 4096;
-  size_t  keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
-
-  initResultSizeInfo(&pOperator->resultInfo, numOfRows);
-  int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
-  initBasicInfo(&pInfo->binfo, pResBlock);
-  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
-
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  initResultRowInfo(&pInfo->binfo.resultRowInfo);
-
-  pOperator->name = "StreamTimeIntervalAggOperator";
-  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
-  pOperator->blocking = true;
-  pOperator->status = OP_NOT_OPENED;
-  pOperator->exprSupp.pExprInfo = pExprInfo;
-  pOperator->exprSupp.numOfExprs = numOfCols;
-  pOperator->info = pInfo;
-
-  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL,
-                                         destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
-
-  code = appendDownstream(pOperator, &downstream, 1);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  return pOperator;
-
-_error:
-  destroyIntervalOperatorInfo(pInfo, numOfCols);
-  taosMemoryFreeClear(pOperator);
-  pTaskInfo->code = code;
-  return NULL;
-}
-
 // todo handle multiple timeline cases. assume no timeline interweaving
 static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) {
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -2154,7 +2102,7 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo
 
 static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock) {
   int32_t rows = pResBlock->info.rows;
-
+  blockDataEnsureCapacity(pResBlock, rows + 1);
   // todo set the correct primary timestamp column
 
   // output the result
@@ -2408,12 +2356,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
             break;
           }
         }
+      }
 
       if (pSliceInfo->current > pSliceInfo->win.ekey) {
         doSetOperatorCompleted(pOperator);
         break;
       }
-      }
 
       if (ts == pSliceInfo->current) {
         for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
@@ -2644,7 +2592,6 @@ void destroyTimeSliceOperatorInfo(void* param, int32_t numOfOutput) {
   taosMemoryFreeClear(param);
 }
 
-
 SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
   STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
   SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@@ -3105,9 +3052,10 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) {
 
 static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
+
   SOperatorInfo* downstream = pOperator->pDownstream[0];
   SArray*        pUpdated = taosArrayInit(4, POINTER_BYTES);
-  _hash_fn_t                        hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP);
+  _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   SHashObj*      pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
   TSKEY          maxTs = INT64_MIN;
 
@@ -3542,7 +3490,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
 }
 
 void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark,
-                    uint8_t type) {
+                    uint16_t type) {
   ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN);
   SStreamScanInfo* pScanInfo = downstream->info;
   pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type};

source/libs/executor/src/tsimplehash.c

@@ -14,8 +14,8 @@
  */
 
 #include "tsimplehash.h"
-#include "os.h"
 #include "taoserror.h"
+#include "tlog.h"
 
 #define SHASH_DEFAULT_LOAD_FACTOR 0.75
 #define HASH_MAX_CAPACITY         (1024 * 1024 * 16)
@@ -31,10 +31,14 @@
     taosMemoryFreeClear(_n); \
   } while (0);
 
+#pragma pack(push, 4)
 typedef struct SHNode {
   struct SHNode *next;
+  uint32_t       keyLen : 20;
+  uint32_t       dataLen : 12;
   char           data[];
 } SHNode;
+#pragma pack(pop)
 
 struct SSHashObj {
   SHNode    **hashList;
@@ -42,8 +46,6 @@ struct SSHashObj {
   int64_t     size;     // number of elements in hash table
   _hash_fn_t  hashFp;   // hash function
   _equal_fn_t equalFp;  // equal function
-  int32_t     keyLen;
-  int32_t     dataLen;
 };
 
 static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
@@ -54,7 +56,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
   return i;
 }
 
-SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen) {
+SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) {
   ASSERT(fn != NULL);
 
   if (capacity == 0) {
@@ -74,8 +76,6 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t
   pHashObj->hashFp = fn;
   ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
 
-  pHashObj->keyLen = keyLen;
-  pHashObj->dataLen = dataLen;
 
   pHashObj->hashList = (SHNode **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *));
   if (!pHashObj->hashList) {
@@ -93,40 +93,41 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) {
   return (int32_t)atomic_load_64((int64_t *)&pHashObj->size);
 }
 
-static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
+static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *data, size_t dataLen, uint32_t hashVal) {
-  SHNode *pNewNode = taosMemoryMalloc(sizeof(SHNode) + keyLen + dsize);
+  SHNode *pNewNode = taosMemoryMalloc(sizeof(SHNode) + keyLen + dataLen);
   if (!pNewNode) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
-
+  pNewNode->keyLen = keyLen;
+  pNewNode->dataLen = dataLen;
   pNewNode->next = NULL;
-  memcpy(GET_SHASH_NODE_DATA(pNewNode), pData, dsize);
+  memcpy(GET_SHASH_NODE_DATA(pNewNode), data, dataLen);
-  memcpy(GET_SHASH_NODE_KEY(pNewNode, dsize), key, keyLen);
+  memcpy(GET_SHASH_NODE_KEY(pNewNode, dataLen), key, keyLen);
   return pNewNode;
 }
 
-static void taosHashTableResize(SSHashObj *pHashObj) {
+static void tSimpleHashTableResize(SSHashObj *pHashObj) {
   if (!SHASH_NEED_RESIZE(pHashObj)) {
     return;
   }
 
   int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
   if (newCapacity > HASH_MAX_CAPACITY) {
-    //    uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
+    uDebug("current capacity:%zu, maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached",
-    //           pHashObj->capacity, HASH_MAX_CAPACITY);
+           pHashObj->capacity, HASH_MAX_CAPACITY);
     return;
   }
 
   int64_t st = taosGetTimestampUs();
   void   *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity);
   if (!pNewEntryList) {
-    //    qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
+    uWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
     return;
   }
 
   size_t inc = newCapacity - pHashObj->capacity;
-  memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc);
+  memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc * sizeof(void *));
 
   pHashObj->hashList = pNewEntryList;
   pHashObj->capacity = newCapacity;
@@ -141,8 +142,8 @@ static void taosHashTableResize(SSHashObj *pHashObj) {
     SHNode *pPrev = NULL;
 
     while (pNode != NULL) {
-      void    *key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen);
+      void    *key = GET_SHASH_NODE_KEY(pNode, pNode->dataLen);
-      uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
+      uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pNode->keyLen);
 
       int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity);
       pNext = pNode->next;
@@ -170,23 +171,23 @@ static void taosHashTableResize(SSHashObj *pHashObj) {
   //         ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
 }
 
-int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) {
+int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t dataLen) {
   if (!pHashObj || !key) {
     return -1;
   }
 
-  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
+  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
 
   // need the resize process, write lock applied
   if (SHASH_NEED_RESIZE(pHashObj)) {
-    taosHashTableResize(pHashObj);
+    tSimpleHashTableResize(pHashObj);
   }
 
   int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
 
   SHNode *pNode = pHashObj->hashList[slot];
   if (!pNode) {
-    SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->dataLen, hashVal);
+    SHNode *pNewNode = doCreateHashNode(key, keyLen, data, dataLen, hashVal);
     if (!pNewNode) {
       return -1;
     }
@@ -197,14 +198,14 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) {
   }
 
   while (pNode) {
-    if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) {
+    if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pNode->dataLen), key, keyLen) == 0) {
       break;
     }
     pNode = pNode->next;
   }
 
   if (!pNode) {
-    SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->dataLen, hashVal);
+    SHNode *pNewNode = doCreateHashNode(key, keyLen, data, dataLen, hashVal);
     if (!pNewNode) {
       return -1;
     }
@@ -212,16 +213,16 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) {
     pHashObj->hashList[slot] = pNewNode;
     atomic_add_fetch_64(&pHashObj->size, 1);
   } else {  // update data
-    memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen);
+    memcpy(GET_SHASH_NODE_DATA(pNode), data, dataLen);
   }
 
   return 0;
 }
 
-static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, int32_t index) {
+static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, size_t keyLen, int32_t index) {
   SHNode *pNode = pHashObj->hashList[index];
   while (pNode) {
-    if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) {
+    if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pNode->dataLen), key, keyLen) == 0) {
       break;
     }
 
@@ -233,12 +234,12 @@ static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void
 
 static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { return tSimpleHashGetSize(pHashObj) == 0; }
 
-void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) {
+void *tSimpleHashGet(SSHashObj *pHashObj, const void *key, size_t keyLen) {
   if (!pHashObj || taosHashTableEmpty(pHashObj) || !key) {
     return NULL;
   }
 
-  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
+  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
 
   int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
   SHNode *pNode = pHashObj->hashList[slot];
@@ -247,7 +248,7 @@ void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) {
   }
 
   char *data = NULL;
-  pNode = doSearchInEntryList(pHashObj, key, slot);
+  pNode = doSearchInEntryList(pHashObj, key, keyLen, slot);
   if (pNode != NULL) {
     data = GET_SHASH_NODE_DATA(pNode);
   }
@@ -255,19 +256,19 @@ void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) {
   return data;
 }
 
-int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key) {
+int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen) {
   if (!pHashObj || !key) {
     return TSDB_CODE_FAILED;
   }
 
-  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
+  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
 
   int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
 
   SHNode *pNode = pHashObj->hashList[slot];
   SHNode *pPrev = NULL;
   while (pNode) {
-    if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) {
+    if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pNode->dataLen), key, keyLen) == 0) {
       if (!pPrev) {
         pHashObj->hashList[slot] = pNode->next;
       } else {
@@ -312,6 +313,7 @@ void tSimpleHashCleanup(SSHashObj *pHashObj) {
 
   tSimpleHashClear(pHashObj);
   taosMemoryFreeClear(pHashObj->hashList);
+  taosMemoryFree(pHashObj);
 }
 
 size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) {
@@ -322,23 +324,13 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) {
   return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj);
 }
 
-void *tSimpleHashGetKey(const SSHashObj *pHashObj, void *data, size_t *keyLen) {
+void *tSimpleHashGetKey(void *data, size_t *keyLen) {
-#if 0
+  SHNode *node = (SHNode *)((char *)data - offsetof(SHNode, data));
-  int32_t offset = offsetof(SHNode, data);
-  SHNode *node = ((SHNode *)(char *)data - offset);
   if (keyLen) {
-    *keyLen = pHashObj->keyLen;
+    *keyLen = node->keyLen;
   }
 
-  return POINTER_SHIFT(data, pHashObj->dataLen);
+  return POINTER_SHIFT(data, node->dataLen);
-
-  return GET_SHASH_NODE_KEY(node, pHashObj->dataLen);
-#endif
-  if (keyLen) {
-    *keyLen = pHashObj->keyLen;
-  }
-
-  return POINTER_SHIFT(data, pHashObj->dataLen);
 }
 
 void *tSimpleHashIterate(const SSHashObj *pHashObj, void *data, int32_t *iter) {
@@ -378,3 +370,50 @@ void *tSimpleHashIterate(const SSHashObj *pHashObj, void *data, int32_t *iter) {
 
   return NULL;
 }
+
+void *tSimpleHashIterateKV(const SSHashObj *pHashObj, void *data, void **key, int32_t *iter) {
+  if (!pHashObj) {
+    return NULL;
+  }
+
+  SHNode *pNode = NULL;
+
+  if (!data) {
+    for (int32_t i = 0; i < pHashObj->capacity; ++i) {
+      pNode = pHashObj->hashList[i];
+      if (!pNode) {
+        continue;
+      }
+      *iter = i;
+      if (key) {
+        *key = GET_SHASH_NODE_KEY(pNode, pNode->dataLen);
+      }
+      return GET_SHASH_NODE_DATA(pNode);
+    }
+    return NULL;
+  }
+
+  pNode = (SHNode *)((char *)data - offsetof(SHNode, data));
+
+  if (pNode->next) {
+    if (key) {
+      *key = GET_SHASH_NODE_KEY(pNode->next, pNode->next->dataLen);
+    }
+    return GET_SHASH_NODE_DATA(pNode->next);
+  }
+
+  ++(*iter);
+  for (int32_t i = *iter; i < pHashObj->capacity; ++i) {
+    pNode = pHashObj->hashList[i];
+    if (!pNode) {
+      continue;
+    }
+    *iter = i;
+    if (key) {
+      *key = GET_SHASH_NODE_KEY(pNode, pNode->dataLen);
+    }
+    return GET_SHASH_NODE_DATA(pNode);
+  }
+
+  return NULL;
+}

source/libs/executor/test/tSimpleHashTests.cpp

@@ -32,31 +32,33 @@
 
 TEST(testCase, tSimpleHashTest) {
   SSHashObj *pHashObj =
-      tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), sizeof(int64_t), sizeof(int64_t));
+      tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
 
   assert(pHashObj != nullptr);
 
   ASSERT_EQ(0, tSimpleHashGetSize(pHashObj));
 
+  size_t keyLen = sizeof(int64_t);
+  size_t dataLen = sizeof(int64_t);
+
   int64_t originKeySum = 0;
   for (int64_t i = 1; i <= 100; ++i) {
     originKeySum += i;
-    tSimpleHashPut(pHashObj, (const void *)&i, (const void *)&i);
+    tSimpleHashPut(pHashObj, (const void *)&i, keyLen, (const void *)&i, dataLen);
     ASSERT_EQ(i, tSimpleHashGetSize(pHashObj));
   }
 
   for (int64_t i = 1; i <= 100; ++i) {
-    void *data = tSimpleHashGet(pHashObj, (const void *)&i);
+    void *data = tSimpleHashGet(pHashObj, (const void *)&i, keyLen);
     ASSERT_EQ(i, *(int64_t *)data);
   }
 
-
   void   *data = NULL;
   int32_t iter = 0;
   int64_t keySum = 0;
   int64_t dataSum = 0;
   while ((data = tSimpleHashIterate(pHashObj, data, &iter))) {
-    void *key = tSimpleHashGetKey(pHashObj, data, NULL);
+    void *key = tSimpleHashGetKey(data, NULL);
     keySum += *(int64_t *)key;
     dataSum += *(int64_t *)data;
   }
@@ -65,7 +67,7 @@ TEST(testCase, tSimpleHashTest) {
   ASSERT_EQ(keySum, originKeySum);
 
   for (int64_t i = 1; i <= 100; ++i) {
-    tSimpleHashRemove(pHashObj, (const void *)&i);
+    tSimpleHashRemove(pHashObj, (const void *)&i, keyLen);
     ASSERT_EQ(100 - i, tSimpleHashGetSize(pHashObj));
   }
 

source/libs/function/src/builtins.c

@@ -1503,11 +1503,17 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 0));
+  uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  if (!IS_NUMERIC_TYPE(paraType) || QUERY_NODE_VALUE == nodeType) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
   if (3 <= numOfParams) {
     int64_t timeVal[2] = {0};
     for (int32_t i = 1; i < 3; ++i) {
-      uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, i));
+      nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, i));
-      uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type;
+      paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type;
       if (!IS_VAR_DATA_TYPE(paraType) || QUERY_NODE_VALUE != nodeType) {
         return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
       }
@@ -1525,8 +1531,8 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   }
 
   if (4 == numOfParams) {
-    uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 3));
+    nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 3));
-    uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type;
+    paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type;
     if (!IS_INTEGER_TYPE(paraType) || QUERY_NODE_VALUE != nodeType) {
       return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
     }

source/libs/function/src/builtinsimpl.c

@@ -3970,16 +3970,16 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) {
     TSKEY*  ptsList = (int64_t*)colDataGetData(pCol, 0);
     if (pCtx->order == TSDB_ORDER_DESC) {
       if (pCtx->start.key == INT64_MIN) {
-        pInfo->max =
+        pInfo->max = (pInfo->max < ptsList[start]) ? ptsList[start] : pInfo->max;
-            (pInfo->max < ptsList[start + pInput->numOfRows - 1]) ? ptsList[start + pInput->numOfRows - 1] : pInfo->max;
       } else {
         pInfo->max = pCtx->start.key + 1;
       }
 
-      if (pCtx->end.key != INT64_MIN) {
+      if (pCtx->end.key == INT64_MIN) {
-        pInfo->min = pCtx->end.key;
+        pInfo->min = (pInfo->min > ptsList[start + pInput->numOfRows - 1]) ?
+                     ptsList[start + pInput->numOfRows - 1] : pInfo->min;
       } else {
-        pInfo->min = ptsList[start];
+        pInfo->min = pCtx->end.key;
       }
     } else {
       if (pCtx->start.key == INT64_MIN) {
@@ -3988,10 +3988,11 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) {
         pInfo->min = pCtx->start.key;
       }
 
-      if (pCtx->end.key != INT64_MIN) {
+      if (pCtx->end.key == INT64_MIN) {
-        pInfo->max = pCtx->end.key + 1;
+        pInfo->max = (pInfo->max < ptsList[start + pInput->numOfRows - 1]) ?
+                     ptsList[start + pInput->numOfRows - 1] : pInfo->max;
       } else {
-        pInfo->max = ptsList[start + pInput->numOfRows - 1];
+        pInfo->max = pCtx->end.key + 1;
       }
     }
   }
@@ -5573,6 +5574,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) {
   if (pCtx->end.key != INT64_MIN) {
     pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end);
     pInfo->p = pCtx->end;
+    numOfElems += 1;
   }
 
   pInfo->win.ekey = pInfo->p.key;

source/libs/nodes/src/nodesCodeFuncs.c

@@ -4673,7 +4673,6 @@ static int32_t jsonToNode(const SJson* pJson, void* pObj) {
 
   int32_t code;
   tjsonGetNumberValue(pJson, jkNodeType, pNode->type, code);
-  ;
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonToObject(pJson, nodesNodeName(pNode->type), jsonToSpecificNode, pNode);
     if (TSDB_CODE_SUCCESS != code) {

source/libs/nodes/src/nodesToSQLFuncs.c

@@ -21,36 +21,89 @@
 #include "taoserror.h"
 #include "thash.h"
 
-char *gOperatorStr[] = {NULL,
+const char *operatorTypeStr(EOperatorType type) {
-                        "+",
+  switch (type) {
-                        "-",
+    case OP_TYPE_ADD:
-                        "*",
+      return "+";
-                        "/",
+    case OP_TYPE_SUB:
-                        "%",
+      return "-";
-                        "-",
+    case OP_TYPE_MULTI:
-                        "&",
+      return "*";
-                        "|",
+    case OP_TYPE_DIV:
-                        ">",
+      return "/";
-                        ">=",
+    case OP_TYPE_REM:
-                        "<",
+      return "%";
-                        "<=",
+    case OP_TYPE_MINUS:
-                        "=",
+      return "-";
-                        "<>",
+    case OP_TYPE_BIT_AND:
-                        "IN",
+      return "&";
-                        "NOT IN",
+    case OP_TYPE_BIT_OR:
-                        "LIKE",
+      return "|";
-                        "NOT LIKE",
+    case OP_TYPE_GREATER_THAN:
-                        "MATCH",
+      return ">";
-                        "NMATCH",
+    case OP_TYPE_GREATER_EQUAL:
-                        "IS NULL",
+      return ">=";
-                        "IS NOT NULL",
+    case OP_TYPE_LOWER_THAN:
-                        "IS TRUE",
+      return "<";
-                        "IS FALSE",
+    case OP_TYPE_LOWER_EQUAL:
-                        "IS UNKNOWN",
+      return "<=";
-                        "IS NOT TRUE",
+    case OP_TYPE_EQUAL:
-                        "IS NOT FALSE",
+      return "=";
-                        "IS NOT UNKNOWN"};
+    case OP_TYPE_NOT_EQUAL:
-char *gLogicConditionStr[] = {"AND", "OR", "NOT"};
+      return "<>";
+    case OP_TYPE_IN:
+      return "IN";
+    case OP_TYPE_NOT_IN:
+      return "NOT IN";
+    case OP_TYPE_LIKE:
+      return "LIKE";
+    case OP_TYPE_NOT_LIKE:
+      return "NOT LIKE";
+    case OP_TYPE_MATCH:
+      return "MATCH";
+    case OP_TYPE_NMATCH:
+      return "NMATCH";
+    case OP_TYPE_IS_NULL:
+      return "IS NULL";
+    case OP_TYPE_IS_NOT_NULL:
+      return "IS NOT NULL";
+    case OP_TYPE_IS_TRUE:
+      return "IS TRUE";
+    case OP_TYPE_IS_FALSE:
+      return "IS FALSE";
+    case OP_TYPE_IS_UNKNOWN:
+      return "IS UNKNOWN";
+    case OP_TYPE_IS_NOT_TRUE:
+      return "IS NOT TRUE";
+    case OP_TYPE_IS_NOT_FALSE:
+      return "IS NOT FALSE";
+    case OP_TYPE_IS_NOT_UNKNOWN:
+      return "IS NOT UNKNOWN";
+    case OP_TYPE_JSON_GET_VALUE:
+      return "=>";
+    case OP_TYPE_JSON_CONTAINS:
+      return "CONTAINS";
+    case OP_TYPE_ASSIGN:
+      return "=";
+    default:
+      break;
+  }
+  return "UNKNOWN";
+}
+
+const char *logicConditionTypeStr(ELogicConditionType type) {
+  switch (type) {
+    case LOGIC_COND_TYPE_AND:
+      return "AND";
+    case LOGIC_COND_TYPE_OR:
+      return "OR";
+    case LOGIC_COND_TYPE_NOT:
+      return "NOT";
+    default:
+      break;
+  }
+  return "UNKNOWN";
+}
 
 int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) {
   switch (pNode->type) {
@@ -94,12 +147,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) {
         NODES_ERR_RET(nodesNodeToSQL(pOpNode->pLeft, buf, bufSize, len));
       }
 
-      if (pOpNode->opType >= (sizeof(gOperatorStr) / sizeof(gOperatorStr[0]))) {
+      *len += snprintf(buf + *len, bufSize - *len, " %s ", operatorTypeStr(pOpNode->opType));
-        nodesError("unknown operation type:%d", pOpNode->opType);
-        NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
-      }
-
-      *len += snprintf(buf + *len, bufSize - *len, " %s ", gOperatorStr[pOpNode->opType]);
 
       if (pOpNode->pRight) {
         NODES_ERR_RET(nodesNodeToSQL(pOpNode->pRight, buf, bufSize, len));
@@ -118,7 +166,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) {
 
       FOREACH(node, pLogicNode->pParameterList) {
         if (!first) {
-          *len += snprintf(buf + *len, bufSize - *len, " %s ", gLogicConditionStr[pLogicNode->condType]);
+          *len += snprintf(buf + *len, bufSize - *len, " %s ", logicConditionTypeStr(pLogicNode->condType));
         }
         NODES_ERR_RET(nodesNodeToSQL(node, buf, bufSize, len));
         first = false;

source/libs/parser/src/parInsert.c

@@ -143,9 +143,9 @@ static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, con
     }
     char name[TSDB_DB_FNAME_LEN] = {0};
     strncpy(name, pTableName->z, dbLen);
-    dbLen = strdequote(name);
+    int32_t actualDbLen = strdequote(name);
 
-    code = tNameSetDbName(pName, acctId, name, dbLen);
+    code = tNameSetDbName(pName, acctId, name, actualDbLen);
     if (code != TSDB_CODE_SUCCESS) {
       return buildInvalidOperationMsg(pMsgBuf, msg1);
     }

source/libs/parser/src/parTranslater.c

@@ -2822,6 +2822,29 @@ static int32_t createDefaultFillNode(STranslateContext* pCxt, SNode** pOutput) {
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t checkEvery(STranslateContext* pCxt, SValueNode* pInterval) {
+  int32_t len = strlen(pInterval->literal);
+
+  char *unit = &pInterval->literal[len - 1];
+  if (*unit == 'n' || *unit == 'y') {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE,
+                                   "Unsupported time unit in EVERY clause");
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateInterpEvery(STranslateContext* pCxt, SNode** pEvery) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  code = checkEvery(pCxt, (SValueNode *)(*pEvery));
+  if (TSDB_CODE_SUCCESS == code) {
+    code = translateExpr(pCxt, pEvery);
+  }
+
+  return code;
+}
+
 static int32_t translateInterpFill(STranslateContext* pCxt, SSelectStmt* pSelect) {
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -2856,7 +2879,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
 
   int32_t code = translateExpr(pCxt, &pSelect->pRange);
   if (TSDB_CODE_SUCCESS == code) {
-    code = translateExpr(pCxt, &pSelect->pEvery);
+    code = translateInterpEvery(pCxt, &pSelect->pEvery);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = translateInterpFill(pCxt, pSelect);

source/libs/scalar/inc/filterInt.h

@@ -350,7 +350,6 @@ struct SFilterInfo {
 
 extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
 extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
-extern OptrStr gOptrStr[];
 
 #ifdef __cplusplus
 }

source/libs/scalar/src/filter.c

@@ -24,46 +24,6 @@
 #include "ttime.h"
 #include "functionMgt.h"
 
-OptrStr gOptrStr[] = {
-  {0,                                      "invalid"},
-  {OP_TYPE_ADD,                            "+"},
-  {OP_TYPE_SUB,                            "-"},
-  {OP_TYPE_MULTI,                          "*"},
-  {OP_TYPE_DIV,                            "/"},
-  {OP_TYPE_REM,                            "%"},
-  {OP_TYPE_MINUS,                          "minus"},
-  {OP_TYPE_ASSIGN,                         "assign"},
-  // bit operator
-  {OP_TYPE_BIT_AND,                        "&"},
-  {OP_TYPE_BIT_OR,                         "|"},
-
-  // comparison operator
-  {OP_TYPE_GREATER_THAN,                   ">"},
-  {OP_TYPE_GREATER_EQUAL,                  ">="},
-  {OP_TYPE_LOWER_THAN,                     "<"},
-  {OP_TYPE_LOWER_EQUAL,                    "<="},
-  {OP_TYPE_EQUAL,                          "=="},
-  {OP_TYPE_NOT_EQUAL,                      "!="},
-  {OP_TYPE_IN,                             "in"},
-  {OP_TYPE_NOT_IN,                         "not in"},
-  {OP_TYPE_LIKE,                           "like"},
-  {OP_TYPE_NOT_LIKE,                       "not like"},
-  {OP_TYPE_MATCH,                          "match"},
-  {OP_TYPE_NMATCH,                         "nmatch"},
-  {OP_TYPE_IS_NULL,                        "is null"},
-  {OP_TYPE_IS_NOT_NULL,                    "not null"},
-  {OP_TYPE_IS_TRUE,                        "is true"},
-  {OP_TYPE_IS_FALSE,                       "is false"},
-  {OP_TYPE_IS_UNKNOWN,                     "is unknown"},
-  {OP_TYPE_IS_NOT_TRUE,                    "not true"},
-  {OP_TYPE_IS_NOT_FALSE,                   "not false"},
-  {OP_TYPE_IS_NOT_UNKNOWN,                 "not unknown"},
-
-  // json operator
-  {OP_TYPE_JSON_GET_VALUE,                 "->"},
-  {OP_TYPE_JSON_CONTAINS,                  "json contains"}
-};
-
 bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   int32_t result = cfunc(maxv, minr);
   return result >= 0;
@@ -986,7 +946,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
   } else {
     int32_t paramNum = scalarGetOperatorParamNum(optr);
     if (1 != paramNum) {
-      fltError("invalid right field in unit, operator:%s, rightType:%d", gOptrStr[optr].str, u->right.type);
+      fltError("invalid right field in unit, operator:%s, rightType:%d", operatorTypeStr(optr), u->right.type);
       return TSDB_CODE_QRY_APP_ERROR;
     }
   }
@@ -1517,7 +1477,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
         SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
         SColumnNode *refNode = (SColumnNode *)left->desc;
         if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){
-          len = sprintf(str, "UNIT[%d] => [%d][%d]  %s  [", i, refNode->dataBlockId, refNode->slotId, gOptrStr[unit->compare.optr].str);
+          len = sprintf(str, "UNIT[%d] => [%d][%d]  %s  [", i, refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr));
         }
 
         if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) {
@@ -1536,7 +1496,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
         if (unit->compare.optr2) {
           strcat(str, " && ");
           if (unit->compare.optr2 >= 0 && unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS){
-            sprintf(str + strlen(str), "[%d][%d]  %s  [", refNode->dataBlockId, refNode->slotId, gOptrStr[unit->compare.optr2].str);
+            sprintf(str + strlen(str), "[%d][%d]  %s  [", refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr2));
           }
 
           if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) {

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -1089,16 +1089,16 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
 
  }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV ||
             opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){
-   printf("op:%s,1result:%f,except:%f\n", gOptrStr[opType].str, *((double *)colDataGetData(column, 0)), exceptValue);
+   printf("op:%s,1result:%f,except:%f\n", operatorTypeStr(opType), *((double *)colDataGetData(column, 0)), exceptValue);
    ASSERT_TRUE(fabs(*((double *)colDataGetData(column, 0)) - exceptValue) < 0.0001);
  }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){
-   printf("op:%s,2result:%" PRId64 ",except:%f\n", gOptrStr[opType].str, *((int64_t *)colDataGetData(column, 0)), exceptValue);
+   printf("op:%s,2result:%" PRId64 ",except:%f\n", operatorTypeStr(opType), *((int64_t *)colDataGetData(column, 0)), exceptValue);
    ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue);
  }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN ||
             opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL ||
             opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE ||
             opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){
-   printf("op:%s,3result:%d,except:%f\n", gOptrStr[opType].str, *((bool *)colDataGetData(column, 0)), exceptValue);
+   printf("op:%s,3result:%d,except:%f\n", operatorTypeStr(opType), *((bool *)colDataGetData(column, 0)), exceptValue);
    ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue);
  }
 

source/libs/scheduler/src/schRemote.c

@@ -20,7 +20,7 @@
 #include "tmsg.h"
 #include "tref.h"
 #include "trpc.h"
-
+// clang-format off
 int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) {
   int32_t lastMsgType = pTask->lastMsgType;
   int32_t taskStatus = SCH_GET_TASK_STATUS(pTask);
@@ -1104,7 +1104,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
 
 #if 1
   SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)};
-  schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL));
+  code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL));
   msg = NULL;
   SCH_ERR_JRET(code);
 
@@ -1114,7 +1114,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
 #else
   if (TDMT_VND_SUBMIT != msgType) {
     SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)};
-    schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL));
+    code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL));
     msg = NULL;
     SCH_ERR_JRET(code);
 
@@ -1136,3 +1136,4 @@ _return:
   taosMemoryFreeClear(msg);
   SCH_RET(code);
 }
+// clang-format on

source/libs/stream/inc/streamInc.h

@@ -32,7 +32,6 @@ typedef struct {
 
 static SStreamGlobalEnv streamEnv;
 
-int32_t streamExec(SStreamTask* pTask);
 int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch);
 
 int32_t streamDispatch(SStreamTask* pTask);

source/libs/stream/src/stream.c

@@ -185,7 +185,9 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
   tFreeStreamDispatchReq(pReq);
 
   if (exec) {
-    streamTryExec(pTask);
+    if (streamTryExec(pTask) < 0) {
+      return -1;
+    }
 
     if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
       streamDispatch(pTask);
@@ -221,7 +223,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) {
 }
 
 int32_t streamProcessRunReq(SStreamTask* pTask) {
-  streamTryExec(pTask);
+  if (streamTryExec(pTask) < 0) {
+    return -1;
+  }
 
   if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
     streamDispatch(pTask);

source/libs/stream/src/streamMeta.c

@@ -15,6 +15,7 @@
 
 #include "executor.h"
 #include "tstream.h"
+#include "ttimer.h"
 
 SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc) {
   SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta));
@@ -99,16 +100,19 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char*
     goto FAIL;
   }
 
-  taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*));
+  if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) {
+    goto FAIL;
+  }
 
   if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, &pMeta->txn) < 0) {
+    taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t));
     ASSERT(0);
-    return -1;
+    goto FAIL;
   }
   return 0;
 
 FAIL:
-  if (pTask) taosMemoryFree(pTask);
+  if (pTask) tFreeSStreamTask(pTask);
   return -1;
 }
 
@@ -158,11 +162,28 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
     SStreamTask* pTask = *ppTask;
     taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t));
     atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING);
-  }
 
     if (tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), &pMeta->txn) < 0) {
       /*return -1;*/
     }
+
+    if (pTask->triggerParam != 0) {
+      taosTmrStop(pTask->timer);
+    }
+
+    while (1) {
+      int8_t schedStatus =
+          atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__DROPPING);
+      if (schedStatus == TASK_SCHED_STATUS__INACTIVE) {
+        tFreeSStreamTask(pTask);
+        break;
+      } else if (schedStatus == TASK_SCHED_STATUS__DROPPING) {
+        break;
+      }
+      taosMsleep(10);
+    }
+  }
+
   return 0;
 }
 

source/libs/stream/src/streamQueue.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tstream.h"
+#include "streamInc.h"
 
 SStreamQueue* streamQueueOpen() {
   SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue));
@@ -36,9 +36,12 @@ void streamQueueClose(SStreamQueue* queue) {
   while (1) {
     void* qItem = streamQueueNextItem(queue);
     if (qItem) {
-      taosFreeQitem(qItem);
+      streamFreeQitem(qItem);
     } else {
-      return;
+      break;
     }
   }
+  taosFreeQall(queue->qall);
+  taosCloseQueue(queue->queue);
+  taosMemoryFree(queue);
 }

source/libs/stream/src/streamTask.c

@@ -152,9 +152,18 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
 }
 
 void tFreeSStreamTask(SStreamTask* pTask) {
-  streamQueueClose(pTask->inputQueue);
+  qDebug("free stream task %d", pTask->taskId);
-  streamQueueClose(pTask->outputQueue);
+  if (pTask->inputQueue) streamQueueClose(pTask->inputQueue);
+  if (pTask->outputQueue) streamQueueClose(pTask->outputQueue);
   if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg);
   if (pTask->exec.executor) qDestroyTask(pTask->exec.executor);
+  taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree);
+  if (pTask->outputType == TASK_OUTPUT__TABLE) {
+    tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper);
+    taosMemoryFree(pTask->tbSink.pTSchema);
+  }
+  if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
+    taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos);
+  }
   taosMemoryFree(pTask);
 }