Merge branch '3.0' into feat/sangshuduo/TD-14141-update-taostools-for3.0

cmake/cmake.platform

@@ -76,15 +76,19 @@ IF ("${CPUTYPE}" STREQUAL "")
   IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)")
     MESSAGE(STATUS "The current platform is amd64")
     SET(PLATFORM_ARCH_STR "amd64")
+    SET(TD_INTEL_64 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)")
     MESSAGE(STATUS "The current platform is x86")
     SET(PLATFORM_ARCH_STR "i386")
+    SET(TD_INTEL_32 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l")
     MESSAGE(STATUS "The current platform is aarch32")
     SET(PLATFORM_ARCH_STR "arm")
+    SET(TD_ARM_32 TRUE)
   ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
     MESSAGE(STATUS "The current platform is aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
+    SET(TD_ARM_64 TRUE)
   ENDIF ()
 ELSE ()
   # if generate ARM version:
@@ -92,18 +96,23 @@ ELSE ()
   IF (${CPUTYPE} MATCHES "aarch32")
     SET(PLATFORM_ARCH_STR "arm")
     MESSAGE(STATUS "input cpuType: aarch32")
+    SET(TD_ARM_32 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
     MESSAGE(STATUS "input cpuType: aarch64")
+    SET(TD_ARM_64 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "mips64")
     SET(PLATFORM_ARCH_STR "mips")
     MESSAGE(STATUS "input cpuType: mips64")
+    SET(TD_MIPS_64 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "x64")
     SET(PLATFORM_ARCH_STR "amd64")
     MESSAGE(STATUS "input cpuType: x64")
+    SET(TD_INTEL_64 TRUE)
   ELSEIF (${CPUTYPE} MATCHES "x86")
     SET(PLATFORM_ARCH_STR "i386")
     MESSAGE(STATUS "input cpuType: x86")
+    SET(TD_INTEL_32 TRUE)
   ELSE ()
     MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE})
   ENDIF ()

docs/en/14-reference/03-connector/java.mdx

@@ -202,6 +202,10 @@ The configuration parameters in the URL are as follows.
 - batchfetch: true: pull the result set in batch when executing the query; false: pull the result set row by row. The default value is false. batchfetch uses HTTP for data transfer. The JDBC REST connection supports bulk data pulling function in taos-jdbcdriver-2.0.38 and TDengine 2.4.0.12 and later versions. taos-jdbcdriver and TDengine transfer data via WebSocket connection. Compared with HTTP, WebSocket enables JDBC REST connection to support large data volume querying and improve query performance.
 - charset: specify the charset to parse the string, this parameter is valid only when set batchfetch to true.
 - batchErrorIgnore: true: when executing executeBatch of Statement, if one SQL execution fails in the middle, continue to execute the following SQL. false: no longer execute any statement after the failed SQL. The default value is: false.
+- httpConnectTimeout: REST connection timeout in milliseconds, the default value is 5000 ms.
+- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when batchfetch is false.
+- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when batchfetch is true.
+- useSSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection.
 
 **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection.
 
@@ -257,14 +261,18 @@ In the above example, a connection is established to `taosdemo.com`, port is 603
 
 The configuration parameters in properties are as follows.
 
-- TSDBDriver.PROPERTY_KEY_USER: Login TDengine user name, default value 'root'.
+- TSDBDriver.PROPERTY_KEY_USER: login TDengine user name, default value 'root'.
 - TSDBDriver.PROPERTY_KEY_PASSWORD: user login password, default value 'taosdata'.
 - TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true: pull the result set in batch when executing query; false: pull the result set row by row. The default value is: false.
 - TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE: true: when executing executeBatch of Statement, if there is a SQL execution failure in the middle, continue to execute the following sq. false: no longer execute any statement after the failed SQL. The default value is: false.
-- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: Only works when using JDBC native connection. Client configuration file directory path, default value `/etc/taos` on Linux OS, default value `C:/TDengine/cfg` on Windows OS.
+- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: only works when using JDBC native connection. Client configuration file directory path, default value `/etc/taos` on Linux OS, default value `C:/TDengine/cfg` on Windows OS.
 - TSDBDriver.PROPERTY_KEY_CHARSET: In the character set used by the client, the default value is the system character set.
 - TSDBDriver.PROPERTY_KEY_LOCALE: this only takes effect when using JDBC native connection. Client language environment, the default value is system current locale.
 - TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone.
+- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection.
+- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection and batchfetch is false.
+- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when using JDBC REST connection and batchfetch is true.
+- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. It only takes effect when using using JDBC REST connection.
   For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only).
 
 ### Priority of configuration parameters
@@ -812,11 +820,12 @@ Please refer to: [JDBC example](https://github.com/taosdata/TDengine/tree/develo
 
 ## Recent update logs
 
-| taos-jdbcdriver version |                major changes                 |
+| taos-jdbcdriver version |                 major changes                  |
-| :---------------------: | :------------------------------------------: |
+| :---------------------: | :--------------------------------------------: |
-|         2.0.38          | JDBC REST connections add bulk pull function |
+|     2.0.39 - 2.0.40     | Add REST connection/request timeout parameters |
-|         2.0.37          |         Added support for json tags          |
+|         2.0.38          |  JDBC REST connections add bulk pull function  |
-|         2.0.36          |      Add support for schemaless writing      |
+|         2.0.37          |               Support json tags                |
+|         2.0.36          |           Support schemaless writing           |
 
 ## Frequently Asked Questions
 

docs/zh/12-taos-sql/12-interval.md

@@ -3,9 +3,9 @@ sidebar_label: 时序数据特色查询
 title: 时序数据特色查询
 ---
 
-TDengine是专为时序数据而研发的大数据平台，存储和计算都针对时序数据的特定进行了量身定制，在支持标准SQL的基础之上，还提供了一系列贴合时序业务场景的特色查询语法，极大的方便时序场景的应用开发。
+TDengine 是专为时序数据而研发的大数据平台，存储和计算都针对时序数据的特定进行了量身定制，在支持标准 SQL 的基础之上，还提供了一系列贴合时序业务场景的特色查询语法，极大的方便时序场景的应用开发。
 
-TDengine提供的特色查询包括标签切分查询和窗口切分查询。
+TDengine 提供的特色查询包括标签切分查询和窗口切分查询。
 
 ## 标签切分查询
 
@@ -14,13 +14,14 @@ TDengine提供的特色查询包括标签切分查询和窗口切分查询。
 ```sql
 PARTITION BY tag_list
 ```
-其中 `tag_list` 是标签列的列表，还可以包括tbname伪列。
 
-TDengine按如下方式处理标签切分子句：
+其中 `tag_list` 是标签列的列表，还可以包括 tbname 伪列。
+
+TDengine 按如下方式处理标签切分子句：
 
 标签切分子句位于 `WHERE` 子句之后，且不能和 `JOIN` 子句一起使用。
 标签切分子句将超级表数据按指定的标签组合进行切分，然后对每个切分的分片进行指定的计算。计算由之后的子句定义（窗口子句、`GROUP BY` 子句或`SELECT` 子句）。
-标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按10分钟进行降采样，取其最大值。
+标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按 10 分钟进行降采样，取其最大值。
 
 ```sql
 select max(current) from meters partition by location interval(10m)
@@ -42,27 +43,31 @@ SELECT function_list FROM tb_name
 在上述语法中的具体限制如下
 
 ### 窗口切分查询中使用函数的限制
+
 - 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。
 - 此外 LAST_ROW 查询也不能与窗口聚合同时出现。
 - 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。
 
 ### 窗口子句的规则
-- 窗口子句位于标签切分子句之后，GROUP BY子句之前，且不可以和GROUP BY子句一起使用。
+
-- 窗口子句将数据按窗口进行切分，对每个窗口进行SELECT列表中的表达式的计算，SELECT列表中的表达式只能包含：
+- 窗口子句位于标签切分子句之后，GROUP BY 子句之前，且不可以和 GROUP BY 子句一起使用。
+- 窗口子句将数据按窗口进行切分，对每个窗口进行 SELECT 列表中的表达式的计算，SELECT 列表中的表达式只能包含：
   - 常量。
   - 聚集函数。
   - 包含上面表达式的表达式。
-- 窗口子句不可以和GROUP BY子句一起使用。
+- 窗口子句不可以和 GROUP BY 子句一起使用。
 - WHERE 语句可以指定查询的起止时间和其他过滤条件。
 
 ### FILL 子句
- FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种：
+
-  1. 不进行填充：NONE（默认填充模式）。
+FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种：
-  2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如FILL(VALUE, 1.23)，相应列为INT类型，则填充值为1。
+
-  3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。
+1. 不进行填充：NONE（默认填充模式）。
-  4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。
+2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1。
-  5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。
+3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。
-  6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。
+4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。
+5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。
+6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。
 
 :::info
 
@@ -93,6 +98,7 @@ SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m);
 ```
 
 使用时间窗口需要注意：
+
 - 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。
 - 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。
 - 返回的结果中时间序列严格单调递增。

docs/zh/12-taos-sql/22-information.md

@@ -0,0 +1,5 @@
+---
+sidebar_label: Information内置数据库
+title: Information内置数据库
+---
+

docs/zh/14-reference/03-connector/java.mdx

@@ -201,6 +201,10 @@ url 中的配置参数如下：
 - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。从 taos-jdbcdriver-2.0.38 和 TDengine 2.4.0.12 版本开始，JDBC REST 连接增加批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。
 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。
 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。
+- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。
+- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。
+- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。
+- useSSL: 连接中是否使用 SSL。
 
 **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。
 
@@ -264,7 +268,11 @@ properties 中的配置参数如下：
 - TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。
 - TSDBDriver.PROPERTY_KEY_LOCALE：仅在使用 JDBC 原生连接时生效。 客户端语言环境，默认值系统当前 locale。
 - TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。
-- 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。
+- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 5000。仅在 REST 连接时生效。
+- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 5000。仅在 REST 连接且 batchfetch 设置为 false 时生效。
+- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。
+- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。
+  此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。
 
 ### 配置参数的优先级
 
@@ -809,6 +817,7 @@ Query OK, 1 row(s) in set (0.000141s)
 
 | taos-jdbcdriver 版本 |            主要变化            |
 | :------------------: | :----------------------------: |
+|   2.0.39 - 2.0.40    |  增加 REST 连接/请求 超时设置  |
 |        2.0.38        | JDBC REST 连接增加批量拉取功能 |
 |        2.0.37        |      增加对 json tag 支持      |
 |        2.0.36        |   增加对 schemaless 写入支持   |

examples/c/tmq.c

@@ -28,8 +28,9 @@ static void msg_process(TAOS_RES* msg) {
   printf("db: %s\n", tmq_get_db_name(msg));
   printf("vg: %d\n", tmq_get_vgroup_id(msg));
   if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
-    tmq_raw_data* raw = tmq_get_raw_meta(msg);
+    tmq_raw_data raw = {0};
-    if (raw) {
+    int32_t code = tmq_get_raw_meta(msg, &raw);
+    if (code == 0) {
       TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0);
       if (pConn == NULL) {
         return;
@@ -53,7 +54,6 @@ static void msg_process(TAOS_RES* msg) {
       printf("write raw data: %s\n", tmq_err2str(ret));
       taos_close(pConn);
     }
-    tmq_free_raw_meta(raw);
     char* result = tmq_get_json_meta(msg);
     if (result) {
       printf("meta result: %s\n", result);

include/client/taos.h

@@ -259,13 +259,17 @@ enum tmq_res_t {
   TMQ_RES_TABLE_META = 2,
 };
 
+typedef struct tmq_raw_data{
+  void*    raw_meta;
+  uint32_t raw_meta_len;
+  uint16_t raw_meta_type;
+} tmq_raw_data;
+
 typedef enum tmq_res_t tmq_res_t;
-typedef struct tmq_raw_data tmq_raw_data;
 
 DLL_EXPORT tmq_res_t     tmq_get_res_type(TAOS_RES *res);
-DLL_EXPORT tmq_raw_data *tmq_get_raw_meta(TAOS_RES *res);
+DLL_EXPORT int32_t       tmq_get_raw_meta(TAOS_RES *res, tmq_raw_data *raw_meta);
-DLL_EXPORT int32_t       taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta);
+DLL_EXPORT int32_t       taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta);
-DLL_EXPORT void          tmq_free_raw_meta(tmq_raw_data *rawMeta);
 DLL_EXPORT char         *tmq_get_json_meta(TAOS_RES *res);   // Returning null means error. Returned result need to be freed by tmq_free_json_meta
 DLL_EXPORT void          tmq_free_json_meta(char* jsonMeta);
 DLL_EXPORT const char   *tmq_get_topic_name(TAOS_RES *res);

include/common/tcommon.h

@@ -80,8 +80,6 @@ typedef struct {
   SArray*   pTableList;
   SHashObj* map;  // speedup acquire the tableQueryInfo by table uid
   bool      needSortTableByGroupId;
-  void*     pTagCond;
-  void*     pTagIndexCond;
   uint64_t  suid;
 } STableListInfo;
 

include/common/tmsg.h

@@ -2071,9 +2071,10 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp);
 
 // TDMT_VND_ALTER_TABLE =====================
 typedef struct {
-  char*  tbName;
+  char*   tbName;
-  int8_t action;
+  int8_t  action;
-  char*  colName;
+  char*   colName;
+  int32_t colId;
   // TSDB_ALTER_TABLE_ADD_COLUMN
   int8_t  type;
   int8_t  flags;

include/common/ttime.h

@@ -63,12 +63,13 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
                       : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000
                                                                  : 1000000000;
   time_t     t = taosTime(NULL);
-  struct tm* tm = taosLocalTime(&t, NULL);
+  struct tm  tm;
-  tm->tm_hour = 0;
+  taosLocalTime(&t, &tm);
-  tm->tm_min = 0;
+  tm.tm_hour = 0;
-  tm->tm_sec = 0;
+  tm.tm_min = 0;
+  tm.tm_sec = 0;
 
-  return (int64_t)taosMktime(tm) * factor;
+  return (int64_t)taosMktime(&tm) * factor;
 }
 
 int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);

include/libs/executor/executor.h

@@ -40,6 +40,7 @@ typedef struct SReadHandle {
   bool    initMetaReader;
   bool    initTableReader;
   bool    initTqReader;
+  int32_t numOfVgroups;
 } SReadHandle;
 
 // in queue mode, data streams are seperated by msg

include/libs/nodes/plannodes.h

@@ -26,6 +26,12 @@ extern "C" {
 
 #define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN
 
+typedef enum EDataOrderLevel {
+  DATA_ORDER_LEVEL_NONE = 1,
+  DATA_ORDER_LEVEL_IN_BLOCK,
+  DATA_ORDER_LEVEL_IN_GROUP
+} EDataOrderLevel;
+
 typedef struct SLogicNode {
   ENodeType          type;
   SNodeList*         pTargets;  // SColumnNode
@@ -36,6 +42,8 @@ typedef struct SLogicNode {
   uint8_t            precision;
   SNode*             pLimit;
   SNode*             pSlimit;
+  EDataOrderLevel    requireDataOrder;  // requirements for input data
+  EDataOrderLevel    resultDataOrder;   // properties of the output data
 } SLogicNode;
 
 typedef enum EScanType {
@@ -78,7 +86,7 @@ typedef struct SScanLogicNode {
   SNodeList*    pGroupTags;
   bool          groupSort;
   int8_t        cacheLastMode;
-  bool          hasNormalCols; // neither tag column nor primary key tag column
+  bool          hasNormalCols;  // neither tag column nor primary key tag column
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -317,6 +325,7 @@ typedef STableScanPhysiNode SStreamScanPhysiNode;
 typedef struct SProjectPhysiNode {
   SPhysiNode node;
   SNodeList* pProjections;
+  bool       mergeDataBlock;
 } SProjectPhysiNode;
 
 typedef struct SIndefRowsFuncPhysiNode {

include/libs/qcom/query.h

@@ -162,9 +162,12 @@ typedef struct SRequestConnInfo {
   SEpSet   mgmtEps;
 } SRequestConnInfo;
 
+typedef void (*__freeFunc)(void *param);
+
 typedef struct SMsgSendInfo {
   __async_send_cb_fn_t fp;      // async callback function
   STargetInfo          target;  // for update epset
+  __freeFunc           paramFreeFp;
   void*                param;
   uint64_t             requestId;
   uint64_t             requestObjRefId;
@@ -188,6 +191,8 @@ int32_t cleanupTaskQueue();
  */
 int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
 
+void destroySendMsgInfo(SMsgSendInfo* pMsgBody);
+
 int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo,
                                 bool persistHandle, void* ctx);
 

include/libs/scalar/scalar.h

@@ -117,6 +117,12 @@ int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
 int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 #ifdef __cplusplus
 }

include/libs/stream/tstream.h

@@ -262,6 +262,7 @@ typedef struct SStreamTask {
   int64_t startVer;
   int64_t checkpointVer;
   int64_t processedVer;
+  int32_t numOfVgroups;
 
   // children info
   SArray* childEpInfo;  // SArray<SStreamChildEpInfo*>

include/libs/sync/sync.h

@@ -26,7 +26,7 @@ extern "C" {
 
 extern bool gRaftDetailLog;
 
-#define SYNC_RESP_TTL_MS 5000
+#define SYNC_RESP_TTL_MS 10000
 
 #define SYNC_MAX_BATCH_SIZE 500
 #define SYNC_INDEX_BEGIN 0

packaging/deb/makedeb.sh

@@ -73,7 +73,7 @@ cp ${compile_dir}/../include/client/taos.h          ${pkg_dir}${install_home_pat
 cp ${compile_dir}/../include/common/taosdef.h       ${pkg_dir}${install_home_path}/include
 cp ${compile_dir}/../include/util/taoserror.h       ${pkg_dir}${install_home_path}/include
 cp ${compile_dir}/../include/libs/function/taosudf.h       ${pkg_dir}${install_home_path}/include
-cp ${compile_dir}/../src/inc/taosws.h               ${pkg_dir}${install_home_path}/include ||:
+cp ${compile_dir}/build/include/taosws.h            ${pkg_dir}${install_home_path}/include ||:
 cp -r ${top_dir}/examples/*                         ${pkg_dir}${install_home_path}/examples
 #cp -r ${top_dir}/src/connector/python               ${pkg_dir}${install_home_path}/connector
 #cp -r ${top_dir}/src/connector/go                   ${pkg_dir}${install_home_path}/connector

packaging/rpm/tdengine.spec

@@ -80,7 +80,7 @@ cp %{_compiledir}/../include/client/taos.h          %{buildroot}%{homepath}/incl
 cp %{_compiledir}/../include/common/taosdef.h       %{buildroot}%{homepath}/include
 cp %{_compiledir}/../include/util/taoserror.h       %{buildroot}%{homepath}/include
 cp %{_compiledir}/../include/libs/function/taosudf.h       %{buildroot}%{homepath}/include
-cp %{_compiledir}/../src/inc/taosws.h               %{buildroot}%{homepath}/include ||:
+cp %{_compiledir}/build/include/taosws.h            %{buildroot}%{homepath}/include ||:
 #cp -r %{_compiledir}/../src/connector/python        %{buildroot}%{homepath}/connector
 #cp -r %{_compiledir}/../src/connector/go            %{buildroot}%{homepath}/connector
 #cp -r %{_compiledir}/../src/connector/nodejs        %{buildroot}%{homepath}/connector

packaging/tools/install.sh

@@ -229,13 +229,13 @@ function install_lib() {
   ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
   ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
 
-  ${csudo}ln -s ${lib_link_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so || :
+  [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
 
   if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
     ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
     ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
 
-    ${csudo}ln -s ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
+    ${csudo}ln -s ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
   fi
 
   ${csudo}ldconfig

packaging/tools/install_client.sh

@@ -116,6 +116,7 @@ function install_bin() {
 
 function clean_lib() {
     sudo rm -f /usr/lib/libtaos.* || :
+    sudo rm -f /usr/lib/libtaosws.* || :
     sudo rm -rf ${lib_dir} || :
 }
 
@@ -123,6 +124,9 @@ function install_lib() {
     # Remove links
     ${csudo}rm -f ${lib_link_dir}/libtaos.*         || :
     ${csudo}rm -f ${lib64_link_dir}/libtaos.*       || :
+
+    ${csudo}rm -f ${lib_link_dir}/libtaosws.*         || :
+    ${csudo}rm -f ${lib64_link_dir}/libtaosws.*       || :
     #${csudo}rm -rf ${v15_java_app_dir}              || :
 
     ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/*
@@ -131,13 +135,19 @@ function install_lib() {
         ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
         ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
 
+        [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so
+
         if [ -d "${lib64_link_dir}" ]; then
             ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1       || :
             ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so               || :
+
+            [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib64_link_dir}/libtaosws.so       || :
         fi
     else
         ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib
         ${csudo}ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib
+
+        [ -f ${install_main_dir}/driver/libtaosws.dylib ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib
     fi
 
     if [ "$osType" != "Darwin" ]; then
@@ -154,6 +164,8 @@ function install_header() {
     ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
     ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
     ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h    
+
+    [ -f ${install_main_dir}/include/taosws.h ] && ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taos.h
 }
 
 function install_jemalloc() {

packaging/tools/makeclient.sh

@@ -57,12 +57,16 @@ if [ "$osType" != "Darwin" ]; then
         ${script_dir}/get_client.sh"
   fi
   lib_files="${build_dir}/lib/libtaos.so.${version}"
+  wslib_files="${build_dir}/lib/libtaosws.so"
 else
   bin_files="${build_dir}/bin/${clientName} ${script_dir}/remove_client.sh"
   lib_files="${build_dir}/lib/libtaos.${version}.dylib"
+  wslib_files="${build_dir}/lib/libtaosws.dylib"
 fi
 
 header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
+wsheader_files="${build_dir}/include/taosws.h"
+
 if [ "$dbName" != "taos" ]; then
   cfg_dir="${top_dir}/../enterprise/packaging/cfg"
 else
@@ -74,6 +78,8 @@ install_files="${script_dir}/install_client.sh"
 # make directories.
 mkdir -p ${install_dir}
 mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
+[ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc
+
 mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile}
 mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/*
 
@@ -187,6 +193,7 @@ cp ${lib_files} ${install_dir}/driver
 # Copy connector
 connector_dir="${code_dir}/connector"
 mkdir -p ${install_dir}/connector
+[ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver
 
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   if [ "$osType" != "Darwin" ]; then

packaging/tools/makepkg.sh

@@ -96,7 +96,7 @@ lib_files="${build_dir}/lib/libtaos.so.${version}"
 wslib_files="${build_dir}/lib/libtaosws.so."
 header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
 
-wsheader_files="${code_dir}/inc/taosws.h"
+wsheader_files="${build_dir}/include/taosws.h"
 
 if [ "$dbName" != "taos" ]; then
   cfg_dir="${top_dir}/../enterprise/packaging/cfg"
@@ -114,7 +114,7 @@ init_file_rpm=${script_dir}/../rpm/taosd
 mkdir -p ${install_dir}
 mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
 
-${wsheader_files} ${install_dir}/inc || :
+[ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc || :
 
 mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile}
 

packaging/tools/post.sh

@@ -82,22 +82,33 @@ function kill_taosd() {
 
 function install_include() {
     ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || :
+    ${csudo}rm -f ${inc_link_dir}/taosws.h
+
     ${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h
     ${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h
     ${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h
     ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h    
+
+    [ -f ${inc_dir}/taosws.h ] && ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h ||:
 }
 
 function install_lib() {
     ${csudo}rm -f ${lib_link_dir}/libtaos* || :
     ${csudo}rm -f ${lib64_link_dir}/libtaos* || :
 
+    ${csudo}rm -f ${lib_link_dir}/libtaosws* || :
+    ${csudo}rm -f ${lib64_link_dir}/libtaosws* || :
+
     ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1
     ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
 
+    [ -f ${lib_dir}/libtaosws.so ]${csudo}ln -s ${lib_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so
+
     if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
       ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1           || :
       ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so   || :
+
+      [ -ff ${lib_dir}/libtaosws.so ] && ${csudo}ln -s ${lib_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so           || :
     fi
 
     ${csudo}ldconfig

source/client/src/clientHb.c

@@ -286,13 +286,10 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
   if (pInst == NULL || NULL == *pInst) {
     taosThreadMutexUnlock(&appInfo.mutex);
     tscError("cluster not exist, key:%s", key);
-    taosMemoryFreeClear(param);
     tFreeClientHbBatchRsp(&pRsp);
     return -1;
   }
 
-  taosMemoryFreeClear(param);
-
   if (code != 0) {
     (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1);
     tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, (*pInst)->totalDnodes);
@@ -716,6 +713,7 @@ static void *hbThreadFunc(void *param) {
       pInfo->msgInfo.len = tlen;
       pInfo->msgType = TDMT_MND_HEARTBEAT;
       pInfo->param = strdup(pAppHbMgr->key);
+      pInfo->paramFreeFp = taosMemoryFree;      
       pInfo->requestId = generateRequestId();
       pInfo->requestObjRefId = 0;
 

source/client/src/clientImpl.c

@@ -29,7 +29,6 @@
 
 static int32_t       initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
 static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
-static void          destroySendMsgInfo(SMsgSendInfo* pMsgBody);
 
 static bool stringLengthCheck(const char* str, size_t maxsize) {
   if (str == NULL) {
@@ -1215,13 +1214,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
   return pMsgSendInfo;
 }
 
-static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
-  assert(pMsgBody != NULL);
-  taosMemoryFreeClear(pMsgBody->target.dbFName);
-  taosMemoryFreeClear(pMsgBody->msgInfo.pData);
-  taosMemoryFreeClear(pMsgBody);
-}
-
 void updateTargetEpSet(SMsgSendInfo* pSendInfo, STscObj* pTscObj, SRpcMsg* pMsg, SEpSet* pEpSet) {
   if (NULL == pEpSet) {
     return;
@@ -2101,8 +2093,9 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
 
   taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
   tsem_wait(&param->sem);
-
+  if (param->pRequest != NULL) {
-  param->pRequest->syncQuery = true;
+    param->pRequest->syncQuery = true;
+  }
   return param->pRequest;
 #else
   size_t sqlLen = strlen(sql);

source/client/src/clientMsgHandler.c

@@ -255,6 +255,8 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
     catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid);
   }
 
+  taosMemoryFree(pMsg->pData);
+
   if (pRequest->body.queryFp != NULL) {
     pRequest->body.queryFp(pRequest->body.param, pRequest, code);
   } else {
@@ -278,6 +280,8 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) {
     pRequest->body.resInfo.execRes.res = alterRsp.pMeta;
   }
 
+  taosMemoryFree(pMsg->pData);
+
   if (pRequest->body.queryFp != NULL) {
     SExecResult* pRes = &pRequest->body.resInfo.execRes;
 
@@ -387,6 +391,8 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
     tFreeSShowVariablesRsp(&rsp);
   }
 
+  taosMemoryFree(pMsg->pData);
+
   if (pRequest->body.queryFp != NULL) {
     pRequest->body.queryFp(pRequest->body.param, pRequest, code);
   } else {

source/client/src/clientSml.c

@@ -268,7 +268,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
     *actionNeeded = true;
   }
   if (*actionNeeded) {
-    uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, colField->name,
+    uDebug("SML:0x%" PRIx64 " generate schema action. kv->name: %s, action: %d", info->id, kv->key,
            action->action);
   }
   return 0;
@@ -436,6 +436,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
                                       SSchemaAction *action, bool isTag) {
   int32_t code = TSDB_CODE_SUCCESS;
   for (int j = 0; j < taosArrayGetSize(cols); ++j) {
+    if(j == 0 && !isTag) continue;
     SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j);
     bool    actionNeeded = false;
     code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, &actionNeeded, info);
@@ -452,18 +453,25 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols) {
+static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) {
   SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
-  for (uint16_t i = 0; i < length; i++) {
+  int32_t i = 0;
+  for ( ;i < length; i++) {
     taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES);
   }
 
-  for (int32_t i = 0; i < taosArrayGetSize(cols); i++) {
+  if (isTag){
+    i = 0;
+  } else {
+    i = 1;
+  }
+  for (; i < taosArrayGetSize(cols); i++) {
     SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
     if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) {
       return -1;
     }
   }
+  taosHashCleanup(hashTmp);
   return 0;
 }
 
@@ -523,7 +531,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
       }
 
       taosHashClear(hashTmp);
-      for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
+      for (uint16_t i = 1; i < pTableMeta->tableInfo.numOfColumns; i++) {
         taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES);
       }
       code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &schemaAction, false);
@@ -551,12 +559,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
 
     if (needCheckMeta) {
       code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags,
-                          sTableData->tags);
+                          sTableData->tags, true);
       if (code != TSDB_CODE_SUCCESS) {
         uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, (char *)superTable);
         goto end;
       }
-      code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols);
+      code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false);
       if (code != TSDB_CODE_SUCCESS) {
         uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, (char *)superTable);
         goto end;
@@ -832,6 +840,7 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t
 static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) {
   int64_t ts = 0;
   if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
+//    uError("SML:data:%s,len:%d", data, len);
     ts = smlParseInfluxTime(info, data, len);
   } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
     ts = smlParseOpenTsdbTime(info, data, len);
@@ -1208,6 +1217,9 @@ static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *c
     kv->value = value;
     kv->length = valueLen;
     if (isTag) {
+      if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+        return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      }
       kv->type = TSDB_DATA_TYPE_NCHAR;
     } else {
       int32_t ret = smlParseValue(kv, msg);
@@ -2031,6 +2043,8 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *
 
 static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) {
   SSmlLineInfo elements = {0};
+  uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql);
+
   int          ret = smlParseInfluxString(sql, &elements, &info->msgBuf);
   if (ret != TSDB_CODE_SUCCESS) {
     uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id);

source/client/src/tmq.c

@@ -113,12 +113,6 @@ struct tmq_t {
   tsem_t rspSem;
 };
 
-struct tmq_raw_data {
-  void*   raw_meta;
-  int32_t raw_meta_len;
-  int16_t raw_meta_type;
-};
-
 enum {
   TMQ_VG_STATUS__IDLE = 0,
   TMQ_VG_STATUS__WAIT,
@@ -363,7 +357,11 @@ tmq_list_t* tmq_list_new() {
 
 int32_t tmq_list_append(tmq_list_t* list, const char* src) {
   SArray* container = &list->container;
-  char*   topic = strdup(src);
+  if (src == NULL || src[0] == 0) return -1;
+  char* topic = strdup(src);
+  if (topic[0] != '`') {
+    strtolower(topic, src);
+  }
   if (taosArrayPush(container, &topic) == NULL) return -1;
   return 0;
 }
@@ -506,6 +504,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
   pMsgSendInfo->requestId = generateRequestId();
   pMsgSendInfo->requestObjRefId = 0;
   pMsgSendInfo->param = pParam;
+  pMsgSendInfo->paramFreeFp = taosMemoryFree;      
   pMsgSendInfo->fp = tmqCommitCb2;
   pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET;
   // send msg
@@ -1914,16 +1913,15 @@ const char* tmq_get_table_name(TAOS_RES* res) {
   return NULL;
 }
 
-tmq_raw_data* tmq_get_raw_meta(TAOS_RES* res) {
+int32_t tmq_get_raw_meta(TAOS_RES* res, tmq_raw_data *raw) {
-  if (TD_RES_TMQ_META(res)) {
+  if (TD_RES_TMQ_META(res) && raw) {
-    tmq_raw_data*  raw = taosMemoryCalloc(1, sizeof(tmq_raw_data));
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
     raw->raw_meta = pMetaRspObj->metaRsp.metaRsp;
     raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen;
     raw->raw_meta_type = pMetaRspObj->metaRsp.resMsgType;
-    return raw;
+    return TSDB_CODE_SUCCESS;
   }
-  return NULL;
+  return TSDB_CODE_INVALID_PARA;
 }
 
 static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
@@ -2195,17 +2193,19 @@ static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray*
     cJSON* ttype = cJSON_CreateNumber(pTagVal->type);
     cJSON_AddItemToObject(tag, "type", ttype);
 
-    char* buf = NULL;
+    cJSON* tvalue = NULL;
     if (IS_VAR_DATA_TYPE(pTagVal->type)) {
-      buf = taosMemoryCalloc(pTagVal->nData + 1, 1);
+      char* buf = taosMemoryCalloc(pTagVal->nData + 3, 1);
+      if(!buf) goto end;
       dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL);
+      tvalue = cJSON_CreateString(buf);
+      taosMemoryFree(buf);
     } else {
-      buf = taosMemoryCalloc(32, 1);
+      double val = 0;
-      dataConverToStr(buf, pTagVal->type, &pTagVal->i64, tDataTypes[pTagVal->type].bytes, NULL);
+      GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64);
+      tvalue = cJSON_CreateNumber(val);
     }
 
-    cJSON* tvalue = cJSON_CreateString(buf);
-    taosMemoryFree(buf);
     cJSON_AddItemToObject(tag, "value", tvalue);
     cJSON_AddItemToArray(tags, tag);
   }
@@ -2931,23 +2931,23 @@ end:
   return code;
 }
 
-int32_t taos_write_raw_meta(TAOS* taos, tmq_raw_data* raw_meta) {
+int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta){
-  if (!taos || !raw_meta) {
+  if (!taos) {
     return TSDB_CODE_INVALID_PARA;
   }
 
-  if (raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) {
+  if(raw_meta.raw_meta_type == TDMT_VND_CREATE_STB) {
-    return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+    return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
-  } else if (raw_meta->raw_meta_type == TDMT_VND_ALTER_STB) {
+  }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_STB){
-    return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+    return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
-  } else if (raw_meta->raw_meta_type == TDMT_VND_DROP_STB) {
+  }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_STB){
-    return taosDropStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+    return taosDropStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
-  } else if (raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE) {
+  }else if(raw_meta.raw_meta_type == TDMT_VND_CREATE_TABLE){
-    return taosCreateTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+    return taosCreateTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
-  } else if (raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE) {
+  }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_TABLE){
-    return taosAlterTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+    return taosAlterTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
-  } else if (raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE) {
+  }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_TABLE){
-    return taosDropTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+    return taosDropTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len);
   }
   return TSDB_CODE_INVALID_PARA;
 }

source/common/src/tdatablock.c

@@ -228,7 +228,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
   uint32_t finalNumOfRows = numOfRow1 + numOfRow2;
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     // Handle the bitmap
-    if (finalNumOfRows > *capacity || numOfRow1 == 0) {
+    if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) {
       char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
       if (p == NULL) {
         return TSDB_CODE_OUT_OF_MEMORY;
@@ -262,7 +262,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
     memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len);
     pColumnInfoData->varmeta.length = len + oldLen;
   } else {
-    if (finalNumOfRows > *capacity || numOfRow1 == 0) {
+    if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) {
       ASSERT(finalNumOfRows * pColumnInfoData->info.bytes);
       char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes);
       if (tmp == NULL) {
@@ -1661,9 +1661,6 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
     }
     */
 
-#ifdef WINDOWS
-  if (tt < 0) tt = 0;
-#endif
   if (tt <= 0 && ms < 0) {
     tt--;
     if (precision == TSDB_TIME_PRECISION_NANO) {
@@ -1674,9 +1671,9 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
       ms += 1000;
     }
   }
-
+  struct tm ptm = {0};
-  struct tm* ptm = taosLocalTime(&tt, NULL);
+  taosLocalTime(&tt, &ptm);
-  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
+  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
 
   if (precision == TSDB_TIME_PRECISION_NANO) {
     sprintf(buf + pos, ".%09d", ms);

source/common/src/tglobal.c

@@ -556,7 +556,7 @@ static void taosSetSystemCfg(SConfig *pCfg) {
   osSetSystemLocale(locale, charset);
 
   bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
-  taosSetConsoleEcho(enableCore);
+  taosSetCoreDump(enableCore);
 
   // todo
   tsVersion = 30000000;
@@ -675,7 +675,7 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
     case 'e': {
       if (strcasecmp("enableCoreFile", name) == 0) {
         bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
-        taosSetConsoleEcho(enableCore);
+        taosSetCoreDump(enableCore);
       }
       break;
     }

source/common/src/ttime.c

@@ -902,7 +902,7 @@ const char* fmtts(int64_t ts) {
 
 void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) {
   char       ts[40] = {0};
-  struct tm* ptm;
+  struct tm  ptm;
 
   int32_t fractionLen;
   char*   format = NULL;
@@ -939,10 +939,10 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
       assert(false);
   }
 
-  ptm = taosLocalTime(&quot, NULL);
+  taosLocalTime(&quot, &ptm);
-  int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm);
+  int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm);
   length += snprintf(ts + length, fractionLen, format, mod);
-  length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm);
+  length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm);
 
   tstrncpy(buf, ts, bufLen);
 }

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -49,9 +49,9 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
         dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
       }
     }
-    rpcFreeCont(pRsp->pCont);
     tFreeSStatusRsp(&statusRsp);
   }
+  rpcFreeCont(pRsp->pCont);
 }
 
 void dmSendStatusReq(SDnodeMgmt *pMgmt) {

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -179,6 +179,16 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
       } else {
         dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
         taosWriteQitem(pVnode->pWriteQ, pMsg);
+#if 0  // tests for batch writes
+        if (pMsg->msgType == TDMT_VND_CREATE_TABLE) {
+          SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
+          memcpy(pDup, pMsg, sizeof(SRpcMsg));
+          pDup->pCont = rpcMallocCont(pMsg->contLen);
+          memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen);
+          pDup->info.handle = NULL;
+          taosWriteQitem(pVnode->pWriteQ, pDup);
+        }
+#endif
       }
       break;
     case SYNC_QUEUE:

source/dnode/mnode/impl/inc/mndDef.h

@@ -35,6 +35,46 @@
 extern "C" {
 #endif
 
+typedef enum {
+  MND_OPER_CONNECT = 1,
+  MND_OPER_CREATE_ACCT,
+  MND_OPER_DROP_ACCT,
+  MND_OPER_ALTER_ACCT,
+  MND_OPER_CREATE_USER,
+  MND_OPER_DROP_USER,
+  MND_OPER_ALTER_USER,
+  MND_OPER_CREATE_BNODE,
+  MND_OPER_DROP_BNODE,
+  MND_OPER_CREATE_DNODE,
+  MND_OPER_DROP_DNODE,
+  MND_OPER_CONFIG_DNODE,
+  MND_OPER_CREATE_MNODE,
+  MND_OPER_DROP_MNODE,
+  MND_OPER_CREATE_QNODE,
+  MND_OPER_DROP_QNODE,
+  MND_OPER_CREATE_SNODE,
+  MND_OPER_DROP_SNODE,
+  MND_OPER_REDISTRIBUTE_VGROUP,
+  MND_OPER_MERGE_VGROUP,
+  MND_OPER_SPLIT_VGROUP,
+  MND_OPER_BALANCE_VGROUP,
+  MND_OPER_CREATE_FUNC,
+  MND_OPER_DROP_FUNC,
+  MND_OPER_KILL_TRANS,
+  MND_OPER_KILL_CONN,
+  MND_OPER_KILL_QUERY,
+  MND_OPER_CREATE_DB,
+  MND_OPER_ALTER_DB,
+  MND_OPER_DROP_DB,
+  MND_OPER_COMPACT_DB,
+  MND_OPER_TRIM_DB,
+  MND_OPER_USE_DB,
+  MND_OPER_WRITE_DB,
+  MND_OPER_READ_DB,
+  MND_OPER_READ_OR_WRITE_DB,
+  MND_OPER_SHOW_VARIBALES,
+} EOperType;
+
 typedef enum {
   MND_AUTH_ACCT_START = 0,
   MND_AUTH_ACCT_USER,
@@ -109,9 +149,9 @@ typedef struct {
   ETrnPolicy     policy;
   ETrnConflct    conflict;
   ETrnExec       exec;
+  EOperType      oper;
   int32_t        code;
   int32_t        failedTimes;
-  SRpcHandleInfo rpcInfo;
   void*          rpcRsp;
   int32_t        rpcRspLen;
   int32_t        redoActionPos;
@@ -130,6 +170,7 @@ typedef struct {
   int32_t        stopFunc;
   int32_t        paramLen;
   void*          param;
+  SArray*        pRpcArray;
 } STrans;
 
 typedef struct {

source/dnode/mnode/impl/inc/mndPrivilege.h

@@ -22,46 +22,6 @@
 extern "C" {
 #endif
 
-typedef enum {
-  MND_OPER_CONNECT = 1,
-  MND_OPER_CREATE_ACCT,
-  MND_OPER_DROP_ACCT,
-  MND_OPER_ALTER_ACCT,
-  MND_OPER_CREATE_USER,
-  MND_OPER_DROP_USER,
-  MND_OPER_ALTER_USER,
-  MND_OPER_CREATE_BNODE,
-  MND_OPER_DROP_BNODE,
-  MND_OPER_CREATE_DNODE,
-  MND_OPER_DROP_DNODE,
-  MND_OPER_CONFIG_DNODE,
-  MND_OPER_CREATE_MNODE,
-  MND_OPER_DROP_MNODE,
-  MND_OPER_CREATE_QNODE,
-  MND_OPER_DROP_QNODE,
-  MND_OPER_CREATE_SNODE,
-  MND_OPER_DROP_SNODE,
-  MND_OPER_REDISTRIBUTE_VGROUP,
-  MND_OPER_MERGE_VGROUP,
-  MND_OPER_SPLIT_VGROUP,
-  MND_OPER_BALANCE_VGROUP,
-  MND_OPER_CREATE_FUNC,
-  MND_OPER_DROP_FUNC,
-  MND_OPER_KILL_TRANS,
-  MND_OPER_KILL_CONN,
-  MND_OPER_KILL_QUERY,
-  MND_OPER_CREATE_DB,
-  MND_OPER_ALTER_DB,
-  MND_OPER_DROP_DB,
-  MND_OPER_COMPACT_DB,
-  MND_OPER_TRIM_DB,
-  MND_OPER_USE_DB,
-  MND_OPER_WRITE_DB,
-  MND_OPER_READ_DB,
-  MND_OPER_READ_OR_WRITE_DB,
-  MND_OPER_SHOW_VARIBALES,
-} EOperType;
-
 int32_t mndInitPrivilege(SMnode *pMnode);
 void    mndCleanupPrivilege(SMnode *pMnode);
 

source/dnode/mnode/impl/inc/mndTrans.h

@@ -73,12 +73,14 @@ void    mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
 void    mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen);
 void    mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2);
 void    mndTransSetSerial(STrans *pTrans);
+void    mndTransSetOper(STrans *pTrans, EOperType oper);
 
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
 int32_t mndTransProcessRsp(SRpcMsg *pRsp);
 void    mndTransPullup(SMnode *pMnode);
 int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans);
 void    mndTransExecute(SMnode *pMnode, STrans *pTrans);
+int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndDb.c

@@ -487,6 +487,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
   mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db);
 
   mndTransSetDbName(pTrans, dbObj.name, NULL);
+  mndTransSetOper(pTrans, MND_OPER_CREATE_DB);
   if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
   if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
   if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER;
@@ -534,6 +535,14 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
       terrno = TSDB_CODE_MND_DB_ALREADY_EXIST;
       goto _OVER;
     }
+  } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) {
+    if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) {
+      mDebug("db:%s, is creating and response after trans finished", createReq.db);
+      code = TSDB_CODE_ACTION_IN_PROGRESS;
+      goto _OVER;
+    } else {
+      goto _OVER;
+    }
   } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) {
     goto _OVER;
   }

source/dnode/mnode/impl/src/mndScheduler.c

@@ -383,6 +383,11 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
       // exec
       pInnerTask->execType = TASK_EXEC__PIPE;
 
+      SDbObj* pSourceDb = mndAcquireDb(pMnode, pStream->sourceDb);
+      ASSERT(pDbObj != NULL);
+      sdbRelease(pSdb, pSourceDb);
+      pInnerTask->numOfVgroups = pSourceDb->cfg.numOfVgroups;
+
       if (tsSchedStreamToSnode) {
         SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode);
         if (pSnode == NULL) {

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -298,7 +298,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
                 .pVgEp = pVgEp,
             };
             taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-            mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId);
+            mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId,
+                  pConsumerEp->consumerId);
           }
           imbCnt++;
         }
@@ -312,7 +313,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
               .pVgEp = pVgEp,
           };
           taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-          mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId);
+          mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId,
+                pConsumerEp->consumerId);
         }
       }
     }
@@ -354,7 +356,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
       pRebVg->newConsumerId = pConsumerEp->consumerId;
       taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId,
+            pConsumerEp->consumerId);
     }
   }
 
@@ -371,8 +374,14 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       ASSERT(pConsumerEp->consumerId > 0);
       taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
       pRebVg->newConsumerId = pConsumerEp->consumerId;
+      if (pRebVg->newConsumerId == pRebVg->oldConsumerId) {
+        mInfo("mq rebalance: skip vg %d for same consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId,
+              pConsumerEp->consumerId);
+        continue;
+      }
       taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId,
+            pConsumerEp->consumerId);
     }
   } else {
     // if all consumer is removed, put all vg into unassigned

source/dnode/mnode/impl/src/mndSync.c

@@ -144,9 +144,11 @@ int32_t mndSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWrit
 }
 
 int32_t mndSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) {
-  mInfo("stop to apply snapshot to sdb, apply:%d", isApply);
+  mInfo("stop to apply snapshot to sdb, apply:%d, index:%" PRId64 " term:%" PRIu64 " config:%" PRId64, isApply,
+        pSnapshot->lastApplyIndex, pSnapshot->lastApplyTerm, pSnapshot->lastApplyIndex);
   SMnode *pMnode = pFsm->data;
-  return sdbStopWrite(pMnode->pSdb, pWriter, isApply);
+  return sdbStopWrite(pMnode->pSdb, pWriter, isApply, pSnapshot->lastApplyIndex, pSnapshot->lastApplyTerm,
+                      pSnapshot->lastConfigIndex);
 }
 
 int32_t mndSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) {
@@ -157,7 +159,7 @@ int32_t mndSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int
 void mndLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   SMnode *pMnode = pFsm->data;
   atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 1);
-  mDebug("vgId:1, mnd leader transfer finish");
+  mDebug("vgId:1, mnode leader transfer finish");
 }
 
 SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) {

source/dnode/mnode/impl/src/mndTrans.c

@@ -122,6 +122,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
   SDB_SET_INT8(pRaw, dataPos, pTrans->policy, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pTrans->conflict, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pTrans->exec, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, pTrans->oper, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, 0, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, 0, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, 0, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER)
   SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER)
   SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER)
@@ -269,15 +273,22 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
   int8_t policy = 0;
   int8_t conflict = 0;
   int8_t exec = 0;
+  int8_t oper = 0;
+  int8_t reserved = 0;
   int8_t actionType = 0;
   SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &policy, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &conflict, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &exec, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &oper, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER)
   pTrans->stage = stage;
   pTrans->policy = policy;
   pTrans->conflict = conflict;
   pTrans->exec = exec;
+  pTrans->oper = oper;
   SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER)
   SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER)
   SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER)
@@ -495,6 +506,10 @@ static void mndTransDropData(STrans *pTrans) {
     mndTransDropActions(pTrans->commitActions);
     pTrans->commitActions = NULL;
   }
+  if (pTrans->pRpcArray != NULL) {
+    taosArrayDestroy(pTrans->pRpcArray);
+    pTrans->pRpcArray = NULL;
+  }
   if (pTrans->rpcRsp != NULL) {
     taosMemoryFree(pTrans->rpcRsp);
     pTrans->rpcRsp = NULL;
@@ -585,14 +600,18 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
   pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
   pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
   pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
+  pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo));
 
-  if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL) {
+  if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL ||
+      pTrans->pRpcArray == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     mError("failed to create transaction since %s", terrstr());
     return NULL;
   }
 
-  if (pReq != NULL) pTrans->rpcInfo = pReq->info;
+  if (pReq != NULL) {
+    taosArrayPush(pTrans->pRpcArray, &pReq->info);
+  }
   mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans);
   return pTrans;
 }
@@ -677,6 +696,31 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *
   pTrans->paramLen = paramLen;
 }
 
+int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname) {
+  STrans *pTrans = NULL;
+  void   *pIter = NULL;
+  int32_t code = -1;
+
+  while (1) {
+    pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans);
+    if (pIter == NULL) break;
+
+    if (pTrans->oper == oper) {
+      if (strcasecmp(dbname, pTrans->dbname1) == 0) {
+        mDebug("trans:%d, db:%s oper:%d matched with input", pTrans->id, dbname, oper);
+        if (taosArrayPush(pTrans->pRpcArray, &pMsg->info) != NULL) {
+          code = 0;
+        }
+        sdbRelease(pMnode->pSdb, pTrans);
+        break;
+      }
+    }
+
+    sdbRelease(pMnode->pSdb, pTrans);
+  }
+  return code;
+}
+
 void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2) {
   if (dbname1 != NULL) {
     memcpy(pTrans->dbname1, dbname1, TSDB_DB_FNAME_LEN);
@@ -688,6 +732,8 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2)
 
 void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }
 
+void mndTransSetOper(STrans *pTrans, EOperType oper) { pTrans->oper = oper; }
+
 static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
   SSdbRaw *pRaw = mndTransActionEncode(pTrans);
   if (pRaw == NULL) {
@@ -711,7 +757,7 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
 
 static bool mndCheckDbConflict(const char *db, STrans *pTrans) {
   if (db[0] == 0) return false;
-  if (strcmp(db, pTrans->dbname1) == 0 || strcmp(db, pTrans->dbname2) == 0) return true;
+  if (strcasecmp(db, pTrans->dbname1) == 0 || strcasecmp(db, pTrans->dbname2) == 0) return true;
   return false;
 }
 
@@ -784,9 +830,10 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
     return -1;
   }
 
-  pNew->rpcInfo = pTrans->rpcInfo;
+  pNew->pRpcArray = pTrans->pRpcArray;
   pNew->rpcRsp = pTrans->rpcRsp;
   pNew->rpcRspLen = pTrans->rpcRspLen;
+  pTrans->pRpcArray = NULL;
   pTrans->rpcRsp = NULL;
   pTrans->rpcRspLen = 0;
 
@@ -835,29 +882,34 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
     }
   }
 
-  if (sendRsp && pTrans->rpcInfo.handle != NULL) {
+  if (!sendRsp) return;
-    mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage),
-           pTrans->rpcInfo.ahandle);
-    if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
-      code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL;
-    }
-    SRpcMsg rspMsg = {.code = code, .info = pTrans->rpcInfo};
 
-    if (pTrans->rpcRspLen != 0) {
+  int32_t size = taosArrayGetSize(pTrans->pRpcArray);
-      void *rpcCont = rpcMallocCont(pTrans->rpcRspLen);
+  if (size <= 0) return;
-      if (rpcCont != NULL) {
+
-        memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen);
+  for (int32_t i = 0; i < size; ++i) {
-        rspMsg.pCont = rpcCont;
+    SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i);
-        rspMsg.contLen = pTrans->rpcRspLen;
+    if (pInfo->handle != NULL) {
+      mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage),
+             pInfo->ahandle);
+      if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+        code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL;
       }
-      taosMemoryFree(pTrans->rpcRsp);
+      SRpcMsg rspMsg = {.code = code, .info = *pInfo};
-    }
 
-    tmsgSendRsp(&rspMsg);
+      if (pTrans->rpcRspLen != 0) {
-    pTrans->rpcInfo.handle = NULL;
+        void *rpcCont = rpcMallocCont(pTrans->rpcRspLen);
-    pTrans->rpcRsp = NULL;
+        if (rpcCont != NULL) {
-    pTrans->rpcRspLen = 0;
+          memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen);
+          rspMsg.pCont = rpcCont;
+          rspMsg.contLen = pTrans->rpcRspLen;
+        }
+      }
+
+      tmsgSendRsp(&rspMsg);
+    }
   }
+  taosArrayClear(pTrans->pRpcArray);
 }
 
 int32_t mndTransProcessRsp(SRpcMsg *pRsp) {

source/dnode/mnode/impl/test/sdb/sdbTest.cpp

@@ -925,7 +925,7 @@ TEST_F(MndTestSdb, 01_Read_Str) {
     }
 
     sdbStopRead(pSdb, pReader);
-    sdbStopWrite(pSdb, pWritter, true);
+    sdbStopWrite(pSdb, pWritter, true, -1, -1, -1);
   }
 
   ASSERT_EQ(sdbGetSize(pSdb, SDB_CONSUMER), 1);

source/dnode/mnode/sdb/inc/sdb.h

@@ -394,7 +394,7 @@ int32_t sdbStopRead(SSdb *pSdb, SSdbIter *pIter);
 int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len);
 
 int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter);
-int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply);
+int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, int64_t term, int64_t config);
 int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len);
 
 const char *sdbTableName(ESdbType type);

source/dnode/mnode/sdb/src/sdbFile.c

@@ -613,7 +613,7 @@ int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) {
   return 0;
 }
 
-int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply) {
+int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, int64_t term, int64_t config) {
   int32_t code = 0;
 
   if (!isApply) {
@@ -641,6 +641,19 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply) {
     return -1;
   }
 
+  if (config > 0) {
+    ASSERT(pSdb->commitConfig == config);
+    pSdb->commitConfig = config;
+  }
+  if (term > 0) {
+    ASSERT(pSdb->commitTerm == term);
+    pSdb->commitTerm = term;
+  }
+  if (index > 0) {
+    ASSERT(pSdb->commitIndex == index);
+    pSdb->commitIndex = index;
+  }
+
   mDebug("sdbiter:%p, successfully applyed to sdb", pIter);
   return 0;
 }

source/dnode/mnode/sdb/src/sdbHash.c

@@ -153,23 +153,22 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
     return terrno;
   }
 
-  taosThreadRwlockUnlock(pLock);
-
   int32_t     code = 0;
   SdbInsertFp insertFp = pSdb->insertFps[pRow->type];
   if (insertFp != NULL) {
     code = (*insertFp)(pSdb, pRow->pObj);
     if (code != 0) {
       code = terrno;
-      taosThreadRwlockWrlock(pLock);
       taosHashRemove(hash, pRow->pObj, keySize);
-      taosThreadRwlockUnlock(pLock);
       sdbFreeRow(pSdb, pRow, false);
       terrno = code;
+      taosThreadRwlockUnlock(pLock);
       return terrno;
     }
   }
 
+  taosThreadRwlockUnlock(pLock);
+
   if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) {
     pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj));
   }
@@ -194,7 +193,6 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   SSdbRow *pOldRow = *ppOldRow;
   pOldRow->status = pRaw->status;
   sdbPrintOper(pSdb, pOldRow, "update");
-  taosThreadRwlockUnlock(pLock);
 
   int32_t     code = 0;
   SdbUpdateFp updateFp = pSdb->updateFps[pNewRow->type];
@@ -202,6 +200,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
     code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj);
   }
 
+  taosThreadRwlockUnlock(pLock);
   sdbFreeRow(pSdb, pNewRow, false);
 
   pSdb->tableVer[pOldRow->type]++;

source/dnode/vnode/src/inc/sma.h

@@ -43,9 +43,9 @@ typedef struct SRSmaInfo     SRSmaInfo;
 typedef struct SRSmaInfoItem SRSmaInfoItem;
 
 struct SSmaEnv {
-  SRWLatch       lock;
+  SRWLatch  lock;
-  int8_t         type;
+  int8_t    type;
-  SSmaStat      *pStat;
+  SSmaStat *pStat;
 };
 
 typedef struct {
@@ -103,10 +103,15 @@ struct SRSmaInfoItem {
 };
 
 struct SRSmaInfo {
-  STSchema     *pTSchema;
+  STSchema *pTSchema;
-  int64_t       suid;
+  int64_t   suid;
+  int8_t    delFlag;
+  T_REF_DECLARE()
   SRSmaInfoItem items[TSDB_RETENTION_L2];
 };
+#define RSMA_INFO_HEAD_LEN   24
+#define RSMA_INFO_IS_DEL(r)  ((r)->delFlag == 1)
+#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1)
 
 enum {
   TASK_TRIGGER_STAT_INIT = 0,
@@ -120,8 +125,8 @@ enum {
 enum {
   RSMA_ROLE_CREATE = 0,
   RSMA_ROLE_DROP = 1,
-  RSMA_ROLE_FETCH = 2,
+  RSMA_ROLE_SUBMIT = 2,
-  RSMA_ROLE_SUBMIT = 3,
+  RSMA_ROLE_FETCH = 3,
   RSMA_ROLE_ITERATE = 4,
 };
 
@@ -134,6 +139,8 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg);
 
 int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
 int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
+int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo);
+int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo);
 
 void   *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
 int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
@@ -193,6 +200,7 @@ void           tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t le
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree);
+void           tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid);
 int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
 
 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
@@ -258,8 +266,9 @@ void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
 void    tdCloseTFile(STFile *pTFile);
 void    tdDestroyTFile(STFile *pTFile);
 
-void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName);
+void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version,
-void tdGetVndDirName(int32_t vgId,const char *pdname,  const char *dname, bool endWithSep, char *outputName);
+                      char *outputName);
+void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/tsdb.h

@@ -174,6 +174,10 @@ void    tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg);
 int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
 void    tsdbMemTableDestroy(SMemTable *pMemTable);
 void    tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
+void    tsdbRefMemTable(SMemTable *pMemTable);
+void    tsdbUnrefMemTable(SMemTable *pMemTable);
+int32_t tsdbTakeMemSnapshot(STsdb *pTsdb, SMemTable **ppMem, SMemTable **ppIMem);
+void    tsdbUntakeMemSnapshot(STsdb *pTsdb, SMemTable *pMem, SMemTable *pIMem);
 // STbDataIter
 int32_t  tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
 void    *tsdbTbDataIterDestroy(STbDataIter *pIter);
@@ -186,6 +190,7 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData);
 typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
 void    tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]);
 bool    tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype);
+bool    tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
 int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype);
 int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
 int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype);
@@ -272,23 +277,14 @@ typedef struct {
 } SRtn;
 
 struct STsdb {
-  char         *path;
+  char          *path;
-  SVnode       *pVnode;
+  SVnode        *pVnode;
-  TdThreadMutex mutex;
+  STsdbKeepCfg   keepCfg;
-  bool          repoLocked;
+  TdThreadRwlock rwLock;
-  STsdbKeepCfg  keepCfg;
+  SMemTable     *mem;
-  SMemTable    *mem;
+  SMemTable     *imem;
-  SMemTable    *imem;
+  STsdbFS       *pFS;
-  SRtn          rtn;
+  SLRUCache     *lruCache;
-  STsdbFS      *fs;
-  SLRUCache    *lruCache;
-};
-
-struct STable {
-  uint64_t  suid;
-  uint64_t  uid;
-  STSchema *pSchema;       // latest schema
-  STSchema *pCacheSchema;  // cached cache
 };
 
 struct TSDBKEY {
@@ -329,21 +325,19 @@ struct STbData {
 };
 
 struct SMemTable {
-  SRWLatch latch;
+  SRWLatch         latch;
-  STsdb   *pTsdb;
+  STsdb           *pTsdb;
-  int32_t  nRef;
+  SVBufPool       *pPool;
-  TSKEY    minKey;
+  volatile int32_t nRef;
-  TSKEY    maxKey;
+  TSKEY            minKey;
-  int64_t  minVersion;
+  TSKEY            maxKey;
-  int64_t  maxVersion;
+  int64_t          minVersion;
-  int64_t  nRow;
+  int64_t          maxVersion;
-  int64_t  nDel;
+  int64_t          nRow;
-  SArray  *aTbData;  // SArray<STbData*>
+  int64_t          nDel;
+  SArray          *aTbData;  // SArray<STbData*>
 };
 
-int tsdbLockRepo(STsdb *pTsdb);
-int tsdbUnlockRepo(STsdb *pTsdb);
-
 struct TSDBROW {
   int8_t type;  // 0 for row from tsRow, 1 for row from block data
   union {

source/dnode/vnode/src/inc/vnd.h

@@ -62,12 +62,13 @@ struct SVBufPoolNode {
 };
 
 struct SVBufPool {
-  SVBufPool*     next;
+  SVBufPool*       next;
-  int64_t        nRef;
+  SVnode*          pVnode;
-  int64_t        size;
+  volatile int32_t nRef;
-  uint8_t*       ptr;
+  int64_t          size;
-  SVBufPoolNode* pTail;
+  uint8_t*         ptr;
-  SVBufPoolNode  node;
+  SVBufPoolNode*   pTail;
+  SVBufPoolNode    node;
 };
 
 int32_t vnodeOpenBufPool(SVnode* pVnode, int64_t size);
@@ -78,7 +79,7 @@ void    vnodeBufPoolReset(SVBufPool* pPool);
 int32_t vnodeQueryOpen(SVnode* pVnode);
 void    vnodeQueryClose(SVnode* pVnode);
 int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg);
-int     vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg);
+int     vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg);
 
 // vnodeCommit.c
 int32_t vnodeBegin(SVnode* pVnode);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -77,6 +77,8 @@ typedef struct SSnapDataHdr    SSnapDataHdr;
 // vnd.h
 void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
 void  vnodeBufPoolFree(SVBufPool* pPool, void* p);
+void  vnodeBufPoolRef(SVBufPool* pPool);
+void  vnodeBufPoolUnRef(SVBufPool* pPool);
 
 // meta
 typedef struct SMCtbCursor SMCtbCursor;
@@ -247,34 +249,34 @@ struct STsdbKeepCfg {
 };
 
 struct SVnode {
-  char*      path;
+  char*         path;
-  SVnodeCfg  config;
+  SVnodeCfg     config;
-  SVState    state;
+  SVState       state;
-  STfs*      pTfs;
+  STfs*         pTfs;
-  SMsgCb     msgCb;
+  SMsgCb        msgCb;
-  SVBufPool* pPool;
+  TdThreadMutex mutex;
-  SVBufPool* inUse;
+  TdThreadCond  poolNotEmpty;
-  SVBufPool* onCommit;
+  SVBufPool*    pPool;
-  SVBufPool* onRecycle;
+  SVBufPool*    inUse;
-  SMeta*     pMeta;
+  SMeta*        pMeta;
-  SSma*      pSma;
+  SSma*         pSma;
-  STsdb*     pTsdb;
+  STsdb*        pTsdb;
-  SWal*      pWal;
+  SWal*         pWal;
-  STQ*       pTq;
+  STQ*          pTq;
-  SSink*     pSink;
+  SSink*        pSink;
-  tsem_t     canCommit;
+  tsem_t        canCommit;
-  int64_t    sync;
+  int64_t       sync;
-  int32_t    blockCount;
+  int32_t       blockCount;
-  tsem_t     syncSem;
+  tsem_t        syncSem;
-  SQHandle*  pQuery;
+  SQHandle*     pQuery;
 };
 
 #define TD_VID(PVNODE) ((PVNODE)->config.vgId)
 
 #define VND_TSDB(vnd)       ((vnd)->pTsdb)
 #define VND_RSMA0(vnd)      ((vnd)->pTsdb)
-#define VND_RSMA1(vnd)      ((vnd)->pSma->pRSmaTsdb1)
+#define VND_RSMA1(vnd)      ((vnd)->pSma->pRSmaTsdb[TSDB_RETENTION_L0])
-#define VND_RSMA2(vnd)      ((vnd)->pSma->pRSmaTsdb2)
+#define VND_RSMA2(vnd)      ((vnd)->pSma->pRSmaTsdb[TSDB_RETENTION_L1])
 #define VND_RETENTIONS(vnd) (&(vnd)->config.tsdbCfg.retentions)
 #define VND_IS_RSMA(v)      ((v)->config.isRsma == 1)
 #define VND_IS_TSMA(v)      ((v)->config.isTsma == 1)
@@ -289,8 +291,7 @@ struct SSma {
   bool          locked;
   TdThreadMutex mutex;
   SVnode*       pVnode;
-  STsdb*        pRSmaTsdb1;
+  STsdb*        pRSmaTsdb[TSDB_RETENTION_L2];
-  STsdb*        pRSmaTsdb2;
   void*         pTSmaEnv;
   void*         pRSmaEnv;
 };
@@ -305,8 +306,8 @@ struct SSma {
 #define SMA_TSMA_ENV(s)   ((s)->pTSmaEnv)
 #define SMA_RSMA_ENV(s)   ((s)->pRSmaEnv)
 #define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb)
-#define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb1)
+#define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L0])
-#define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb2)
+#define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L1])
 
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);

source/dnode/vnode/src/meta/metaTable.c

@@ -277,7 +277,7 @@ _drop_super_table:
 _exit:
   tdbFree(pKey);
   tdbFree(pData);
-  metaDebug("vgId:%d,  super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
+  metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
   return 0;
 }
 
@@ -374,6 +374,13 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
   }
   metaReaderClear(&mr);
 
+  if (pReq->type == TSDB_CHILD_TABLE) {
+    tb_uid_t suid = metaGetTableEntryUidByName(pMeta, pReq->ctb.name);
+    if (suid == 0) {
+      terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
+      return -1;
+    }
+  }
   // build SMetaEntry
   me.version = version;
   me.type = pReq->type;

source/dnode/vnode/src/sma/smaCommit.c

@@ -191,7 +191,7 @@ static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) {
   if ((pDir = taosOpenDir(dir)) == NULL) {
     regfree(&regex);
     terrno = TAOS_SYSTEM_ERROR(errno);
-    smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
+    smaDebug("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
     return TSDB_CODE_FAILED;
   }
 
@@ -392,5 +392,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
   // step 2: cleanup outdated qtaskinfo files
   tdCleanupQTaskInfoFiles(pSma, pRSmaStat);
 
+  atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0);
+
   return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/sma/smaEnv.c

@@ -169,6 +169,26 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
   return 0;
 }
 
+int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
+  if (!pRSmaInfo) return 0;
+
+  int ref = T_REF_INC(pRSmaInfo);
+  smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
+  return 0;
+}
+
+int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) {
+  if (!pRSmaInfo) return 0;
+
+  int ref = T_REF_DEC(pRSmaInfo);
+  smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref);
+
+  if (ref == 0) {
+    tdRemoveRSmaInfoBySuid(pSma, pRSmaInfo->suid);
+  }
+  return 0;
+}
+
 static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) {
   ASSERT(pSmaStat != NULL);
 

source/dnode/vnode/src/sma/smaOpen.c

@@ -123,11 +123,9 @@ int32_t smaOpen(SVnode *pVnode) {
     }
 
     // restore the rsma
-#if 1
     if (rsmaRestore(pSma) < 0) {
       goto _err;
     }
-#endif
   }
 
   return 0;

source/dnode/vnode/src/sma/smaRollup.c

@@ -24,6 +24,7 @@ SSmaMgmt smaMgmt = {
 };
 
 #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
+#define TD_RSMAINFO_DEL_FILE      "rsmainfo.del"
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
@@ -48,14 +49,11 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables);
 static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
 static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);
 
-
-
 static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) {
   // adapt accordingly if definition of SRSmaInfo update
   SRSmaInfo *pResult = NULL;
-  int32_t    rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *);
   ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2);
-  pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + rsmaInfoHeadLen));
+  pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + RSMA_INFO_HEAD_LEN));
   ASSERT(pResult->pTSchema->numOfCols > 1);
   return pResult;
 }
@@ -116,8 +114,8 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
       if (pItem->taskInfo) {
         if (isDeepFree && pItem->tmrId) {
-          smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
+          smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid,
-                   i + 1);
+                   pItem->tmrId, i + 1);
           taosTmrStopA(&pItem->tmrId);
         }
         tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1);
@@ -337,6 +335,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
   }
   pRSmaInfo->pTSchema = pTSchema;
   pRSmaInfo->suid = suid;
+  T_REF_INIT_VAL(pRSmaInfo, 1);
 
   if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) {
     goto _err;
@@ -392,11 +391,33 @@ int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) {
 int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) {
   SVnode *pVnode = pSma->pVnode;
   if (!VND_IS_RSMA(pVnode)) {
-    smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name,
+    smaTrace("vgId:%d, not drop rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name,
              pReq->suid);
     return TSDB_CODE_SUCCESS;
   }
 
+  SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
+  if (!pSmaEnv) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSmaStat  *pStat = SMA_ENV_STAT(pSmaEnv);
+  SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
+
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pReq->suid);
+
+  if (!pRSmaInfo) {
+    smaWarn("vgId:%d, drop rsma for stable %s %" PRIi64 " failed no rsma in hash", TD_VID(pVnode), pReq->name,
+            pReq->suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // set del flag for data in mem
+  RSMA_INFO_SET_DEL(pRSmaInfo);
+  tdUnRefRSmaInfo(pSma, pRSmaInfo);
+
+  // save to file
+
   smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid);
   return TSDB_CODE_SUCCESS;
 }
@@ -584,7 +605,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche
     snprintf(flag, 10, "level %" PRIi8, pItem->level);
     blockDebugShowDataBlocks(pResult, flag);
 #endif
-    STsdb      *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
+    STsdb      *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
     SSubmitReq *pReq = NULL;
     // TODO: the schema update should be handled
     if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) < 0) {
@@ -650,10 +671,10 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
 
 /**
  * @brief During async commit, the SRSmaInfo object would be COW from iRSmaInfoHash and write lock should be applied.
- * 
+ *
- * @param pSma 
+ * @param pSma
- * @param suid 
+ * @param suid
- * @return SRSmaInfo* 
+ * @return SRSmaInfo*
  */
 static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
@@ -661,7 +682,6 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   SRSmaInfo *pRSmaInfo = NULL;
 
   if (!pEnv) {
-    // only applicable when rsma env exists
     return NULL;
   }
 
@@ -683,18 +703,21 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   SRSmaInfo *pCowRSmaInfo = NULL;
   // lock
   taosWLockLatch(SMA_ENV_LOCK(pEnv));
-  void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+  if (!taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t))) {  // 2-phase lock
-  if (iRSmaInfo) {
+    void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
-    SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo;
+    if (iRSmaInfo) {
-    if (pIRSmaInfo) {
+      SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo;
-      if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) {
+      if (pIRSmaInfo) {
-        taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
+        if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) {
-        smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr());
+          taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
-        return NULL;
+          smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr());
-      }
+          return NULL;
-      if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) {
+        }
-        taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
+        smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid);
-        return NULL;
+        if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) {
+          taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
+          return NULL;
+        }
       }
     }
   }
@@ -703,21 +726,56 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
   return pCowRSmaInfo;
 }
 
+/**
+ * @brief During the drop procedure, only need to delete the object in rsmaInfoHash.
+ *
+ * @param pSma
+ * @param suid
+ * @return SRSmaInfo*
+ */
+void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) {
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pStat = NULL;
+  SRSmaInfo *pRSmaInfo = NULL;
+
+  if (!pEnv) {
+    return;
+  }
+
+  pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+  if (!pStat || !RSMA_INFO_HASH(pStat)) {
+    return;
+  }
+
+  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+  if (pRSmaInfo) {
+    if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+      tdFreeRSmaInfo(pSma, pRSmaInfo, true);
+    }
+    taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+    smaDebug("vgId:%d, remove from infoHash for table:%" PRIu64 " succeed", SMA_VID(pSma), suid);
+  }
+}
+
 static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
   SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid);
   if (!pRSmaInfo) {
-    smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
+    smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
     return TSDB_CODE_SUCCESS;
   }
 
   if (!pRSmaInfo->items[0].taskInfo) {
-    smaDebug("vgId:%d, return as no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
+    smaDebug("vgId:%d, execute rsma, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid);
     return TSDB_CODE_SUCCESS;
   }
 
   if (inputType == STREAM_INPUT__DATA_SUBMIT) {
+    tdRefRSmaInfo(pSma, pRSmaInfo);
+
     tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1);
     tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2);
+
+    tdUnRefRSmaInfo(pSma, pRSmaInfo);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -891,13 +949,8 @@ _err:
  * @return int32_t
  */
 static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed) {
-  // TODO
+  // NOTHING TODO: the data would be restored from the unified WAL replay procedure
-  smaDebug("vgId:%d, rsma restore from %" PRIi64 ", ts data reload success", SMA_VID(pSma), committed);
   return TSDB_CODE_SUCCESS;
-_err:
-  smaError("vgId:%d, rsma restore from %" PRIi64 ", ts data reload failed since %s", SMA_VID(pSma), committed,
-           terrstr());
-  return TSDB_CODE_FAILED;
 }
 
 int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
@@ -931,10 +984,10 @@ _err:
 
 /**
  * @brief Restore from SRSmaQTaskInfoItem
- * 
+ *
- * @param pSma 
+ * @param pSma
- * @param pItem 
+ * @param pItem
- * @return int32_t 
+ * @return int32_t
  */
 static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) {
   SRSmaInfo *pRSmaInfo = NULL;
@@ -1271,6 +1324,9 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
   }
 
   SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem);
+  if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+    goto _end;
+  }
 
   int8_t fetchTriggerStat =
       atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
@@ -1279,13 +1335,14 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
                pItem->level, pRSmaInfo->suid);
 
-      tdRefSmaStat(pSma, (SSmaStat *)pStat);
+      // sync procedure => async process
+      tdRefRSmaInfo(pSma, pRSmaInfo);
 
       SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
       qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
       tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK);
 
-      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
+      tdUnRefRSmaInfo(pSma, pRSmaInfo);
     } break;
     case TASK_TRIGGER_STAT_PAUSED: {
       smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",

source/dnode/vnode/src/tq/tq.c

@@ -591,7 +591,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
       };
       pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
     } else {
-      pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
+      SReadHandle mgHandle = {
+          .vnode = NULL,
+          .numOfVgroups = pTask->numOfVgroups,
+      };
+      pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle);
     }
     ASSERT(pTask->exec.executor);
   }

source/dnode/vnode/src/tq/tqExec.c

@@ -68,7 +68,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa
       pRsp->rspOffset = *pOffset;
       return 0;
     } else {
-      tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1);
+      tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
       if (qStreamPrepareScan(task, pOffset) < 0) {
         pRsp->rspOffset = *pOffset;
         return 0;
@@ -106,7 +106,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa
     }
 
     if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-      tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1);
+      tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
       qStreamPrepareScan(task, pOffset);
       continue;
     }

source/dnode/vnode/src/tq/tqPush.c

@@ -243,7 +243,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
     void* data = taosMemoryMalloc(msgLen);
     if (data == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      qError("failed to copy data for stream since out of memory");
+      tqError("failed to copy data for stream since out of memory");
       return -1;
     }
     memcpy(data, msg, msgLen);

source/dnode/vnode/src/tq/tqSink.c

@@ -49,8 +49,11 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
       }
 
       SVCreateTbReq createTbReq = {0};
+      SName         name = {0};
+      tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+
       createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
-      createTbReq.ctb.name = strdup(stbFullName);
+      createTbReq.ctb.name = strdup((char*)tNameGetTableName(&name));  // strdup(stbFullName);
       createTbReq.flags = 0;
       createTbReq.type = TSDB_CHILD_TABLE;
       createTbReq.ctb.suid = suid;

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -464,7 +464,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
 
   switch (state->state) {
     case SFSNEXTROW_FS:
-      state->aDFileSet = state->pTsdb->fs->cState->aDFileSet;
+      state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet;
       state->nFileSet = taosArrayGetSize(state->aDFileSet);
       state->iFileSet = state->nFileSet;
 
@@ -793,6 +793,9 @@ typedef struct {
   TSDBROW         memRow, imemRow, fsRow;
 
   TsdbNextRowState input[3];
+  SMemTable       *pMemTable;
+  SMemTable       *pIMemTable;
+  STsdb           *pTsdb;
 } CacheNextRowIter;
 
 static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb) {
@@ -800,21 +803,25 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
 
   tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
 
+  tsdbTakeMemSnapshot(pTsdb, &pIter->pMemTable, &pIter->pIMemTable);
+
   STbData *pMem = NULL;
-  if (pTsdb->mem) {
+  if (pIter->pMemTable) {
-    tsdbGetTbDataFromMemTable(pTsdb->mem, suid, uid, &pMem);
+    tsdbGetTbDataFromMemTable(pIter->pMemTable, suid, uid, &pMem);
   }
 
   STbData *pIMem = NULL;
-  if (pTsdb->imem) {
+  if (pIter->pIMemTable) {
-    tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem);
+    tsdbGetTbDataFromMemTable(pIter->pIMemTable, suid, uid, &pIMem);
   }
 
+  pIter->pTsdb = pTsdb;
+
   pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY));
 
   SDelIdx delIdx;
 
-  SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState);
+  SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState);
   if (pDelFile) {
     SDelFReader *pDelFReader;
 
@@ -878,6 +885,8 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) {
     taosArrayDestroy(pIter->pSkyline);
   }
 
+  tsdbUntakeMemSnapshot(pIter->pTsdb, pIter->pMemTable, pIter->pIMemTable);
+
   return code;
 _err:
   return code;
@@ -1189,7 +1198,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
 
   SDelIdx delIdx;
 
-  SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState);
+  SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState);
   if (pDelFile) {
     SDelFReader *pDelFReader;
 
@@ -1377,7 +1386,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
 
   SDelIdx delIdx;
 
-  SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState);
+  SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState);
   if (pDelFile) {
     SDelFReader *pDelFReader;
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -64,9 +64,26 @@ int32_t tsdbBegin(STsdb *pTsdb) {
 
   if (!pTsdb) return code;
 
-  code = tsdbMemTableCreate(pTsdb, &pTsdb->mem);
+  SMemTable *pMemTable;
+  code = tsdbMemTableCreate(pTsdb, &pMemTable);
   if (code) goto _err;
 
+  // lock
+  code = taosThreadRwlockWrlock(&pTsdb->rwLock);
+  if (code) {
+    code = TAOS_SYSTEM_ERROR(code);
+    goto _err;
+  }
+
+  pTsdb->mem = pMemTable;
+
+  // unlock
+  code = taosThreadRwlockUnlock(&pTsdb->rwLock);
+  if (code) {
+    code = TAOS_SYSTEM_ERROR(code);
+    goto _err;
+  }
+
   return code;
 
 _err:
@@ -83,9 +100,11 @@ int32_t tsdbCommit(STsdb *pTsdb) {
 
   // check
   if (pMemTable->nRow == 0 && pMemTable->nDel == 0) {
-    // TODO: lock?
+    taosThreadRwlockWrlock(&pTsdb->rwLock);
     pTsdb->mem = NULL;
-    tsdbMemTableDestroy(pMemTable);
+    taosThreadRwlockUnlock(&pTsdb->rwLock);
+
+    tsdbUnrefMemTable(pMemTable);
     goto _exit;
   }
 
@@ -139,7 +158,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) {
     goto _err;
   }
 
-  SDelFile *pDelFileR = pTsdb->fs->nState->pDelFile;
+  SDelFile *pDelFileR = pTsdb->pFS->nState->pDelFile;
   if (pDelFileR) {
     code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb, NULL);
     if (code) goto _err;
@@ -228,7 +247,7 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) {
   code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter);
   if (code) goto _err;
 
-  code = tsdbFSStateUpsertDelFile(pTsdb->fs->nState, &pCommitter->pDelFWriter->fDel);
+  code = tsdbFSStateUpsertDelFile(pTsdb->pFS->nState, &pCommitter->pDelFWriter->fDel);
   if (code) goto _err;
 
   code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1);
@@ -263,7 +282,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
   taosArrayClear(pCommitter->aBlockIdx);
   tMapDataReset(&pCommitter->oBlockMap);
   tBlockDataReset(&pCommitter->oBlockData);
-  pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid, TD_EQ);
+  pRSet = tsdbFSStateGetDFileSet(pTsdb->pFS->nState, pCommitter->commitFid, TD_EQ);
   if (pRSet) {
     code = tsdbDataFReaderOpen(&pCommitter->pReader, pTsdb, pRSet);
     if (code) goto _err;
@@ -836,7 +855,7 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
   if (code) goto _err;
 
   // upsert SDFileSet
-  code = tsdbFSStateUpsertDFileSet(pCommitter->pTsdb->fs->nState, tsdbDataFWriterGetWSet(pCommitter->pWriter));
+  code = tsdbFSStateUpsertDFileSet(pCommitter->pTsdb->pFS->nState, tsdbDataFWriterGetWSet(pCommitter->pWriter));
   if (code) goto _err;
 
   // close and sync
@@ -941,10 +960,10 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
   memset(pCommitter, 0, sizeof(*pCommitter));
   ASSERT(pTsdb->mem && pTsdb->imem == NULL);
 
-  // lock();
+  taosThreadRwlockWrlock(&pTsdb->rwLock);
   pTsdb->imem = pTsdb->mem;
   pTsdb->mem = NULL;
-  // unlock();
+  taosThreadRwlockUnlock(&pTsdb->rwLock);
 
   pCommitter->pTsdb = pTsdb;
   pCommitter->commitID = pTsdb->pVnode->state.commitID;
@@ -954,7 +973,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
   pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
   pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
 
-  code = tsdbFSBegin(pTsdb->fs);
+  code = tsdbFSBegin(pTsdb->pFS);
   if (code) goto _err;
 
   return code;
@@ -1135,13 +1154,16 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
   SMemTable *pMemTable = pTsdb->imem;
 
   if (eno == 0) {
-    code = tsdbFSCommit(pTsdb->fs);
+    code = tsdbFSCommit(pTsdb->pFS);
   } else {
-    code = tsdbFSRollback(pTsdb->fs);
+    code = tsdbFSRollback(pTsdb->pFS);
   }
 
-  tsdbMemTableDestroy(pMemTable);
+  taosThreadRwlockWrlock(&pTsdb->rwLock);
   pTsdb->imem = NULL;
+  taosThreadRwlockUnlock(&pTsdb->rwLock);
+
+  tsdbUnrefMemTable(pMemTable);
 
   tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode));
   return code;

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -245,7 +245,7 @@ static int32_t tsdbApplyDelFileChange(STsdbFS *pFS, SDelFile *pFrom, SDelFile *p
   char    fname[TSDB_FILENAME_LEN];
 
   if (pFrom && pTo) {
-    if (pFrom != pTo) {
+    if (!tsdbDelFileIsSame(pFrom, pTo)) {
       tsdbDelFileName(pFS->pTsdb, pFrom, fname);
       if (taosRemoveFile(fname) < 0) {
         code = TAOS_SYSTEM_ERROR(errno);

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -140,6 +140,8 @@ bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT fty
   }
 }
 
+bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; }
+
 int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) {
   int32_t code = 0;
   int64_t n;

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -41,6 +41,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) {
   }
   taosInitRWLatch(&pMemTable->latch);
   pMemTable->pTsdb = pTsdb;
+  pMemTable->pPool = pTsdb->pVnode->inUse;
   pMemTable->nRef = 1;
   pMemTable->minKey = TSKEY_MAX;
   pMemTable->maxKey = TSKEY_MIN;
@@ -54,6 +55,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) {
     taosMemoryFree(pMemTable);
     goto _err;
   }
+  vnodeBufPoolRef(pMemTable->pPool);
 
   *ppMemTable = pMemTable;
   return code;
@@ -65,6 +67,7 @@ _err:
 
 void tsdbMemTableDestroy(SMemTable *pMemTable) {
   if (pMemTable) {
+    vnodeBufPoolUnRef(pMemTable->pPool);
     taosArrayDestroy(pMemTable->aTbData);
     taosMemoryFree(pMemTable);
   }
@@ -590,3 +593,58 @@ _err:
 }
 
 int32_t tsdbGetNRowsInTbData(STbData *pTbData) { return pTbData->sl.size; }
+
+void tsdbRefMemTable(SMemTable *pMemTable) {
+  int32_t nRef = atomic_fetch_add_32(&pMemTable->nRef, 1);
+  ASSERT(nRef > 0);
+}
+
+void tsdbUnrefMemTable(SMemTable *pMemTable) {
+  int32_t nRef = atomic_sub_fetch_32(&pMemTable->nRef, 1);
+  if (nRef == 0) {
+    tsdbMemTableDestroy(pMemTable);
+  }
+}
+
+int32_t tsdbTakeMemSnapshot(STsdb *pTsdb, SMemTable **ppMem, SMemTable **ppIMem) {
+  int32_t code = 0;
+
+  // lock
+  code = taosThreadRwlockRdlock(&pTsdb->rwLock);
+  if (code) {
+    code = TAOS_SYSTEM_ERROR(code);
+    goto _exit;
+  }
+
+  // take snapshot
+  *ppMem = pTsdb->mem;
+  *ppIMem = pTsdb->imem;
+
+  if (*ppMem) {
+    tsdbRefMemTable(*ppMem);
+  }
+
+  if (*ppIMem) {
+    tsdbRefMemTable(*ppIMem);
+  }
+
+  // unlock
+  code = taosThreadRwlockUnlock(&pTsdb->rwLock);
+  if (code) {
+    code = TAOS_SYSTEM_ERROR(code);
+    goto _exit;
+  }
+
+_exit:
+  return code;
+}
+
+void tsdbUntakeMemSnapshot(STsdb *pTsdb, SMemTable *pMem, SMemTable *pIMem) {
+  if (pMem) {
+    tsdbUnrefMemTable(pMem);
+  }
+
+  if (pIMem) {
+    tsdbUnrefMemTable(pIMem);
+  }
+}

source/dnode/vnode/src/tsdb/tsdbOpen.c

@@ -54,8 +54,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
   sprintf(pTsdb->path, "%s%s%s", pVnode->path, TD_DIRSEP, dir);
   taosRealPath(pTsdb->path, NULL, slen);
   pTsdb->pVnode = pVnode;
-  pTsdb->repoLocked = false;
+  taosThreadRwlockInit(&pTsdb->rwLock, NULL);
-  taosThreadMutexInit(&pTsdb->mutex, NULL);
   if (!pKeepCfg) {
     tsdbSetKeepCfg(&pTsdb->keepCfg, &pVnode->config.tsdbCfg);
   } else {
@@ -67,7 +66,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
   tfsMkdir(pVnode->pTfs, pTsdb->path);
 
   // open tsdb
-  if (tsdbFSOpen(pTsdb, &pTsdb->fs) < 0) {
+  if (tsdbFSOpen(pTsdb, &pTsdb->pFS) < 0) {
     goto _err;
   }
 
@@ -88,33 +87,10 @@ _err:
 
 int tsdbClose(STsdb **pTsdb) {
   if (*pTsdb) {
-    taosThreadMutexDestroy(&(*pTsdb)->mutex);
+    taosThreadRwlockDestroy(&(*pTsdb)->rwLock);
-    tsdbFSClose((*pTsdb)->fs);
+    tsdbFSClose((*pTsdb)->pFS);
     tsdbCloseCache((*pTsdb)->lruCache);
     taosMemoryFreeClear(*pTsdb);
   }
   return 0;
 }
-
-int tsdbLockRepo(STsdb *pTsdb) {
-  int code = taosThreadMutexLock(&pTsdb->mutex);
-  if (code != 0) {
-    tsdbError("vgId:%d, failed to lock tsdb since %s", TD_VID(pTsdb->pVnode), strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  pTsdb->repoLocked = true;
-  return 0;
-}
-
-int tsdbUnlockRepo(STsdb *pTsdb) {
-  // ASSERT(IS_REPO_LOCKED(pTsdb));
-  pTsdb->repoLocked = false;
-  int code = taosThreadMutexUnlock(&pTsdb->mutex);
-  if (code != 0) {
-    tsdbError("vgId:%d, failed to unlock tsdb since %s", TD_VID(pTsdb->pVnode), strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
-  }
-  return 0;
-}

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -118,6 +118,8 @@ struct STsdbReader {
   char*              idStr;  // query info handle, for debug purpose
   int32_t            type;   // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows
   SBlockLoadSuppInfo suppInfo;
+  SMemTable*         pMem;
+  SMemTable*         pIMem;
 
   SIOCostSummary cost;
   STSchema*      pSchema;
@@ -1453,38 +1455,71 @@ static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVer
          (pBlock->minVersion <= pVerRange->maxVer);
 }
 
-static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) {
-  if (pBlockScanInfo->delSkyline == NULL) {
-    return false;
-  }
 
-  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
+static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) {
-  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
-
-  // ts is not overlap
-  if (pBlock->minKey.ts > pLast->ts || pBlock->maxKey.ts < pFirst->ts) {
-    return false;
-  }
-
-  int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1;
-
-  // version is not overlap
   size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
-  for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += step) {
+
+  for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) {
     TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);
     if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) {
       if (p->version >= pBlock->minVersion) {
         return true;
       }
-    } else if (p->ts > pBlock->maxKey.ts) {
+    } else if (p->ts < pBlock->minKey.ts) {  // p->ts < pBlock->minKey.ts
+      if (p->version >= pBlock->minVersion) {
+        if (i < num - 1) {
+          TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
+          if (i + 1 == num - 1) {  // pnext is the last point
+            if (pnext->ts >= pBlock->minKey.ts) {
+              return true;
+            }
+          } else {
+            if (pnext->ts >= pBlock->minKey.ts && pnext->version >= pBlock->minVersion) {
+              return true;
+            }
+          }
+        } else {  // it must be the last point
+          ASSERT(p->version == 0);
+        }
+      }
+    } else {  // (p->ts > pBlock->maxKey.ts) {
       return false;
     }
   }
 
-  ASSERT(0);
   return false;
 }
 
+static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) {
+  if (pBlockScanInfo->delSkyline == NULL) {
+    return false;
+  }
+
+  // ts is not overlap
+  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
+  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
+  if (pBlock->minKey.ts > pLast->ts || pBlock->maxKey.ts < pFirst->ts) {
+    return false;
+  }
+
+  // version is not overlap
+  if (ASCENDING_TRAVERSE(order)) {
+    return doCheckforDatablockOverlap(pBlockScanInfo, pBlock);
+  } else {
+    int32_t index = pBlockScanInfo->fileDelIndex;
+    while(1) {
+      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);
+      if (p->ts > pBlock->minKey.ts && index > 0) {
+        index -= 1;
+      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.
+        break;
+      }
+    }
+
+    return doCheckforDatablockOverlap(pBlockScanInfo, pBlock);
+  }
+}
+
 // 1. the version of all rows should be less than the endVersion
 // 2. current block should not overlap with next neighbor block
 // 3. current timestamp should not be overlap with each other
@@ -1847,8 +1882,8 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
   int32_t backward = (!ASCENDING_TRAVERSE(pReader->order));
 
   STbData* d = NULL;
-  if (pReader->pTsdb->mem != NULL) {
+  if (pReader->pMem != NULL) {
-    tsdbGetTbDataFromMemTable(pReader->pTsdb->mem, pReader->suid, pBlockScanInfo->uid, &d);
+    tsdbGetTbDataFromMemTable(pReader->pMem, pReader->suid, pBlockScanInfo->uid, &d);
     if (d != NULL) {
       code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter);
       if (code == TSDB_CODE_SUCCESS) {
@@ -1868,8 +1903,8 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
   }
 
   STbData* di = NULL;
-  if (pReader->pTsdb->imem != NULL) {
+  if (pReader->pIMem != NULL) {
-    tsdbGetTbDataFromMemTable(pReader->pTsdb->imem, pReader->suid, pBlockScanInfo->uid, &di);
+    tsdbGetTbDataFromMemTable(pReader->pIMem, pReader->suid, pBlockScanInfo->uid, &di);
     if (di != NULL) {
       code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter);
       if (code == TSDB_CODE_SUCCESS) {
@@ -1905,7 +1940,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
 
   SArray* pDelData = taosArrayInit(4, sizeof(SDelData));
 
-  SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState);
+  SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState);
   if (pDelFile) {
     SDelFReader* pDelFReader = NULL;
     code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL);
@@ -2795,7 +2830,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
 
   SDataBlockIter* pBlockIter = &pReader->status.blockIter;
 
-  STsdbFSState* pFState = pReader->pTsdb->fs->cState;
+  STsdbFSState* pFState = pReader->pTsdb->pFS->cState;
   initFilesetIterator(&pReader->status.fileIter, pFState, pReader->order, pReader->idStr);
   resetDataBlockIterator(&pReader->status.blockIter, pReader->order);
 
@@ -2809,6 +2844,8 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
     }
   }
 
+  tsdbTakeMemSnapshot(pReader->pTsdb, &pReader->pMem, &pReader->pIMem);
+
   tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
   return code;
 
@@ -2824,6 +2861,8 @@ void tsdbReaderClose(STsdbReader* pReader) {
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
 
+  tsdbUntakeMemSnapshot(pReader->pTsdb, pReader->pMem, pReader->pIMem);
+
   taosMemoryFreeClear(pSupInfo->plist);
   taosMemoryFree(pSupInfo->colIds);
 
@@ -3042,7 +3081,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
 
   tsdbDataFReaderClose(&pReader->pFileReader);
 
-  STsdbFSState* pFState = pReader->pTsdb->fs->cState;
+  STsdbFSState* pFState = pReader->pTsdb->pFS->cState;
   initFilesetIterator(&pReader->status.fileIter, pFState, pReader->order, pReader->idStr);
   resetDataBlockIterator(&pReader->status.blockIter, pReader->order);
   resetDataBlockScanInfo(pReader->status.pTableMap);

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -246,7 +246,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
 
   tsdbDelFileName(pTsdb, pFile, fname);
   pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ);
-  if (pDelFReader == NULL) {
+  if (pDelFReader->pReadH == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     taosMemoryFree(pDelFReader);
     goto _err;

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -20,10 +20,10 @@ static int32_t tsdbDoRetentionImpl(STsdb *pTsdb, int64_t now, int8_t try, int8_t
   STsdbFSState *pState;
 
   if (try) {
-    pState = pTsdb->fs->cState;
+    pState = pTsdb->pFS->cState;
     *canDo = 0;
   } else {
-    pState = pTsdb->fs->nState;
+    pState = pTsdb->pFS->nState;
   }
 
   for (int32_t iSet = 0; iSet < taosArrayGetSize(pState->aDFileSet); iSet++) {
@@ -83,7 +83,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
   if (!canDo) goto _exit;
 
   // begin
-  code = tsdbFSBegin(pTsdb->fs);
+  code = tsdbFSBegin(pTsdb->pFS);
   if (code) goto _err;
 
   // do retention
@@ -91,7 +91,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
   if (code) goto _err;
 
   // commit
-  code = tsdbFSCommit(pTsdb->fs);
+  code = tsdbFSCommit(pTsdb->pFS);
   if (code) goto _err;
 
 _exit:
@@ -99,6 +99,6 @@ _exit:
 
 _err:
   tsdbError("vgId:%d tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
-  tsdbFSRollback(pTsdb->fs);
+  tsdbFSRollback(pTsdb->pFS);
   return code;
 }

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -45,7 +45,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
 
   while (true) {
     if (pReader->pDataFReader == NULL) {
-      SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->cState, pReader->fid, TD_GT);
+      SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->pFS->cState, pReader->fid, TD_GT);
 
       if (pSet == NULL) goto _exit;
 
@@ -159,7 +159,7 @@ _err:
 static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
   int32_t   code = 0;
   STsdb*    pTsdb = pReader->pTsdb;
-  SDelFile* pDelFile = pTsdb->fs->cState->pDelFile;
+  SDelFile* pDelFile = pTsdb->pFS->cState->pDelFile;
 
   if (pReader->pDelFReader == NULL) {
     if (pDelFile == NULL) {
@@ -798,7 +798,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
   code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL);
   if (code) goto _err;
 
-  code = tsdbFSStateUpsertDFileSet(pTsdb->fs->nState, tsdbDataFWriterGetWSet(pWriter->pDataFWriter));
+  code = tsdbFSStateUpsertDFileSet(pTsdb->pFS->nState, tsdbDataFWriterGetWSet(pWriter->pDataFWriter));
   if (code) goto _err;
 
   code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1);
@@ -843,7 +843,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     pWriter->fid = fid;
 
     // read
-    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ);
+    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->pFS->nState, fid, TD_EQ);
     if (pSet) {
       code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
       if (code) goto _err;
@@ -907,7 +907,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
   STsdb*  pTsdb = pWriter->pTsdb;
 
   if (pWriter->pDelFWriter == NULL) {
-    SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->nState);
+    SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->nState);
 
     // reader
     if (pDelFile) {
@@ -1017,7 +1017,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
   code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter);
   if (code) goto _err;
 
-  code = tsdbFSStateUpsertDelFile(pTsdb->fs->nState, &pWriter->pDelFWriter->fDel);
+  code = tsdbFSStateUpsertDelFile(pTsdb->pFS->nState, &pWriter->pDelFWriter->fDel);
   if (code) goto _err;
 
   code = tsdbDelFWriterClose(&pWriter->pDelFWriter, 1);
@@ -1096,7 +1096,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
     goto _err;
   }
 
-  code = tsdbFSBegin(pTsdb->fs);
+  code = tsdbFSBegin(pTsdb->pFS);
   if (code) goto _err;
 
   *ppWriter = pWriter;
@@ -1113,7 +1113,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
   STsdbSnapWriter* pWriter = *ppWriter;
 
   if (rollback) {
-    code = tsdbFSRollback(pWriter->pTsdb->fs);
+    code = tsdbFSRollback(pWriter->pTsdb->pFS);
     if (code) goto _err;
   } else {
     code = tsdbSnapWriteDataEnd(pWriter);
@@ -1122,7 +1122,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
     code = tsdbSnapWriteDelEnd(pWriter);
     if (code) goto _err;
 
-    code = tsdbFSCommit(pWriter->pTsdb->fs);
+    code = tsdbFSCommit(pWriter->pTsdb->pFS);
     if (code) goto _err;
   }
 

source/dnode/vnode/src/vnd/vnodeBufPool.c

@@ -17,7 +17,7 @@
 
 /* ------------------------ STRUCTURES ------------------------ */
 
-static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool);
+static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool);
 static int vnodeBufPoolDestroy(SVBufPool *pPool);
 
 int vnodeOpenBufPool(SVnode *pVnode, int64_t size) {
@@ -28,7 +28,7 @@ int vnodeOpenBufPool(SVnode *pVnode, int64_t size) {
 
   for (int i = 0; i < 3; i++) {
     // create pool
-    ret = vnodeBufPoolCreate(size, &pPool);
+    ret = vnodeBufPoolCreate(pVnode, size, &pPool);
     if (ret < 0) {
       vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno));
       vnodeCloseBufPool(pVnode);
@@ -120,7 +120,7 @@ void vnodeBufPoolFree(SVBufPool *pPool, void *p) {
 }
 
 // STATIC METHODS -------------------
-static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool) {
+static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) {
   SVBufPool *pPool;
 
   pPool = taosMemoryMalloc(sizeof(SVBufPool) + size);
@@ -130,6 +130,7 @@ static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool) {
   }
 
   pPool->next = NULL;
+  pPool->pVnode = pVnode;
   pPool->nRef = 0;
   pPool->size = 0;
   pPool->ptr = pPool->node.data;
@@ -146,4 +147,26 @@ static int vnodeBufPoolDestroy(SVBufPool *pPool) {
   vnodeBufPoolReset(pPool);
   taosMemoryFree(pPool);
   return 0;
+}
+
+void vnodeBufPoolRef(SVBufPool *pPool) {
+  int32_t nRef = atomic_fetch_add_32(&pPool->nRef, 1);
+  ASSERT(nRef > 0);
+}
+
+void vnodeBufPoolUnRef(SVBufPool *pPool) {
+  int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1);
+  if (nRef == 0) {
+    SVnode *pVnode = pPool->pVnode;
+
+    vnodeBufPoolReset(pPool);
+
+    taosThreadMutexLock(&pVnode->mutex);
+
+    pPool->next = pVnode->pPool;
+    pVnode->pPool = pPool;
+    taosThreadCondSignal(&pVnode->poolNotEmpty);
+
+    taosThreadMutexUnlock(&pVnode->mutex);
+  }
 }

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -15,7 +15,7 @@
 
 #include "vnd.h"
 
-#define VND_INFO_FNAME "vnode.json"
+#define VND_INFO_FNAME     "vnode.json"
 #define VND_INFO_FNAME_TMP "vnode_tmp.json"
 
 static int  vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData);
@@ -27,18 +27,18 @@ static void vnodeWaitCommit(SVnode *pVnode);
 
 int vnodeBegin(SVnode *pVnode) {
   // alloc buffer pool
-  /* pthread_mutex_lock(); */
+  taosThreadMutexLock(&pVnode->mutex);
 
   while (pVnode->pPool == NULL) {
-    /* pthread_cond_wait(); */
+    taosThreadCondWait(&pVnode->poolNotEmpty, &pVnode->mutex);
   }
 
   pVnode->inUse = pVnode->pPool;
+  pVnode->inUse->nRef = 1;
   pVnode->pPool = pVnode->inUse->next;
   pVnode->inUse->next = NULL;
-  /* ref pVnode->inUse buffer pool */
 
-  /* pthread_mutex_unlock(); */
+  taosThreadMutexUnlock(&pVnode->mutex);
 
   pVnode->state.commitID++;
   // begin meta
@@ -217,7 +217,7 @@ int vnodeCommit(SVnode *pVnode) {
   vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID,
         pVnode->state.applied);
 
-  pVnode->onCommit = pVnode->inUse;
+  vnodeBufPoolUnRef(pVnode->inUse);
   pVnode->inUse = NULL;
 
   // save info
@@ -284,10 +284,6 @@ int vnodeCommit(SVnode *pVnode) {
 
   // apply the commit (TODO)
   walEndSnapshot(pVnode->pWal);
-  vnodeBufPoolReset(pVnode->onCommit);
-  pVnode->onCommit->next = pVnode->pPool;
-  pVnode->pPool = pVnode->onCommit;
-  pVnode->onCommit = NULL;
 
   vInfo("vgId:%d, commit over", TD_VID(pVnode));
 

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -89,6 +89,8 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
 
   tsem_init(&pVnode->syncSem, 0, 0);
   tsem_init(&(pVnode->canCommit), 0, 1);
+  taosThreadMutexInit(&pVnode->mutex, NULL);
+  taosThreadCondInit(&pVnode->poolNotEmpty, NULL);
 
   // open buffer pool
   if (vnodeOpenBufPool(pVnode, pVnode->config.isHeap ? 0 : pVnode->config.szBuf / 3) < 0) {
@@ -195,6 +197,8 @@ void vnodeClose(SVnode *pVnode) {
     // destroy handle
     tsem_destroy(&(pVnode->canCommit));
     tsem_destroy(&pVnode->syncSem);
+    taosThreadCondDestroy(&pVnode->poolNotEmpty);
+    taosThreadMutexDestroy(&pVnode->mutex);
     taosMemoryFree(pVnode);
   }
 }

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -27,6 +27,8 @@ struct SVSnapReader {
   // tsdb
   int8_t           tsdbDone;
   STsdbSnapReader *pTsdbReader;
+  // rsma
+  int8_t rsmaDone[TSDB_RETENTION_L2];
 };
 
 int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
@@ -115,6 +117,42 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
     }
   }
 
+  // RSMA ==============
+#if 0
+  if (VND_IS_RSMA(pReader->pVnode)) {
+    // RSMA1/RSMA2
+    for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+      if (!pReader->rsmaDone[i]) {
+        if (!pReader->pVnode->pSma->pRSmaTsdb[i]) {
+          // no valid tsdb 
+          pReader->rsmaDone[i] = 1;
+          continue;
+        }
+        if (pReader->pTsdbReader == NULL) {
+          code = tsdbSnapReaderOpen(pReader->pVnode->pSma->pRSmaTsdb[i], pReader->sver, pReader->ever,
+                                    &pReader->pTsdbReader);
+          if (code) goto _err;
+        }
+
+        code = tsdbSnapRead(pReader->pTsdbReader, ppData);
+        if (code) {
+          goto _err;
+        } else {
+          if (*ppData) {
+            goto _exit;
+          } else {
+            pReader->tsdbDone = 1;
+            code = tsdbSnapReaderClose(&pReader->pTsdbReader);
+            if (code) goto _err;
+          }
+        }
+      }
+    }
+    // QTaskInfoFile
+    // TODO ...
+  }
+#endif
+
   *ppData = NULL;
   *nData = 0;
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -53,6 +53,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
         *(int64_t *)(dc.data + dc.pos) = uid;
         *(int64_t *)(dc.data + dc.pos + 8) = ctime;
 
+        vTrace("vgId:%d, table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid);
         tEndDecode(&dc);
       }
 
@@ -381,7 +382,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p
     goto end;
   }
 
-  vInfo("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp);
+  vDebug("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp);
   int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids);
   if (ret != 0) {
     goto end;

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -17,35 +17,22 @@
 #include "vnd.h"
 
 static inline bool vnodeIsMsgBlock(tmsg_t type) {
-  return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA);
+  return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) ||
+         (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL);
 }
 
 static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
 
-static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) {
+static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
-  if (!vnodeIsMsgBlock(type)) return;
+  if (vnodeIsMsgBlock(pMsg->msgType)) {
-
+    vTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType));
-  int32_t count = atomic_add_fetch_32(&pVnode->blockCount, 1);
+    tsem_wait(&pVnode->syncSem);
-  vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
+  }
 }
 
-static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
+static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
-  int32_t count = atomic_load_32(&pVnode->blockCount);
+  if (vnodeIsMsgBlock(pMsg->msgType)) {
-  if (count <= 0) return;
+    vTrace("vgId:%d, msg:%p post block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType));
-
-  vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
-  tsem_wait(&pVnode->syncSem);
-}
-
-static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
-  if (!vnodeIsMsgBlock(type)) return;
-
-  int32_t count = atomic_load_32(&pVnode->blockCount);
-  if (count <= 0) return;
-
-  count = atomic_sub_fetch_32(&pVnode->blockCount, 1);
-  vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
-  if (count <= 0) {
     tsem_post(&pVnode->syncSem);
   }
 }
@@ -143,6 +130,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
   int32_t  code = 0;
   SRpcMsg *pMsg = NULL;
 
+  vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs);
+
   for (int32_t m = 0; m < numOfMsgs; m++) {
     if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
     const STraceId *trace = &pMsg->info.traceId;
@@ -165,13 +154,14 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
           if (rsp.info.handle != NULL) {
             tmsgSendRsp(&rsp);
           }
+        } else if (code == 0) {
+          vnodeWaitBlockMsg(pVnode, pMsg);
+        } else {
         }
       }
     }
 
-    if (code == 0) {
+    if (code < 0) {
-      vnodeAccumBlockMsg(pVnode, pMsg->msgType);
-    } else if (code < 0) {
       if (terrno == TSDB_CODE_SYN_NOT_LEADER) {
         vnodeRedirectRpcMsg(pVnode, pMsg);
       } else {
@@ -182,15 +172,12 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
           tmsgSendRsp(&rsp);
         }
       }
-    } else {
     }
 
     vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
     rpcFreeCont(pMsg->pCont);
     taosFreeQitem(pMsg);
   }
-
-  vnodeWaitBlockMsg(pVnode);
 }
 
 void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
@@ -213,7 +200,7 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
       }
     }
 
-    vnodePostBlockMsg(pVnode, pMsg->msgType);
+    vnodePostBlockMsg(pVnode, pMsg);
     if (rsp.info.handle != NULL) {
       tmsgSendRsp(&rsp);
     }
@@ -418,7 +405,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon
     tmsgSendRsp(&rpcMsg);
   }
 
-  vnodePostBlockMsg(pVnode, TDMT_VND_ALTER_REPLICA);
+  vnodePostBlockMsg(pVnode, pMsg);
 }
 
 static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {

source/libs/catalog/inc/catalogInt.h

@@ -532,6 +532,14 @@ typedef struct SCtgOperation {
   }                                                       \
 } while (0)
 
+#define CTG_API_JENTER() do {                             \
+  CTG_API_DEBUG("CTG API enter %s", __FUNCTION__);        \
+  CTG_LOCK(CTG_READ, &gCtgMgmt.lock);                     \
+  if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) {           \
+    CTG_ERR_JRET(TSDB_CODE_CTG_OUT_OF_SERVICE);           \
+  }                                                       \
+} while (0)
+
 
 #define CTG_API_LEAVE_NOLOCK(c) do {                        \
     int32_t __code = c;                                     \

source/libs/catalog/src/ctgRemote.c

@@ -244,10 +244,11 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize,
 int32_t ctgHandleMsgCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
   SCtgTaskCallbackParam* cbParam = (SCtgTaskCallbackParam*)param;
   int32_t code = 0;
+  SCtgJob* pJob = NULL;
   
-  CTG_API_ENTER();
+  CTG_API_JENTER();
 
-  SCtgJob* pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId);
+  pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId);
   if (NULL == pJob) {
     qDebug("ctg job refId 0x%" PRIx64 " already dropped", cbParam->refId);
     goto _return;
@@ -266,8 +267,6 @@ _return:
   if (pJob) {
     taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId);
   }
-  
-  taosMemoryFree(param);
 
   CTG_API_LEAVE(code);
 }
@@ -293,6 +292,7 @@ int32_t ctgMakeMsgSendInfo(SCtgTask* pTask, int32_t msgType, SMsgSendInfo **pMsg
   param->taskId = pTask->taskId;
 
   msgSendInfo->param = param;
+  msgSendInfo->paramFreeFp = taosMemoryFree;
   msgSendInfo->fp = ctgHandleMsgCallback;
 
   *pMsgSendInfo = msgSendInfo;

source/libs/executor/inc/executil.h

@@ -15,6 +15,7 @@
 #ifndef TDENGINE_QUERYUTIL_H
 #define TDENGINE_QUERYUTIL_H
 
+#include "vnode.h"
 #include "function.h"
 #include "nodes.h"
 #include "plannodes.h"
@@ -106,7 +107,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
 SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode);
 
 EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
-int32_t  getTableList(void* metaHandle, void* vnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo);
+int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo);
 SArray*  createSortInfo(SNodeList* pNodeList);
 SArray*  extractPartitionColInfo(SNodeList* pNodeList);
 SArray*  extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
@@ -128,4 +129,6 @@ int32_t convertFillType(int32_t mode);
 
 int32_t resultrowComparAsc(const void* p1, const void* p2);
 
+int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified);
+
 #endif  // TDENGINE_QUERYUTIL_H

source/libs/executor/inc/executorimpl.h

@@ -311,19 +311,10 @@ typedef struct STableScanInfo {
   int32_t         dataBlockLoadFlag;
   SInterval       interval;     // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded.
   SSampleExecInfo sample;       // sample execution info
-
   int32_t         currentGroupId;
   int32_t         currentTable;
-
+  int8_t          scanMode;
-#if 0
+  int8_t          noTable;
-  struct {
-    uint64_t uid;
-    int64_t ts;
-  } lastStatus;
-#endif
-
-  int8_t scanMode;
-  int8_t noTable;
 } STableScanInfo;
 
 typedef struct STagScanInfo {
@@ -429,8 +420,9 @@ typedef struct SStreamScanInfo {
 
   // status for tmq
   // SSchemaWrapper schema;
-  STqOffset offset;
+  STqOffset              offset;
-
+  SNode*                 pTagCond;
+  SNode*                 pTagIndexCond;
 } SStreamScanInfo;
 
 typedef struct SSysTableScanInfo {
@@ -874,8 +866,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
 SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode,
                                                SExecTaskInfo* pTaskInfo);
 
-SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
-    STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId);
+                                            SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup);
 
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
 
@@ -966,12 +958,12 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi
                                        SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
 
 int32_t createScanTableListInfo(SScanPhysiNode *pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
-                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId);
+                                STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idstr);
 
 SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
                                            SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo,
-                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId);
+                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
 
 void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
 

source/libs/executor/src/dataInserter.c

@@ -91,7 +91,6 @@ _return:
   tsem_post(&pInserter->ready);
 
   taosMemoryFree(pMsg->pData);
-  taosMemoryFree(param);
   
   return TSDB_CODE_SUCCESS;
 }
@@ -110,6 +109,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs
   pParam->pInserter = pInserter;
 
   pMsgSendInfo->param = pParam;
+  pMsgSendInfo->paramFreeFp = taosMemoryFree;        
   pMsgSendInfo->msgInfo.pData = pMsg;
   pMsgSendInfo->msgInfo.len = ntohl(pMsg->length);
   pMsgSendInfo->msgType = TDMT_VND_SUBMIT;

source/libs/executor/src/executil.c

@@ -265,7 +265,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) {
+int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) {
   SMetaReader mr = {0};
   metaReaderInit(&mr, metaHandle, 0);
   metaGetTableEntryByUid(&mr, info->uid);
@@ -280,19 +280,22 @@ static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) {
   if (TSDB_CODE_SUCCESS != code) {
     terrno = code;
     nodesDestroyNode(pTagCondTmp);
-    return false;
+    *pQualified = false;
+
+    return code;
   }
 
   ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
   SValueNode* pValue = (SValueNode*)pNew;
 
   ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL);
-  bool result = pValue->datum.b;
+  *pQualified = pValue->datum.b;
+
   nodesDestroyNode(pNew);
-  return result;
+  return TSDB_CODE_SUCCESS;
 }
 
-int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) {
+int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo));
@@ -304,8 +307,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
 
   pListInfo->suid = pScanNode->suid;
 
-  SNode* pTagCond = (SNode*)pListInfo->pTagCond;
-  SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond;
   if (pScanNode->tableType == TSDB_SUPER_TABLE) {
     if (pTagIndexCond) {
       SIndexMetaArg metaArg = {
@@ -345,9 +346,14 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
     int32_t i = 0;
     while (i < taosArrayGetSize(pListInfo->pTableList)) {
       STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i);
-      bool           isOk = isTableOk(info, pTagCond, metaHandle);
+
-      if (terrno) return terrno;
+      bool qualified = true;
-      if (!isOk) {
+      code = isTableOk(info, pTagCond, metaHandle, &qualified);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+
+      if (!qualified) {
         taosArrayRemove(pListInfo->pTableList, i);
         continue;
       }
@@ -362,7 +368,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
 
   // put into list as default group, remove it if grouping sorting is required later
   taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList);
-
   return code;
 }
 

source/libs/executor/src/executor.c

@@ -153,7 +153,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) {
   return pTaskInfo;
 }
 
-static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList) {
+static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) {
   SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
 
   // let's discard the tables those are not created according to the queried super table.
@@ -164,7 +164,7 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons
 
     int32_t code = metaGetTableEntryByUid(&mr, *id);
     if (code != TSDB_CODE_SUCCESS) {
-      qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
+      qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr);
       continue;
     }
 
@@ -172,6 +172,21 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons
     if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) {
       continue;
     }
+
+    if (pScanInfo->pTagCond != NULL) {
+      bool          qualified = false;
+      STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid, .lastKey = 0};
+      code = isTableOk(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified);
+      if (code != TSDB_CODE_SUCCESS) {
+        qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr);
+        continue;
+      }
+
+      if (!qualified) {
+        continue;
+      }
+    }
+
     /*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/
     // handle multiple partition
 
@@ -194,7 +209,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
   int32_t          code = 0;
   SStreamScanInfo* pScanInfo = pInfo->info;
   if (isAdd) {  // add new table id
-    SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList);
+    SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo));
 
     qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
     code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);

source/libs/executor/src/executorimpl.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <executorimpl.h>
 #include "filter.h"
 #include "function.h"
 #include "functionMgt.h"
@@ -1338,7 +1337,9 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
   if (pFilterNode == NULL) {
     return;
   }
-
+  if (pBlock->info.rows == 0) {
+    return;
+  }
   SFilterInfo* filter = NULL;
 
   // todo move to the initialization function
@@ -1995,16 +1996,9 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
   tsem_post(&pExchangeInfo->ready);
   taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId);
 
-  taosMemoryFree(pWrapper);
   return TSDB_CODE_SUCCESS;
 }
 
-static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
-  assert(pMsgBody != NULL);
-  taosMemoryFreeClear(pMsgBody->msgInfo.pData);
-  taosMemoryFreeClear(pMsgBody);
-}
-
 void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
   SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle;
   assert(pMsg->info.ahandle != NULL);
@@ -2064,6 +2058,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
   pWrapper->sourceIndex = sourceIndex;
 
   pMsgSendInfo->param = pWrapper;
+  pMsgSendInfo->paramFreeFp = taosMemoryFree;
   pMsgSendInfo->msgInfo.pData = pMsg;
   pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq);
   pMsgSendInfo->msgType = pSource->fetchMsgType;
@@ -4369,8 +4364,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC
 }
 
 SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
-                                  uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo,
+                                  STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* pUser) {
-                                  const char* pUser) {
   int32_t type = nodeType(pPhyNode);
 
   if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
@@ -4378,7 +4372,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
 
       int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags,
-                                             pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId);
+                                             pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
       if (code) {
         pTaskInfo->code = code;
         return NULL;
@@ -4398,7 +4392,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
       int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags,
-                                             pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId);
+                                             pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
       if (code) {
         pTaskInfo->code = code;
         return NULL;
@@ -4411,7 +4405,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       }
 
       SOperatorInfo* pOperator =
-          createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
+          createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo);
 
       STableScanInfo* pScanInfo = pOperator->info;
       pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
@@ -4426,17 +4420,16 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
             .calTrigger = pTableScanNode->triggerType,
             .maxTs = INT64_MIN,
       };
-      if (pHandle) {
+      if (pHandle->vnode) {
         int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags,
-                                               pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId);
+                                               pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
         if (code) {
           pTaskInfo->code = code;
           return NULL;
         }
       }
 
-      SOperatorInfo* pOperator =
+      SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo, &twSup);
-          createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId);
       return pOperator;
 
     } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
@@ -4445,7 +4438,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
       STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
 
-      int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTableListInfo);
+      int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTagCond, pTagIndexCond, pTableListInfo);
       if (code != TSDB_CODE_SUCCESS) {
         pTaskInfo->code = terrno;
         return NULL;
@@ -4481,8 +4474,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) {
       SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode;
 
-      int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo,
+      int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo));
-                                             queryId, taskId);
       if (code != TSDB_CODE_SUCCESS) {
         pTaskInfo->code = code;
         return NULL;
@@ -4506,7 +4498,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
   SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES);
   for (int32_t i = 0; i < size; ++i) {
     SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i);
-    ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo, pUser);
+    ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pUser);
     if (ops[i] == NULL) {
       return NULL;
     }
@@ -4594,7 +4586,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     int32_t children = 0;
     pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children);
   } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == type) {
-    int32_t children = 1;
+    int32_t children = pHandle->numOfVgroups;
     pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children);
   } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) {
     pOptr = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo);
@@ -4689,6 +4681,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) {
   return pList;
 }
 
+#if 0
 STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
                                 STableListInfo* pTableListInfo, const char* idstr) {
   int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo);
@@ -4722,6 +4715,7 @@ _error:
   terrno = code;
   return NULL;
 }
+#endif
 
 static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) {
   if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
@@ -4765,6 +4759,7 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) {
   return -1;
 }
 
+#if 0
 int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle, int64_t uid, int64_t ts) {
   STableScanInfo* pTableScanInfo = NULL;
   if (extractTbscanInStreamOpTree(pOperator, &pTableScanInfo) < 0) {
@@ -4788,6 +4783,7 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa
   // TODO: set uid and ts to data reader
   return 0;
 }
+#endif
 
 int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) {
   int32_t code = TDB_CODE_SUCCESS;
@@ -4939,10 +4935,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
   }
 
   (*pTaskInfo)->sql = sql;
-  (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond;
+  (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user);
-  (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond;
-  (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
-                                           &(*pTaskInfo)->tableqinfoList, pPlan->user);
 
   if (NULL == (*pTaskInfo)->pRoot) {
     code = (*pTaskInfo)->code;

source/libs/executor/src/scanoperator.c

@@ -1196,7 +1196,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
       if (pResCol->info.colId == pColMatchInfo->colId) {
         SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId);
         colDataAssign(pDst, pResCol, pBlock->info.rows, &pInfo->pRes->info);
-        //        taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol);
         colExists = true;
         break;
       }
@@ -1435,6 +1434,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
         }
       }
     }
+
     qDebug("scan rows: %d", pBlockInfo->rows);
     return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
 
@@ -1497,9 +1497,8 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) {
   taosMemoryFree(pStreamScan);
 }
 
-SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
-                                            SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId,
+                                            SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup) {
-                                            uint64_t taskId) {
   SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo));
   SOperatorInfo*   pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
 
@@ -1508,10 +1507,11 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     goto _error;
   }
 
-  SScanPhysiNode* pScanPhyNode = &pTableScanNode->scan;
+  SScanPhysiNode*     pScanPhyNode = &pTableScanNode->scan;
-
   SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
 
+  pInfo->pTagCond = pTagCond;
+
   int32_t numOfCols = 0;
   pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
 
@@ -1538,7 +1538,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     goto _error;
   }
 
-  if (pHandle) {
+  if (pHandle->vnode) {
     SOperatorInfo*  pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
     STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info;
     if (pHandle->version > 0) {
@@ -2550,26 +2550,19 @@ typedef struct STableMergeScanInfo {
   int32_t         tableEndIndex;
   bool            hasGroupId;
   uint64_t        groupId;
+  SArray*         dataReaders;  // array of tsdbReaderT*
+  SReadHandle     readHandle;
+  int32_t         bufPageSize;
+  uint32_t        sortBufSize;  // max buffer size for in-memory sort
+  SArray*         pSortInfo;
+  SSortHandle*    pSortHandle;
 
-  SArray*     dataReaders;  // array of tsdbReaderT*
+  SSDataBlock*    pSortInputBlock;
-  SReadHandle readHandle;
+  int64_t         startTs;  // sort start time
-
+  SArray*         sortSourceParams;
-  int32_t  bufPageSize;
-  uint32_t sortBufSize;  // max buffer size for in-memory sort
-
-  SArray*      pSortInfo;
-  SSortHandle* pSortHandle;
-
-  SSDataBlock* pSortInputBlock;
-  int64_t      startTs;  // sort start time
-
-  SArray*  sortSourceParams;
-  uint64_t queryId;
-  uint64_t taskId;
 
   SFileBlockLoadRecorder readRecorder;
-  int64_t                numOfRows;
+  int64_t         numOfRows;
-  //  int32_t         prevGroupId;  // previous table group id
   SScanInfo       scanInfo;
   int32_t         scanTimes;
   SNode*          pFilterNode;  // filter info, which is push down by optimizer
@@ -2584,26 +2577,25 @@ typedef struct STableMergeScanInfo {
   SExprInfo*      pPseudoExpr;
   int32_t         numOfPseudoExpr;
   SqlFunctionCtx* pPseudoCtx;
-  //  int32_t*        rowEntryInfoOffset;
 
   SQueryTableDataCond cond;
-  int32_t             scanFlag;  // table scan flag to denote if it is a repeat/reverse/main scan
+  int32_t         scanFlag;  // table scan flag to denote if it is a repeat/reverse/main scan
-  int32_t             dataBlockLoadFlag;
+  int32_t         dataBlockLoadFlag;
-  SInterval interval;  // if the upstream is an interval operator, the interval info is also kept here to get the time
+  // if the upstream is an interval operator, the interval info is also kept here to get the time
-                       // window to check if current data block needs to be loaded.
+  // window to check if current data block needs to be loaded.
-
+  SInterval       interval;
   SSampleExecInfo sample;  // sample execution info
 } STableMergeScanInfo;
 
 int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
-                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) {
+                                STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idStr) {
-  int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTableListInfo);
+  int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
 
   if (taosArrayGetSize(pTableListInfo->pTableList) == 0) {
-    qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
+    qDebug("no table qualified for query, %s" PRIx64, idStr);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -3027,8 +3019,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
 }
 
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo,
-                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId,
+                                                SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) {
-                                                uint64_t taskId) {
   STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo));
   SOperatorInfo*       pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -3067,9 +3058,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
 
   pInfo->pResBlock = createResDataBlock(pDescNode);
   pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
-  pInfo->queryId = queryId;
-  pInfo->taskId = taskId;
-
   pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
 
   pInfo->pSortInfo = generateSortByTsInfo(pInfo->pColMatchInfo, pInfo->cond.order);

source/libs/executor/src/timewindowoperator.c

@@ -63,7 +63,6 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo
                                       SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
                                       int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
                                       SExecTaskInfo* pTaskInfo) {
-  assert(win->skey <= win->ekey);
   SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE,
                                                   masterscan, tableGroupId, pTaskInfo, true, pAggSup);
 

source/libs/function/src/builtins.c

@@ -194,8 +194,9 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
 void static addTimezoneParam(SNodeList* pList) {
   char       buf[6] = {0};
   time_t     t = taosTime(NULL);
-  struct tm* tmInfo = taosLocalTime(&t, NULL);
+  struct tm  tmInfo;
-  strftime(buf, sizeof(buf), "%z", tmInfo);
+  taosLocalTime(&t, &tmInfo);
+  strftime(buf, sizeof(buf), "%z", &tmInfo);
   int32_t len = (int32_t)strlen(buf);
 
   SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
@@ -1422,14 +1423,6 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
 }
 
 static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
-  for (int32_t i = 0; i < numOfParams; ++i) {
-    SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
-    if (QUERY_NODE_VALUE == nodeType(pParamNode)) {
-      return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
-    }
-  }
-
   pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
   return TSDB_CODE_SUCCESS;
 }
@@ -2086,6 +2079,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getTopBotFuncEnv,
     .initFunc     = topBotFunctionSetup,
     .processFunc  = topFunction,
+    .sprocessFunc = topBotScalarFunction,
     .finalizeFunc = topBotFinalize,
     .combineFunc  = topCombine,
     .pPartialFunc = "top",
@@ -2100,6 +2094,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getTopBotFuncEnv,
     .initFunc     = topBotFunctionSetup,
     .processFunc  = bottomFunction,
+    .sprocessFunc = topBotScalarFunction,
     .finalizeFunc = topBotFinalize,
     .combineFunc  = bottomCombine,
     .pPartialFunc = "bottom",
@@ -2229,6 +2224,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = lastRowFunction,
+    .sprocessFunc = firstLastScalarFunction,
     .finalizeFunc = firstLastFinalize
   },
   {
@@ -2249,6 +2245,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = firstFunction,
+    .sprocessFunc = firstLastScalarFunction,
     .finalizeFunc = firstLastFinalize,
     .pPartialFunc = "_first_partial",
     .pMergeFunc   = "_first_merge",
@@ -2284,6 +2281,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = lastFunction,
+    .sprocessFunc = firstLastScalarFunction,
     .finalizeFunc = firstLastFinalize,
     .pPartialFunc = "_last_partial",
     .pMergeFunc   = "_last_merge",
@@ -2464,6 +2462,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getSampleFuncEnv,
     .initFunc     = sampleFunctionSetup,
     .processFunc  = sampleFunction,
+    .sprocessFunc = sampleScalarFunction,
     .finalizeFunc = sampleFinalize
   },
   {
@@ -2475,6 +2474,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getTailFuncEnv,
     .initFunc     = tailFunctionSetup,
     .processFunc  = tailFunction,
+    .sprocessFunc = tailScalarFunction,
     .finalizeFunc = NULL
   },
   {
@@ -2486,6 +2486,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getUniqueFuncEnv,
     .initFunc     = uniqueFunctionSetup,
     .processFunc  = uniqueFunction,
+    .sprocessFunc = uniqueScalarFunction,
     .finalizeFunc = NULL
   },
   {
@@ -2496,6 +2497,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getModeFuncEnv,
     .initFunc     = modeFunctionSetup,
     .processFunc  = modeFunction,
+    .sprocessFunc = modeScalarFunction,
     .finalizeFunc = modeFinalize,
   },
   {

source/libs/function/src/builtinsimpl.c

@@ -5098,12 +5098,7 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
   return true;
 }
 
-static void doModeAdd(SModeInfo* pInfo, char* data, bool isNull) {
+static void doModeAdd(SModeInfo* pInfo, char* data) {
-  // ignore null elements
-  if (isNull) {
-    return;
-  }
-
   int32_t     hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes;
   SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes);
   if (pHashItem == NULL) {
@@ -5128,10 +5123,16 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) {
   SColumnInfoData* pInputCol = pInput->pData[0];
   SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
 
+  int32_t numOfElems = 0;
   int32_t startOffset = pCtx->offset;
   for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
     char* data = colDataGetData(pInputCol, i);
-    doModeAdd(pInfo, data, colDataIsNull_s(pInputCol, i));
+    if (colDataIsNull_s(pInputCol, i)) {
+      continue;
+    }
+
+    numOfElems++;
+    doModeAdd(pInfo, data);
 
     if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) {
       taosHashCleanup(pInfo->pHash);
@@ -5139,7 +5140,7 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) {
     }
   }
 
-  SET_VAL(pResInfo, 1, 1);
+  SET_VAL(pResInfo, numOfElems, 1);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -5661,7 +5662,12 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) {
         double r = ((v - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs);
         if (pDerivInfo->ignoreNegative && r < 0) {
         } else {
-          colDataAppend(pOutput, pos, (const char*)&r, false);
+          if (isinf(r) || isnan(r)) {
+            colDataAppendNULL(pOutput, pos);
+          } else {
+            colDataAppend(pOutput, pos, (const char*)&r, false);
+          }
+
           if (pTsOutput != NULL) {
             colDataAppendInt64(pTsOutput, pos, &tsList[i]);
           }
@@ -5688,7 +5694,12 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) {
         double r = ((pDerivInfo->prevValue - v) * pDerivInfo->tsWindow) / (pDerivInfo->prevTs - tsList[i]);
         if (pDerivInfo->ignoreNegative && r < 0) {
         } else {
-          colDataAppend(pOutput, pos, (const char*)&r, false);
+          if (isinf(r) || isnan(r)) {
+            colDataAppendNULL(pOutput, pos);
+          } else {
+            colDataAppend(pOutput, pos, (const char*)&r, false);
+          }
+
           if (pTsOutput != NULL) {
             colDataAppendInt64(pTsOutput, pos, &pDerivInfo->prevTs);
           }

source/libs/nodes/src/nodesCodeFuncs.c

@@ -548,37 +548,12 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols";
 static const char* jkScanLogicPlanTableType = "TableType";
 static const char* jkScanLogicPlanTableId = "TableId";
 static const char* jkScanLogicPlanStableId = "StableId";
+static const char* jkScanLogicPlanScanType = "ScanType";
 static const char* jkScanLogicPlanScanCount = "ScanCount";
 static const char* jkScanLogicPlanReverseScanCount = "ReverseScanCount";
 static const char* jkScanLogicPlanTagCond = "TagCond";
 static const char* jkScanLogicPlanGroupTags = "GroupTags";
 
-// typedef struct SScanLogicNode {
-//   uint64_t      stableId;
-//   SVgroupsInfo* pVgroupList;
-//   EScanType     scanType;
-//   uint8_t       scanSeq[2];  // first is scan count, and second is reverse scan count
-//   STimeWindow   scanRange;
-//   SName         tableName;
-//   bool          showRewrite;
-//   double        ratio;
-//   SNodeList*    pDynamicScanFuncs;
-//   int32_t       dataRequired;
-//   int64_t       interval;
-//   int64_t       offset;
-//   int64_t       sliding;
-//   int8_t        intervalUnit;
-//   int8_t        slidingUnit;
-//   SNode*        pTagCond;
-//   SNode*        pTagIndexCond;
-//   int8_t        triggerType;
-//   int64_t       watermark;
-//   int8_t        igExpired;
-//   SArray*       pSmaIndexes;
-//   SNodeList*    pGroupTags;
-//   bool          groupSort;
-// } SScanLogicNode;
-
 static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
 
@@ -598,6 +573,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanStableId, pNode->stableId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanType, pNode->scanType);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanCount, pNode->scanSeq[0]);
   }
@@ -634,6 +612,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanStableId, &pNode->stableId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    tjsonGetNumberValue(pJson, jkScanLogicPlanScanType, pNode->scanType, code);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetUTinyIntValue(pJson, jkScanLogicPlanScanCount, &pNode->scanSeq[0]);
   }
@@ -1677,6 +1658,7 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) {
 }
 
 static const char* jkProjectPhysiPlanProjections = "Projections";
+static const char* jkProjectPhysiPlanMergeDataBlock = "MergeDataBlock";
 
 static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj;
@@ -1685,6 +1667,9 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkProjectPhysiPlanMergeDataBlock, pNode->mergeDataBlock);
+  }
 
   return code;
 }
@@ -1696,6 +1681,9 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkProjectPhysiPlanMergeDataBlock, &pNode->mergeDataBlock);
+  }
 
   return code;
 }

source/libs/nodes/src/nodesUtilFuncs.c

@@ -1923,15 +1923,18 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode**
     return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagIndexCond, pTagCond, pOtherCond);
   }
 
+  bool needOutput = false;
   switch (classifyCondition(*pCondition)) {
     case COND_TYPE_PRIMARY_KEY:
       if (NULL != pPrimaryKeyCond) {
         *pPrimaryKeyCond = *pCondition;
+        needOutput = true;
       }
       break;
     case COND_TYPE_TAG_INDEX:
       if (NULL != pTagIndexCond) {
         *pTagIndexCond = *pCondition;
+        needOutput = true;
       }
       if (NULL != pTagCond) {
         SNode* pTempCond = *pCondition;
@@ -1942,21 +1945,26 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode**
           }
         }
         *pTagCond = pTempCond;
+        needOutput = true;
       }
       break;
     case COND_TYPE_TAG:
       if (NULL != pTagCond) {
         *pTagCond = *pCondition;
+        needOutput = true;
       }
       break;
     case COND_TYPE_NORMAL:
     default:
       if (NULL != pOtherCond) {
         *pOtherCond = *pCondition;
+        needOutput = true;
       }
       break;
   }
-  *pCondition = NULL;
+  if (needOutput) {
+    *pCondition = NULL;
+  }
 
   return TSDB_CODE_SUCCESS;
 }

source/libs/parser/src/parCalcConst.c

@@ -201,7 +201,8 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) {
 }
 
 static bool isUselessCol(SExprNode* pProj) {
-  if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId)) {
+  if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId) &&
+      !fmIsPseudoColumnFunc(((SFunctionNode*)pProj)->funcId)) {
     return false;
   }
   return NULL == ((SExprNode*)pProj)->pAssociation;

source/libs/parser/src/parInsert.c

@@ -2161,7 +2161,7 @@ static void smlDestroyTableHandle(void* pHandle) {
   tdDestroySVCreateTbReq(&handle->createTblReq);
 }
 
-static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema) {
+static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema, bool isTag) {
   col_id_t nCols = pColList->numOfCols;
 
   pColList->numOfBound = 0;
@@ -2177,7 +2177,8 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS
     SSmlKv*  kv = taosArrayGetP(cols, i);
     SToken   sToken = {.n = kv->keyLen, .z = (char*)kv->key};
     col_id_t t = lastColIdx + 1;
-    col_id_t index = findCol(&sToken, t, nCols, pSchema);
+    col_id_t index = ((t == 0 && !isTag) ? 0 : findCol(&sToken, t, nCols, pSchema));
+    uDebug("SML, index:%d, t:%d, ncols:%d, kv->name:%s", index, t, nCols, kv->key);
     if (index < 0 && t > 0) {
       index = findCol(&sToken, 0, t, pSchema);
       isOrdered = false;
@@ -2312,7 +2313,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
   smlDestroyTableHandle(&smlHandle->tableExecHandle);  // free for each table
   SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
   setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta));
-  int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema);
+  int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, true);
   if (ret != TSDB_CODE_SUCCESS) {
     buildInvalidOperationMsg(&pBuf, "bound tags error");
     return ret;
@@ -2343,7 +2344,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
 
   SSchema* pSchema = getTableColumnSchema(pTableMeta);
 
-  ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema);
+  ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema, false);
   if (ret != TSDB_CODE_SUCCESS) {
     buildInvalidOperationMsg(&pBuf, "bound cols error");
     return ret;
@@ -2384,7 +2385,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
       if (format) {
         if (j < rowDataSize) {
           kv = taosArrayGetP(rowData, j);
-          if (rowDataSize != spd->numOfBound &&
+          if (rowDataSize != spd->numOfBound && j != 0 &&
               (kv->keyLen != strlen(pColSchema->name) || strncmp(kv->key, pColSchema->name, kv->keyLen) != 0)) {
             kv = NULL;
           } else {
@@ -2401,7 +2402,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
       } else {
         int32_t colLen = kv->length;
         if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
+//          uError("SML:data before:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision);
           kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);
+//          uError("SML:data after:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision);
         }
 
         if (IS_VAR_DATA_TYPE(kv->type)) {

source/libs/parser/src/parTranslater.c

@@ -1686,7 +1686,7 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) {
 static bool sysTableFromVnode(const char* pTable) {
   return (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLES)) ||
          (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) ||
-         (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS)));
+          (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS)));
 }
 
 static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); }
@@ -5968,6 +5968,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
   if (NULL == pReq->tagName) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReq->colId = pSchema->colId;
 
   SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema);
   if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) {
@@ -6051,6 +6052,7 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt,
   if (NULL == pReq->colName) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReq->colId = pSchema->colId;
 
   return TSDB_CODE_SUCCESS;
 }
@@ -6071,6 +6073,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt
   if (NULL == pReq->colName) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReq->colId = pSchema->colId;
 
   return TSDB_CODE_SUCCESS;
 }

source/libs/planner/src/planLogicCreater.c

@@ -20,6 +20,7 @@
 typedef struct SLogicPlanContext {
   SPlanContext* pPlanCxt;
   SLogicNode*   pCurrRoot;
+  bool          hasScan;
 } SLogicPlanContext;
 
 typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, void*, SLogicNode**);
@@ -161,6 +162,10 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
     return SCAN_TYPE_STREAM;
   }
 
+  if (TSDB_SYSTEM_TABLE == tableType) {
+    return SCAN_TYPE_SYSTEM_TABLE;
+  }
+
   if (NULL == pScanCols) {
     return NULL == pScanPseudoCols
                ? SCAN_TYPE_TABLE
@@ -169,17 +174,6 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
                       : SCAN_TYPE_TABLE);
   }
 
-  if (TSDB_SYSTEM_TABLE == tableType) {
-    return SCAN_TYPE_SYSTEM_TABLE;
-  }
-
-  SNode* pCol = NULL;
-  FOREACH(pCol, pScanCols) {
-    if (COLUMN_TYPE_COLUMN == ((SColumnNode*)pCol)->colType) {
-      return SCAN_TYPE_TABLE;
-    }
-  }
-
   return SCAN_TYPE_TABLE;
 }
 
@@ -300,6 +294,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
     nodesDestroyNode((SNode*)pScan);
   }
 
+  pCxt->hasScan = true;
+
   return code;
 }
 
@@ -1339,9 +1335,9 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) {
 
 static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); }
 
-static void setLogicSubplanType(SLogicSubplan* pSubplan) {
+static void setLogicSubplanType(bool hasScan, SLogicSubplan* pSubplan) {
   if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY != nodeType(pSubplan->pNode)) {
-    pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
+    pSubplan->subplanType = hasScan ? SUBPLAN_TYPE_SCAN : SUBPLAN_TYPE_MERGE;
   } else {
     SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pSubplan->pNode;
     pSubplan->subplanType = (MODIFY_TABLE_TYPE_INSERT == pModify->modifyType && NULL != pModify->node.pChildren)
@@ -1351,7 +1347,7 @@ static void setLogicSubplanType(SLogicSubplan* pSubplan) {
 }
 
 int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) {
-  SLogicPlanContext cxt = {.pPlanCxt = pCxt};
+  SLogicPlanContext cxt = {.pPlanCxt = pCxt, .pCurrRoot = NULL, .hasScan = false};
 
   SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
   if (NULL == pSubplan) {
@@ -1364,7 +1360,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) {
   int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode);
   if (TSDB_CODE_SUCCESS == code) {
     setLogicNodeParent(pSubplan->pNode);
-    setLogicSubplanType(pSubplan);
+    setLogicSubplanType(cxt.hasScan, pSubplan);
   }
 
   if (TSDB_CODE_SUCCESS == code) {

source/libs/planner/src/planPhysiCreater.c

@@ -35,7 +35,8 @@ typedef struct SPhysiPlanContext {
   int32_t       errCode;
   int16_t       nextDataBlockId;
   SArray*       pLocationHelper;
-  SArray*       pExecNodeList;  // SArray<SQueryNodeLoad>
+  bool          hasScan;
+  bool          hasSysScan;
 } SPhysiPlanContext;
 
 static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
@@ -255,7 +256,7 @@ static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlo
 
 static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList,
                                            SDataBlockDescNode* pDataBlockDesc) {
-  return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true, false);
+  return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, false, false);
 }
 
 static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) {
@@ -495,8 +496,6 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
-  SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0};
-  taosArrayPush(pCxt->pExecNodeList, &node);
   return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode);
 }
 
@@ -577,8 +576,6 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
     pSubplan->execNode.nodeId = MNODE_HANDLE;
     pSubplan->execNode.epSet = pCxt->pPlanCxt->mgmtEpSet;
   }
-  SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0};
-  taosArrayPush(pCxt->pExecNodeList, &node);
   if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_DNODE_VARIABLES)) {
     pScan->mgmtEpSet = pScanLogicNode->pVgroupList->vgroups->epSet;
   } else {
@@ -586,6 +583,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
   }
   tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
 
+  pCxt->hasSysScan = true;
   return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
 }
 
@@ -601,6 +599,7 @@ static int32_t createTableMergeScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
 
 static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode,
                                    SPhysiNode** pPhyNode) {
+  pCxt->hasScan = true;
   switch (pScanLogicNode->scanType) {
     case SCAN_TYPE_TAG:
     case SCAN_TYPE_BLOCK_INFO:
@@ -1806,23 +1805,31 @@ static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) {
   }
 }
 
+static void setExecNodeList(SPhysiPlanContext* pCxt, SArray* pExecNodeList) {
+  if (NULL == pExecNodeList) {
+    return;
+  }
+  if (pCxt->hasSysScan || !pCxt->hasScan) {
+    SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0};
+    taosArrayPush(pExecNodeList, &node);
+  }
+}
+
 int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) {
   SPhysiPlanContext cxt = {.pPlanCxt = pCxt,
                            .errCode = TSDB_CODE_SUCCESS,
                            .nextDataBlockId = 0,
                            .pLocationHelper = taosArrayInit(32, POINTER_BYTES),
-                           .pExecNodeList = pExecNodeList};
+                           .hasScan = false,
+                           .hasSysScan = false};
   if (NULL == cxt.pLocationHelper) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  if (QUERY_POLICY_VNODE == tsQueryPolicy) {
-    taosArrayClear(pExecNodeList);
-  }
-
   int32_t code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan);
   if (TSDB_CODE_SUCCESS == code) {
     setExplainInfo(pCxt, *pPlan);
+    setExecNodeList(&cxt, pExecNodeList);
   }
 
   destoryPhysiPlanContext(&cxt);

source/libs/qcom/src/queryUtil.c

@@ -138,6 +138,16 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code)
   return 0;
 }
 
+void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
+  assert(pMsgBody != NULL);
+  taosMemoryFreeClear(pMsgBody->target.dbFName);
+  taosMemoryFreeClear(pMsgBody->msgInfo.pData);
+  if (pMsgBody->paramFreeFp) {
+    (*pMsgBody->paramFreeFp)(pMsgBody->param);
+  }
+  taosMemoryFreeClear(pMsgBody);
+}
+
 int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo,
                                 bool persistHandle, void* rpcCtx) {
   char* pMsg = rpcMallocCont(pInfo->msgInfo.len);

source/libs/scalar/src/sclfunc.c

@@ -1062,8 +1062,9 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *
       memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS);
     }
 
-    struct tm *tmInfo = taosLocalTime((const time_t *)&timeVal, NULL);
+    struct tm tmInfo;
-    strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", tmInfo);
+    taosLocalTime((const time_t *)&timeVal, &tmInfo);
+    strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo);
     int32_t len = (int32_t)strlen(buf);
 
     //add timezone string
@@ -2816,3 +2817,48 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
   pOutput->numOfRows = numOfBins;
   return TSDB_CODE_SUCCESS;
 }
+
+int32_t selectScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  SColumnInfoData *pInputData  = pInput->columnData;
+  SColumnInfoData *pOutputData = pOutput->columnData;
+
+  int32_t type = GET_PARAM_TYPE(pInput);
+
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    if (colDataIsNull_s(pInputData, i)) {
+      colDataAppendNULL(pOutputData, 0);
+      continue;
+    }
+
+    char* data = colDataGetData(pInputData, i);
+    colDataAppend(pOutputData, i, data, false);
+  }
+
+
+  pOutput->numOfRows = 1;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}
+
+int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return selectScalarFunction(pInput, inputNum, pOutput);
+}