Merge branch 'enh/dev3.0' into enh/addCompileError

README-CN.md

@@ -52,7 +52,7 @@ TDengine 还提供一组辅助工具软件 taosTools，目前它包含 taosBench
 ### Ubuntu 18.04 及以上版本 & Debian：
 
 ```bash
-sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev
+sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev libgeos-dev
 ```
 
 #### 为 taos-tools 安装编译需要的软件
@@ -60,7 +60,7 @@ sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 li
 为了在 Ubuntu/Debian 系统上编译 [taos-tools](https://github.com/taosdata/taos-tools) 需要安装如下软件：
 
 ```bash
-sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libgeos-dev
+sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config
 ```
 
 ### CentOS 7.9
@@ -68,14 +68,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d
 ```bash
 sudo yum install epel-release
 sudo yum update
-sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel
+sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel geos geos-devel
 sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake
 ```
 
-### CentOS 8 & Fedora
+### CentOS 8/Fedora/Rocky Linux
 
 ```bash
-sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
+sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel geos geos-devel
 ```
 
 #### 在 CentOS 上构建 taosTools 安装依赖软件
@@ -85,23 +85,23 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
 
 
 ```
-sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel
+sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel
 ```
 
-#### CentOS 8/Rocky Linux 
+#### CentOS 8/Fedora/Rocky Linux 
 
 ```
 sudo yum install -y epel-release
 sudo yum install -y dnf-plugins-core
 sudo yum config-manager --set-enabled powertools
-sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel
+sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel
 ```
 
 注意：由于 snappy 缺乏 pkg-config 支持（参考 [链接](https://github.com/google/snappy/pull/86)），会导致 cmake 提示无法发现 libsnappy，实际上工作正常。
 
 若 powertools 安装失败，可以尝试改用：
 ```
-sudo yum config-manager --set-enabled Powertools
+sudo yum config-manager --set-enabled powertools
 ```
 
 #### CentOS + devtoolset

README.md

@@ -60,7 +60,7 @@ To build TDengine, use [CMake](https://cmake.org/) 3.0.2 or higher versions in t
 ### Ubuntu 18.04 and above or Debian
 
 ```bash
-sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev
+sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev libgeos-dev
 ```
 
 #### Install build dependencies for taosTools
@@ -68,7 +68,7 @@ sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 li
 To build the [taosTools](https://github.com/taosdata/taos-tools) on Ubuntu/Debian, the following packages need to be installed.
 
 ```bash
-sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libgeos-dev
+sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config
 ```
 
 ### CentOS 7.9
@@ -76,14 +76,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d
 ```bash
 sudo yum install epel-release
 sudo yum update
-sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel
+sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel geos geos-devel
 sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake
 ```
 
-### CentOS 8 & Fedora
+### CentOS 8/Fedora/Rocky Linux
 
 ```bash
-sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
+sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel geos geos-devel
 ```
 
 #### Install build dependencies for taosTools on CentOS
@@ -91,16 +91,16 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
 #### CentOS 7.9
 
 ```
-sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel
+sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel
 ```
 
-#### CentOS 8/Rocky Linux
+#### CentOS 8/Fedora/Rocky Linux
 
 ```
 sudo yum install -y epel-release
 sudo yum install -y dnf-plugins-core
 sudo yum config-manager --set-enabled powertools
-sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel
+sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel
 ```
 
 Note: Since snappy lacks pkg-config support (refer to [link](https://github.com/google/snappy/pull/86)), it leads a cmake prompt libsnappy not found. But snappy still works well.

cmake/taosadapter_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG 283b50d
+        GIT_TAG 3.0
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/en/07-develop/09-udf.md

@@ -377,7 +377,7 @@ The `pybitand` function implements bitwise addition for multiple columns. If the
 
 #### Aggregate Function [pyl2norm](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pyl2norm.py)
 
-The `pyl2norm` function finds the second-order norm for all data in the input column. This squares the values, takes a cumulative sum, and finds the square root.
+The `pyl2norm` function finds the second-order norm for all data in the input columns. This squares the values, takes a cumulative sum, and finds the square root.
 <details>
 <summary>pyl2norm.py</summary>
 
@@ -387,5 +387,16 @@ The `pyl2norm` function finds the second-order norm for all data in the input co
 
 </details>
 
+#### Aggregate Function [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py)
+
+The `pycumsum` function finds the cumulative sum for all data in the input columns.
+<details>
+<summary>pycumsum.py</summary>
+
+```c
+{{#include tests/script/sh/pycumsum.py}}
+```
+
+</details>
 ## Manage and Use UDF
 You need to add UDF to TDengine before using it in SQL queries. For more information about how to manage UDF and how to invoke UDF, please see [Manage and Use UDF](../12-taos-sql/26-udf.md).

docs/en/14-reference/03-connector/04-java.mdx

@@ -959,6 +959,7 @@ The preceding example uses the SQL statement `select ts, speed from speed_table`
 
 ```java
 Properties config = new Properties();
+config.setProperty("bootstrap.servers", "localhost:6030");
 config.setProperty("enable.auto.commit", "true");
 config.setProperty("group.id", "group1");
 config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer");
@@ -966,12 +967,14 @@ config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.Res
 TaosConsumer consumer = new TaosConsumer<>(config);
 ```
 
+- bootstrap.servers: `ip:port` where the TDengine server is located, or `ip:port` where the taosAdapter is located if WebSocket connection is used.
 - enable.auto.commit: Specifies whether to commit automatically.
 - group.id: consumer: Specifies the group that the consumer is in.
 - value.deserializer: To deserialize the results, you can inherit `com.taosdata.jdbc.tmq.ReferenceDeserializer` and specify the result set bean. You can also inherit `com.taosdata.jdbc.tmq.Deserializer` and perform custom deserialization based on the SQL result set.
 - td.connect.type: Specifies the type connect with TDengine, `jni` or `WebSocket`. default is `jni`
 - httpConnectTimeout: WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type.
 - messageWaitTimeout: socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type.
+- httpPoolSize: Maximum number of concurrent requests on the a connection。It only takes effect when using WebSocket type.
 - For more information, see [Consumer Parameters](../../../develop/tmq).
 
 #### Subscribe to consume data
@@ -1015,10 +1018,20 @@ public abstract class ConsumerLoop {
 
     public ConsumerLoop() throws SQLException {
         Properties config = new Properties();
+        config.setProperty("td.connect.type", "jni");
+        config.setProperty("bootstrap.servers", "localhost:6030");
+        config.setProperty("td.connect.user", "root");
+        config.setProperty("td.connect.pass", "taosdata");
+        config.setProperty("auto.offset.reset", "earliest");
         config.setProperty("msg.with.table.name", "true");
         config.setProperty("enable.auto.commit", "true");
+        config.setProperty("auto.commit.interval.ms", "1000");
         config.setProperty("group.id", "group1");
+        config.setProperty("client.id", "1");
         config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+        config.setProperty("value.deserializer.encoding", "UTF-8");
+        config.setProperty("experimental.snapshot.enable", "true");
+
 
         this.consumer = new TaosConsumer<>(config);
         this.topics = Collections.singletonList("topic_speed");
@@ -1090,12 +1103,19 @@ public abstract class ConsumerLoop {
 
     public ConsumerLoop() throws SQLException {
         Properties config = new Properties();
-        config.setProperty("bootstrap.servers", "localhost:6041");
         config.setProperty("td.connect.type", "ws");
+        config.setProperty("bootstrap.servers", "localhost:6041");
+        config.setProperty("td.connect.user", "root");
+        config.setProperty("td.connect.pass", "taosdata");
+        config.setProperty("auto.offset.reset", "earliest");
         config.setProperty("msg.with.table.name", "true");
         config.setProperty("enable.auto.commit", "true");
+        config.setProperty("auto.commit.interval.ms", "1000");
         config.setProperty("group.id", "group2");
+        config.setProperty("client.id", "1");
         config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+        config.setProperty("value.deserializer.encoding", "UTF-8");
+        config.setProperty("experimental.snapshot.enable", "true");
 
         this.consumer = new TaosConsumer<>(config);
         this.topics = Collections.singletonList("topic_speed");

docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java

@@ -53,20 +53,28 @@ public class SubscribeDemo {
 
             // create consumer
             Properties properties = new Properties();
+            properties.getProperty(TMQConstants.CONNECT_TYPE, "jni");
             properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6030");
+            properties.setProperty(TMQConstants.CONNECT_USER, "root");
+            properties.setProperty(TMQConstants.CONNECT_PASS, "taosdata");
             properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true");
             properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true");
-            properties.setProperty(TMQConstants.GROUP_ID, "test");
+            properties.setProperty(TMQConstants.AUTO_COMMIT_INTERVAL, "1000");
+            properties.setProperty(TMQConstants.GROUP_ID, "test1");
+            properties.setProperty(TMQConstants.CLIENT_ID, "1");
+            properties.setProperty(TMQConstants.AUTO_OFFSET_RESET, "earliest");
             properties.setProperty(TMQConstants.VALUE_DESERIALIZER,
                     "com.taos.example.MetersDeserializer");
+            properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8");
+            properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true");
 
             // poll data
             try (TaosConsumer<Meters> consumer = new TaosConsumer<>(properties)) {
                 consumer.subscribe(Collections.singletonList(TOPIC));
                 while (!shutdown.get()) {
                     ConsumerRecords<Meters> meters = consumer.poll(Duration.ofMillis(100));
-                    for (ConsumerRecord<Meters> recode : meters) {
+                    for (ConsumerRecord<Meters> r : meters) {
-                        Meters meter = recode.value();
+                        Meters meter = r.value();
                         System.out.println(meter);
                     }
                 }

docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java

@@ -1,5 +1,6 @@
 package com.taos.example;
 
+import com.taosdata.jdbc.tmq.ConsumerRecord;
 import com.taosdata.jdbc.tmq.ConsumerRecords;
 import com.taosdata.jdbc.tmq.TMQConstants;
 import com.taosdata.jdbc.tmq.TaosConsumer;
@@ -54,18 +55,26 @@ public class WebsocketSubscribeDemo {
             Properties properties = new Properties();
             properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6041");
             properties.setProperty(TMQConstants.CONNECT_TYPE, "ws");
+            properties.setProperty(TMQConstants.CONNECT_USER, "root");
+            properties.setProperty(TMQConstants.CONNECT_PASS, "taosdata");
+            properties.setProperty(TMQConstants.AUTO_OFFSET_RESET, "earliest");
             properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true");
             properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true");
-            properties.setProperty(TMQConstants.GROUP_ID, "test");
+            properties.setProperty(TMQConstants.AUTO_COMMIT_INTERVAL, "1000");
+            properties.setProperty(TMQConstants.GROUP_ID, "test2");
+            properties.setProperty(TMQConstants.CLIENT_ID, "1");
             properties.setProperty(TMQConstants.VALUE_DESERIALIZER,
                     "com.taos.example.MetersDeserializer");
+            properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8");
+            properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true");
 
             // poll data
             try (TaosConsumer<Meters> consumer = new TaosConsumer<>(properties)) {
                 consumer.subscribe(Collections.singletonList(TOPIC));
                 while (!shutdown.get()) {
                     ConsumerRecords<Meters> meters = consumer.poll(Duration.ofMillis(100));
-                    for (Meters meter : meters) {
+                    for (ConsumerRecord<Meters> r : meters) {
+                        Meters meter = (Meters) r.value();
                         System.out.println(meter);
                     }
                 }

docs/zh/07-develop/09-udf.md

@@ -335,7 +335,7 @@ def init()
 def destroy()
 ```
 
-其中 init 完成初始化工作。 destroy 完成清理工作。如果没有初始化工作，无需定义 init 函数。如果没有清理工作，无需定义 destroy 函数。
+其中 init 完成初始化工作。 destroy 完成清理工作。
 
 ### Python 和 TDengine之间的数据类型映射
 
@@ -386,6 +386,17 @@ pyl2norm 实现了输入列的所有数据的二阶范数，即对每个数据
 
 </details>
 
+### 聚合函数示例 [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py)
+
+pycumsum 使用 numpy 计算输入列所有数据的累积和。
+<details>
+<summary>pycumsum.py</summary>
+
+```c
+{{#include tests/script/sh/pycumsum.py}}
+```
+
+</details>
 ## 管理和使用 UDF
 在使用 UDF 之前需要先将其加入到 TDengine 系统中。关于如何管理和使用 UDF，请参考[管理和使用 UDF](../12-taos-sql/26-udf.md)
 

docs/zh/08-connector/14-java.mdx

@@ -962,6 +962,7 @@ statement.executeUpdate("create topic if not exists topic_speed as select ts, sp
 
 ```java
 Properties config = new Properties();
+config.setProperty("bootstrap.servers", "localhost:6030");
 config.setProperty("enable.auto.commit", "true");
 config.setProperty("group.id", "group1");
 config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer");
@@ -969,12 +970,14 @@ config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.Res
 TaosConsumer consumer = new TaosConsumer<>(config);
 ```
 
+- bootstrap.servers: TDengine 服务端所在的`ip:port`，如果使用 WebSocket 连接，则为 taosAdapter 所在的`ip:port`。
 - enable.auto.commit: 是否允许自动提交。
 - group.id: consumer: 所在的 group。
 - value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。
 - td.connect.type: 连接方式。jni：表示使用动态库连接的方式，ws/WebSocket：表示使用 WebSocket 进行数据通信。默认为 jni 方式。
-- httpConnectTimeout：创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。
+- httpConnectTimeout: 创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。
-- messageWaitTimeout：数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。
+- messageWaitTimeout: 数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。
+- httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。
   其他参数请参考：[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group)
 
 #### 订阅消费数据
@@ -1016,10 +1019,19 @@ public abstract class ConsumerLoop {
 
     public ConsumerLoop() throws SQLException {
         Properties config = new Properties();
+        config.setProperty("td.connect.type", "jni");
+        config.setProperty("bootstrap.servers", "localhost:6030");
+        config.setProperty("td.connect.user", "root");
+        config.setProperty("td.connect.pass", "taosdata");
+        config.setProperty("auto.offset.reset", "earliest");
         config.setProperty("msg.with.table.name", "true");
         config.setProperty("enable.auto.commit", "true");
+        config.setProperty("auto.commit.interval.ms", "1000");
         config.setProperty("group.id", "group1");
+        config.setProperty("client.id", "1");
         config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+        config.setProperty("value.deserializer.encoding", "UTF-8");
+        config.setProperty("experimental.snapshot.enable", "true");
 
         this.consumer = new TaosConsumer<>(config);
         this.topics = Collections.singletonList("topic_speed");
@@ -1093,12 +1105,19 @@ public abstract class ConsumerLoop {
 
     public ConsumerLoop() throws SQLException {
         Properties config = new Properties();
-        config.setProperty("bootstrap.servers", "localhost:6041");
         config.setProperty("td.connect.type", "ws");
+        config.setProperty("bootstrap.servers", "localhost:6041");
+        config.setProperty("td.connect.user", "root");
+        config.setProperty("td.connect.pass", "taosdata");
+        config.setProperty("auto.offset.reset", "earliest");
         config.setProperty("msg.with.table.name", "true");
         config.setProperty("enable.auto.commit", "true");
+        config.setProperty("auto.commit.interval.ms", "1000");
         config.setProperty("group.id", "group2");
+        config.setProperty("client.id", "1");
         config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+        config.setProperty("value.deserializer.encoding", "UTF-8");
+        config.setProperty("experimental.snapshot.enable", "true");
 
         this.consumer = new TaosConsumer<>(config);
         this.topics = Collections.singletonList("topic_speed");

include/common/tcommon.h

@@ -37,6 +37,13 @@ extern "C" {
 )
 // clang-format on
 
+typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
+
+typedef struct STableKeyInfo {
+  uint64_t uid;
+  uint64_t groupId;
+} STableKeyInfo;
+
 typedef struct SWinKey {
   uint64_t groupId;
   TSKEY    ts;

include/libs/executor/dataSinkMgt.h

@@ -59,7 +59,7 @@ typedef struct SDataSinkMgtCfg {
   uint32_t maxDataBlockNumPerQuery;
 } SDataSinkMgtCfg;
 
-int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg);
+int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI);
 
 typedef struct SInputData {
   const struct SSDataBlock* pData;

include/libs/executor/executor.h

@@ -23,6 +23,7 @@ extern "C" {
 #include "query.h"
 #include "tcommon.h"
 #include "tmsgcb.h"
+#include "storageapi.h"
 
 typedef void* qTaskInfo_t;
 typedef void* DataSinkHandle;
@@ -41,7 +42,6 @@ typedef struct {
 
 typedef struct {
   void*   tqReader;
-  void*   meta;
   void*   config;
   void*   vnode;
   void*   mnd;
@@ -51,10 +51,10 @@ typedef struct {
   bool    initTableReader;
   bool    initTqReader;
   int32_t numOfVgroups;
+  void*   sContext;  // SSnapContext*
 
-  void* sContext;  // SSnapContext*
+  void*   pStateBackend;
-
+  struct SStorageAPI api;
-  void* pStateBackend;
 } SReadHandle;
 
 // in queue mode, data streams are seperated by msg

include/libs/executor/storageapi.h

@@ -0,0 +1,523 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_STORAGEAPI_H
+#define TDENGINE_STORAGEAPI_H
+
+#include "tsimplehash.h"
+#include "tscalablebf.h"
+#include "taosdef.h"
+#include "tmsg.h"
+#include "tcommon.h"
+#include "index.h"
+#include "function.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TIMEWINDOW_RANGE_CONTAINED 1
+#define TIMEWINDOW_RANGE_EXTERNAL  2
+
+#define CACHESCAN_RETRIEVE_TYPE_ALL    0x1
+#define CACHESCAN_RETRIEVE_TYPE_SINGLE 0x2
+#define CACHESCAN_RETRIEVE_LAST_ROW    0x4
+#define CACHESCAN_RETRIEVE_LAST        0x8
+
+#define META_READER_NOLOCK 0x1
+
+typedef struct SMeta SMeta;
+typedef TSKEY (*GetTsFun)(void*);
+
+typedef struct SMetaEntry {
+  int64_t  version;
+  int8_t   type;
+  int8_t   flags;  // TODO: need refactor?
+  tb_uid_t uid;
+  char *   name;
+  union {
+    struct {
+      SSchemaWrapper schemaRow;
+      SSchemaWrapper schemaTag;
+      SRSmaParam     rsmaParam;
+    } stbEntry;
+    struct {
+      int64_t  ctime;
+      int32_t  ttlDays;
+      int32_t  commentLen;
+      char *   comment;
+      tb_uid_t suid;
+      uint8_t *pTags;
+    } ctbEntry;
+    struct {
+      int64_t        ctime;
+      int32_t        ttlDays;
+      int32_t        commentLen;
+      char *         comment;
+      int32_t        ncid;  // next column id
+      SSchemaWrapper schemaRow;
+    } ntbEntry;
+    struct {
+      STSma *tsma;
+    } smaEntry;
+  };
+
+  uint8_t *pBuf;
+} SMetaEntry;
+
+typedef struct SMetaReader {
+  int32_t             flags;
+  void *              pMeta;
+  SDecoder            coder;
+  SMetaEntry          me;
+  void *              pBuf;
+  int32_t             szBuf;
+  struct SStoreMeta*  pAPI;
+} SMetaReader;
+
+typedef struct SMTbCursor {
+  void *      pDbc;
+  void *      pKey;
+  void *      pVal;
+  int32_t     kLen;
+  int32_t     vLen;
+  SMetaReader mr;
+} SMTbCursor;
+
+typedef struct SRowBuffPos {
+  void* pRowBuff;
+  void* pKey;
+  bool  beFlushed;
+  bool  beUsed;
+} SRowBuffPos;
+
+// int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables);
+// int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
+//                                SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
+// int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
+//                              SArray *pTableUids);
+// void   *tsdbCacherowsReaderClose(void *pReader);
+// int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
+
+// int32_t      tsdbReaderOpen(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
+//                            SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables);
+// int32_t      tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
+// void         tsdbReaderSetId(STsdbReader *pReader, const char *idstr);
+// void         tsdbReaderClose(STsdbReader *pReader);
+// int32_t      tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
+// int32_t      tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
+// void         tsdbReleaseDataBlock(STsdbReader *pReader);
+// SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
+// int32_t      tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
+// int32_t      tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
+// int64_t      tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
+// void        *tsdbGetIdx(void *pMeta);
+// void        *tsdbGetIvtIdx(void *pMeta);
+// uint64_t     tsdbGetReaderMaxVersion(STsdbReader *pReader);
+// void         tsdbReaderSetCloseFlag(STsdbReader *pReader);
+// int64_t      tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr);
+
+// int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
+// int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
+// int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list);
+// void   *vnodeGetIdx(void *pVnode);
+// void   *vnodeGetIvtIdx(void *pVnode);
+
+// void        metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+// void        metaReaderReleaseLock(SMetaReader *pReader);
+// void        metaReaderClear(SMetaReader *pReader);
+// int32_t     metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
+// int32_t     metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
+// int         metaGetTableEntryByName(SMetaReader *pReader, const char *name);
+// int32_t     metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList);
+// int32_t     metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList);
+// const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
+// int         metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
+//
+// int      metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
+// int      metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
+// int      metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
+// bool     metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
+// int32_t  metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
+//                                   bool *acquired);
+// int32_t  metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+//                               int32_t payloadLen, double selectivityRatio);
+// int32_t  metaUidCacheClear(SMeta *pMeta, uint64_t suid);
+// tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
+// int32_t  metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
+// int32_t  metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
+//                               int32_t payloadLen);
+
+// tq
+typedef struct SMetaTableInfo {
+  int64_t         suid;
+  int64_t         uid;
+  SSchemaWrapper *schema;
+  char            tbName[TSDB_TABLE_NAME_LEN];
+} SMetaTableInfo;
+
+typedef struct SSnapContext {
+  SMeta *     pMeta;     // todo remove it
+  int64_t     snapVersion;
+  void *      pCur;
+  int64_t     suid;
+  int8_t      subType;
+  SHashObj *  idVersion;
+  SHashObj *  suidInfo;
+  SArray *    idList;
+  int32_t     index;
+  bool        withMeta;
+  bool        queryMeta;  // true-get meta, false-get data
+} SSnapContext;
+
+typedef struct {
+  int64_t uid;
+  int64_t ctbNum;
+} SMetaStbStats;
+
+// void    tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
+// int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
+// int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
+// int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
+// bool    tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid);
+// bool    tqCurrentBlockConsumed(const STqReader* pReader);
+// int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id);
+// bool    tqNextBlockInWal(STqReader* pReader, const char* idstr);
+// bool    tqNextBlockImpl(STqReader *pReader, const char* idstr);
+// int32_t        getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
+// SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx);
+// int32_t        setForSnapShot(SSnapContext *ctx, int64_t uid);
+// int32_t        destroySnapContext(SSnapContext *ctx);
+
+/*-------------------------------------------------new api format---------------------------------------------------*/
+
+//  typedef int32_t       (*__store_reader_(STsdbReader *pReader, const void *pTableList, int32_t num);
+//  typedef void          (*tsdbReaderSetId(STsdbReader *pReader, const char *idstr);
+//  typedef void          (*tsdbReaderClose(STsdbReader *pReader);
+//  typedef int32_t       (*tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
+//  typedef int32_t       (*tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
+//  typedef void          (*tsdbReleaseDataBlock(STsdbReader *pReader);
+//  typedef SSDataBlock * (*tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
+//  typedef int32_t       (*tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
+//  typedef int32_t       (*tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
+//  typedef int64_t       (*tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
+//  typedef void        * (*tsdbGetIdx(void *pMeta);
+//  typedef void        * (*tsdbGetIvtIdx(void *pMeta);
+//  typedef uint64_t      (*tsdbGetReaderMaxVersion(STsdbReader *pReader);
+//  typedef void          (*tsdbReaderSetCloseFlag(STsdbReader *pReader);
+//  typedef int64_t       (*tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr);
+
+typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *pCond, void *pTableList,
+                                            int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader,
+                                            const char *idstr, bool countOnly, SHashObj **pIgnoreTables);
+
+typedef struct TsdReader {
+  __store_reader_open_fn_t tsdReaderOpen;
+  void (*tsdReaderClose)();
+  void (*tsdSetReaderTaskId)(void *pReader, const char *pId);
+  int32_t (*tsdSetQueryTableList)();
+  int32_t (*tsdNextDataBlock)();
+
+  int32_t (*tsdReaderRetrieveBlockSMAInfo)();
+  SSDataBlock *(*tsdReaderRetrieveDataBlock)();
+
+  void (*tsdReaderReleaseDataBlock)();
+
+  int32_t (*tsdReaderResetStatus)();
+  int32_t (*tsdReaderGetDataBlockDistInfo)();
+  int64_t (*tsdReaderGetNumOfInMemRows)();
+  void (*tsdReaderNotifyClosing)();
+} TsdReader;
+
+
+/**
+ * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables);
+int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
+                                SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
+int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
+                              SArray *pTableUids);
+void   *tsdbCacherowsReaderClose(void *pReader);
+ */
+typedef struct SStoreCacheReader {
+  int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
+                        SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
+  void *(*closeReader)(void *pReader);
+  int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
+                          SArray *pTableUidList);
+  int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables);
+} SStoreCacheReader;
+
+/*------------------------------------------------------------------------------------------------------------------*/
+/*
+void    tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
+int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
+int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
+int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
+bool    tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid);
+bool    tqCurrentBlockConsumed(const STqReader* pReader);
+
+int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id);
+bool    tqNextBlockInWal(STqReader* pReader, const char* idstr);
+bool    tqNextBlockImpl(STqReader *pReader, const char* idstr);
+
+ int32_t    tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char* idstr);
+STqReader *tqReaderOpen(void *pVnode);
+void       tqReaderClose(STqReader *);
+
+int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
+bool    tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
+SWalReader* tqGetWalReader(STqReader* pReader);
+int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
+*/
+// todo rename
+typedef struct SStoreTqReader {
+  struct STqReader* (*tqReaderOpen)();
+  void (*tqReaderClose)();
+
+  int32_t (*tqReaderSeek)();
+  int32_t (*tqRetrieveBlock)();
+  bool (*tqReaderNextBlockInWal)();
+  bool (*tqNextBlockImpl)();  // todo remove it
+  SSDataBlock* (*tqGetResultBlock)();
+
+  void (*tqReaderSetColIdList)();
+  int32_t (*tqReaderSetQueryTableList)();
+
+  int32_t (*tqReaderAddTables)();
+  int32_t (*tqReaderRemoveTables)();
+
+  bool (*tqReaderIsQueriedTable)();
+  bool (*tqReaderCurrentBlockConsumed)();
+
+  struct SWalReader *(*tqReaderGetWalReader)();  // todo remove it
+  int32_t (*tqReaderRetrieveTaosXBlock)();          // todo remove it
+
+  int32_t (*tqReaderSetSubmitMsg)();  // todo remove it
+  bool (*tqReaderNextBlockFilterOut)();
+} SStoreTqReader;
+
+typedef struct SStoreSnapshotFn {
+  /*
+  int32_t        getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
+  SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx);
+  int32_t        setForSnapShot(SSnapContext *ctx, int64_t uid);
+  int32_t        destroySnapContext(SSnapContext *ctx);
+   */
+  int32_t (*createSnapshot)();
+  int32_t (*destroySnapshot)();
+  SMetaTableInfo (*getMetaTableInfoFromSnapshot)();
+  int32_t (*getTableInfoFromSnapshot)();
+} SStoreSnapshotFn;
+
+/**
+void        metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+void        metaReaderReleaseLock(SMetaReader *pReader);
+void        metaReaderClear(SMetaReader *pReader);
+int32_t     metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
+int32_t     metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
+int         metaGetTableEntryByName(SMetaReader *pReader, const char *name);
+int32_t     metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList);
+const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
+int         metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
+
+int      metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
+int      metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
+int      metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
+bool     metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
+int32_t  metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
+                                   bool *acquired);
+int32_t  metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+                               int32_t payloadLen, double selectivityRatio);
+int32_t  metaUidCacheClear(SMeta *pMeta, uint64_t suid);
+tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
+int32_t  metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
+int32_t  metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
+                               int32_t payloadLen);
+ */
+
+typedef struct SStoreMeta {
+  SMTbCursor *(*openTableMetaCursor)(void *pVnode);   // metaOpenTbCursor
+  void        (*closeTableMetaCursor)(SMTbCursor *pTbCur);  // metaCloseTbCursor
+  int32_t (*cursorNext)();                // metaTbCursorNext
+  int32_t (*cursorPrev)();                // metaTbCursorPrev
+
+  int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList);
+  int32_t (*getTableTagsByUid)(void *pVnode, int64_t suid, SArray *uidList);
+  const void *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal);  // todo remove it
+
+  int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid);
+  int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType);
+  int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName);
+  bool (*isTableExisted)(void *pVnode, tb_uid_t uid);
+
+  /**
+   * int32_t  metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+                               int32_t payloadLen, double selectivityRatio);
+int32_t  metaUidCacheClear(SMeta *pMeta, uint64_t suid);
+tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
+int32_t  metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
+int32_t  metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
+                               int32_t payloadLen);
+   */
+  void (*getCachedTableList)();
+  void (*putTableListIntoCache)();
+
+  /**
+   *
+   */
+  void *(*storeGetIndexInfo)();
+  void *(*getInvertIndex)(void* pVnode);
+  int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list);  // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
+  int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList);    // vnodeGetStbIdList  & vnodeGetAllTableList
+  void *storeGetVersionRange;
+  void *storeGetLastTimestamp;
+
+  int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);  // tsdbGetTableSchema
+
+  // db name, vgId, numOfTables, numOfSTables
+  int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables);  // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo);
+  void (*getBasicInfo)(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);// vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta);
+
+  int64_t (*getNumOfRowsInMem)(void* pVnode);
+  /**
+int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
+int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
+int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list);
+ */
+} SStoreMeta;
+
+typedef struct SStoreMetaReader {
+  void    (*initReader)(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI);
+  void    (*clearReader)(SMetaReader *pReader);
+  void    (*readerReleaseLock)(SMetaReader *pReader);
+  int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid);
+  int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name);
+  int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid);
+} SStoreMetaReader;
+
+typedef struct SUpdateInfo {
+  SArray      *pTsBuckets;
+  uint64_t     numBuckets;
+  SArray      *pTsSBFs;
+  uint64_t     numSBFs;
+  int64_t      interval;
+  int64_t      watermark;
+  TSKEY        minTS;
+  SScalableBf *pCloseWinSBF;
+  SHashObj    *pMap;
+  uint64_t     maxDataVersion;
+} SUpdateInfo;
+
+typedef struct {
+  void*   iter;      //  rocksdb_iterator_t*    iter;
+  void*   snapshot;  //  rocksdb_snapshot_t*    snapshot;
+  void*   readOpt;   //  rocksdb_readoptions_t* readOpt;
+  void*   db;        //  rocksdb_t*             db;
+  void*   pCur;
+  int64_t number;
+} SStreamStateCur;
+
+typedef struct SStateStore {
+  int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
+  int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal);
+
+  int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+  int32_t (*streamStateReleaseBuf)(SStreamState* pState, const SWinKey* key, void* pVal);
+  void    (*streamStateFreeVal)(void* val);
+
+  int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
+  int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+  bool    (*streamStateCheck)(SStreamState* pState, const SWinKey* key);
+  int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal);
+  int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key);
+  int32_t (*streamStateClear)(SStreamState* pState);
+  void    (*streamStateSetNumber)(SStreamState* pState, int32_t number);
+  int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
+  int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
+
+  int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
+  int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+  int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key);
+
+  int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur);
+  int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur);
+
+  SStreamStateCur*  (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key);
+  SStreamStateCur*  (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key);
+  SStreamStateCur*  (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key);
+  SStreamStateCur*  (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key);
+  void   (*streamStateFreeCur)(SStreamStateCur* pCur);
+
+  int32_t (*streamStateGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
+  int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
+
+  int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen);
+  int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen);
+  int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
+  int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key);
+  int32_t (*streamStateSessionClear)(SStreamState* pState);
+  int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
+  int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
+                                     state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
+  int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
+
+  SUpdateInfo*   (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark);
+  TSKEY   (*updateInfoFillBlockData)(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol);
+  bool    (*updateInfoIsUpdated)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts);
+  bool    (*updateInfoIsTableInserted)(SUpdateInfo *pInfo, int64_t tbUid);
+  void    (*updateInfoDestroy)(SUpdateInfo *pInfo);
+
+  SUpdateInfo* (*updateInfoInitP)(SInterval *pInterval, int64_t watermark);
+  void        (*updateInfoAddCloseWindowSBF)(SUpdateInfo *pInfo);
+  void        (*updateInfoDestoryColseWinSBF)(SUpdateInfo *pInfo);
+  int32_t     (*updateInfoSerialize)(void *buf, int32_t bufLen, const SUpdateInfo *pInfo);
+  int32_t     (*updateInfoDeserialize)(void *buf, int32_t bufLen, SUpdateInfo *pInfo);
+
+  SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);
+  SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key);
+  SStreamStateCur* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key);
+
+  struct SStreamFileState* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize,
+                                                  uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark);
+
+  void (*streamFileStateDestroy)(struct SStreamFileState* pFileState);
+  void (*streamFileStateClear)(struct SStreamFileState* pFileState);
+  bool (*needClearDiskBuff)(struct SStreamFileState* pFileState);
+
+  SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
+  void          (*streamStateClose)(SStreamState* pState, bool remove);
+  int32_t       (*streamStateBegin)(SStreamState* pState);
+  int32_t       (*streamStateCommit)(SStreamState* pState);
+  void          (*streamStateDestroy)(SStreamState* pState, bool remove);
+  int32_t       (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
+} SStateStore;
+
+typedef struct SStorageAPI {
+  SStoreMeta          metaFn;  // todo: refactor
+  TsdReader           tsdReader;
+  SStoreMetaReader    metaReaderFn;
+  SStoreCacheReader   cacheFn;
+  SStoreSnapshotFn    snapshotFn;
+  SStoreTqReader      tqReaderFn;
+  SStateStore         stateStore;
+  SMetaDataFilterAPI  metaFilter;
+  SFunctionStateStore functionStore;
+} SStorageAPI;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_STORAGEAPI_H

include/libs/function/function.h

@@ -22,6 +22,7 @@ extern "C" {
 
 #include "tcommon.h"
 #include "tvariant.h"
+#include "tsimplehash.h"
 
 struct SqlFunctionCtx;
 struct SResultRowEntryInfo;
@@ -127,18 +128,59 @@ typedef struct SSerializeDataHandle {
   void                 *pState;
 } SSerializeDataHandle;
 
+// incremental state storage
+typedef struct STdbState {
+  void*               rocksdb;
+  void**              pHandle;
+  void*               writeOpts;
+  void*               readOpts;
+  void**              cfOpts;
+  void*               dbOpt;
+  struct SStreamTask* pOwner;
+  void*               param;
+  void*               env;
+  SListNode*          pComparNode;
+  void*               pBackendHandle;
+  char                idstr[64];
+  void*               compactFactory;
+
+  void* db;
+  void* pStateDb;
+  void* pFuncStateDb;
+  void* pFillStateDb;  // todo refactor
+  void* pSessionStateDb;
+  void* pParNameDb;
+  void* pParTagDb;
+  void* txn;
+} STdbState;
+
+typedef struct {
+  STdbState*        pTdbState;
+  struct SStreamFileState* pFileState;
+  int32_t           number;
+  SSHashObj*        parNameMap;
+  int64_t           checkPointId;
+  int32_t           taskId;
+  int64_t           streamId;
+} SStreamState;
+
+typedef struct SFunctionStateStore {
+  int32_t (*streamStateFuncPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
+  int32_t (*streamStateFuncGet)(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
+} SFunctionStateStore;
+
 // sql function runtime context
 typedef struct SqlFunctionCtx {
   SInputColumnInfoData input;
   SResultDataInfo      resDataInfo;
-  uint32_t             order;       // data block scanner order: asc|desc
+  uint32_t             order;          // data block scanner order: asc|desc
-  uint8_t              isPseudoFunc;// denote current function is pseudo function or not [added for perf reason]
+  uint8_t              isPseudoFunc;   // denote current function is pseudo function or not [added for perf reason]
-  uint8_t              isNotNullFunc;// not return null value.
+  uint8_t              isNotNullFunc;  // not return null value.
-  uint8_t              scanFlag;    // record current running step, default: 0
+  uint8_t              scanFlag;       // record current running step, default: 0
-  int16_t              functionId;  // function id
+  int16_t              functionId;     // function id
-  char                *pOutput;     // final result output buffer, point to sdata->data
+  char                *pOutput;        // final result output buffer, point to sdata->data
   // input parameter, e.g., top(k, 20), the number of results of top query is kept in param
-  SFunctParam *param;
+  SFunctParam         *param;
   // corresponding output buffer for timestamp of each result, e.g., diff/csum
   SColumnInfoData     *pTsOutput;
   int32_t              numOfParams;
@@ -155,6 +197,7 @@ typedef struct SqlFunctionCtx {
   SSerializeDataHandle saveHandle;
   int32_t              exprIdx;
   char                *udfName;
+  SFunctionStateStore *pStore;
 } SqlFunctionCtx;
 
 typedef struct tExprNode {

include/libs/index/index.h

@@ -212,13 +212,38 @@ typedef struct SIndexMetaArg {
   void*    idx;
   void*    ivtIdx;
   uint64_t suid;
+  int (*metaFilterFunc)(void* metaEx, void* param, SArray* result);
 } SIndexMetaArg;
 
+/**
+ * the underlying storage module must implement this API to employ the index functions.
+ * @param pMeta
+ * @param param
+ * @param results
+ * @return
+ */
+typedef struct SMetaFltParam {
+  uint64_t suid;
+  int16_t  cid;
+  int16_t  type;
+  void    *val;
+  bool     reverse;
+  bool     equal;
+  int (*filterFunc)(void *a, void *b, int16_t type);
+} SMetaFltParam;
+
+typedef struct SMetaDataFilterAPI {
+  int32_t (*metaFilterTableIds)(void *pVnode, SMetaFltParam *arg, SArray *pUids);
+  int32_t (*metaFilterCreateTime)(void *pVnode, SMetaFltParam *arg, SArray *pUids);
+  int32_t (*metaFilterTableName)(void *pVnode, SMetaFltParam *arg, SArray *pUids);
+  int32_t (*metaFilterTtl)(void *pVnode, SMetaFltParam *arg, SArray *pUids);
+} SMetaDataFilterAPI;
+
 typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus;
 
-SIdxFltStatus idxGetFltStatus(SNode* pFilterNode);
+SIdxFltStatus idxGetFltStatus(SNode* pFilterNode, SMetaDataFilterAPI* pAPI);
 
-int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status);
+int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status, SMetaDataFilterAPI* pAPI);
 
 /*
  *  init index env

include/libs/stream/streamState.h

@@ -27,66 +27,63 @@
 extern "C" {
 #endif
 
+#include "storageapi.h"
+
 // void*      streamBackendInit(const char* path);
 // void       streamBackendCleanup(void* arg);
 // SListNode* streamBackendAddCompare(void* backend, void* arg);
 // void       streamBackendDelCompare(void* backend, void* arg);
-typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
 
-typedef struct STdbState {
+// <<<<<<< HEAD
-  rocksdb_t*                       rocksdb;
+// typedef struct STdbState {
-  rocksdb_column_family_handle_t** pHandle;
+//   rocksdb_t*                       rocksdb;
-  rocksdb_writeoptions_t*          writeOpts;
+//   rocksdb_column_family_handle_t** pHandle;
-  rocksdb_readoptions_t*           readOpts;
+//   rocksdb_writeoptions_t*          writeOpts;
-  rocksdb_options_t**              cfOpts;
+//   rocksdb_readoptions_t*           readOpts;
-  rocksdb_options_t*               dbOpt;
+//   rocksdb_options_t**              cfOpts;
-  struct SStreamTask*              pOwner;
+//   rocksdb_options_t*               dbOpt;
-  void*                            param;
+//   struct SStreamTask*              pOwner;
-  void*                            env;
+//   void*                            param;
-  SListNode*                       pComparNode;
+//   void*                            env;
-  void*                            pBackend;
+//   SListNode*                       pComparNode;
-  char                             idstr[64];
+//   void*                            pBackend;
-  void*                            compactFactory;
+//   char                             idstr[64];
-  TdThreadRwlock                   rwLock;
+//   void*                            compactFactory;
+//   TdThreadRwlock                   rwLock;
+// =======
+// typedef struct STdbState {
+//  rocksdb_t*                       rocksdb;
+//  rocksdb_column_family_handle_t** pHandle;
+//  rocksdb_writeoptions_t*          writeOpts;
+//  rocksdb_readoptions_t*           readOpts;
+//  rocksdb_options_t**              cfOpts;
+//  rocksdb_options_t*               dbOpt;
+//  struct SStreamTask*              pOwner;
+//  void*                            param;
+//  void*                            env;
+//  SListNode*                       pComparNode;
+//  void*                            pBackendHandle;
+//  char                             idstr[64];
+//  void*                            compactFactory;
+//
+//  TDB* db;
+//  TTB* pStateDb;
+//  TTB* pFuncStateDb;
+//  TTB* pFillStateDb;  // todo refactor
+//  TTB* pSessionStateDb;
+//  TTB* pParNameDb;
+//  TTB* pParTagDb;
+//  TXN* txn;
+//} STdbState;
+//>>>>>>> enh/dev3.0
 
-  TDB* db;
+SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
-  TTB* pStateDb;
-  TTB* pFuncStateDb;
-  TTB* pFillStateDb;  // todo refactor
-  TTB* pSessionStateDb;
-  TTB* pParNameDb;
-  TTB* pParTagDb;
-  TXN* txn;
-} STdbState;
-
-// incremental state storage
-typedef struct {
-  STdbState*        pTdbState;
-  SStreamFileState* pFileState;
-  int32_t           number;
-  SSHashObj*        parNameMap;
-  int64_t           checkPointId;
-  int32_t           taskId;
-  int64_t           streamId;
-} SStreamState;
-
-SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages);
 void          streamStateClose(SStreamState* pState, bool remove);
 int32_t       streamStateBegin(SStreamState* pState);
 int32_t       streamStateCommit(SStreamState* pState);
 void          streamStateDestroy(SStreamState* pState, bool remove);
 int32_t       streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
 
-typedef struct {
-  rocksdb_iterator_t*    iter;
-  rocksdb_snapshot_t*    snapshot;
-  rocksdb_readoptions_t* readOpt;
-  rocksdb_t*             db;
-
-  TBC*    pCur;
-  int64_t number;
-} SStreamStateCur;
-
 int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
 int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
 
@@ -120,7 +117,7 @@ int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key);
 
 int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
 int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal);
-void    streamFreeVal(void* val);
+void    streamStateFreeVal(void* val);
 
 SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
 SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key);

include/libs/stream/tstreamFileState.h

@@ -21,23 +21,16 @@
 #include "tarray.h"
 #include "tdef.h"
 #include "tlist.h"
+#include "storageapi.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 typedef struct SStreamFileState SStreamFileState;
-typedef struct SRowBuffPos {
-  void* pRowBuff;
-  void* pKey;
-  bool  beFlushed;
-  bool  beUsed;
-} SRowBuffPos;
 
 typedef SList SStreamSnapshot;
 
-typedef TSKEY (*GetTsFun)(void*);
-
 SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
                                       GetTsFun fp, void* pFile, TSKEY delMark);
 void              streamFileStateDestroy(SStreamFileState* pFileState);

include/libs/stream/tstreamUpdate.h

@@ -19,7 +19,7 @@
 #include "tarray.h"
 #include "tcommon.h"
 #include "tmsg.h"
-#include "tscalablebf.h"
+#include "storageapi.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -30,18 +30,18 @@ typedef struct SUpdateKey {
   TSKEY   ts;
 } SUpdateKey;
 
-typedef struct SUpdateInfo {
+//typedef struct SUpdateInfo {
-  SArray      *pTsBuckets;
+//  SArray      *pTsBuckets;
-  uint64_t     numBuckets;
+//  uint64_t     numBuckets;
-  SArray      *pTsSBFs;
+//  SArray      *pTsSBFs;
-  uint64_t     numSBFs;
+//  uint64_t     numSBFs;
-  int64_t      interval;
+//  int64_t      interval;
-  int64_t      watermark;
+//  int64_t      watermark;
-  TSKEY        minTS;
+//  TSKEY        minTS;
-  SScalableBf *pCloseWinSBF;
+//  SScalableBf *pCloseWinSBF;
-  SHashObj    *pMap;
+//  SHashObj    *pMap;
-  uint64_t     maxDataVersion;
+//  uint64_t     maxDataVersion;
-} SUpdateInfo;
+//} SUpdateInfo;
 
 SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark);
 SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);

include/util/taoserror.h

@@ -409,6 +409,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MNODE_ALREADY_IS_VOTER        TAOS_DEF_ERROR_CODE(0, 0x0413) // internal
 #define TSDB_CODE_MNODE_ONLY_TWO_MNODE          TAOS_DEF_ERROR_CODE(0, 0x0414) // internal
 #define TSDB_CODE_MNODE_NO_NEED_RESTORE         TAOS_DEF_ERROR_CODE(0, 0x0415) // internal
+#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE   TAOS_DEF_ERROR_CODE(0, 0x0416)
 
 // vnode
 // #define TSDB_CODE_VND_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x

source/client/inc/clientSml.h

@@ -232,9 +232,9 @@ int           smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *
 int           smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
 //SArray       *smlJsonParseTags(char *start, char *end);
 bool          smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
-void*         nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
+//void*         nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
-int           nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
+//int           nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
-int           nodeListSize(NodeList* list);
+//int           nodeListSize(NodeList* list);
 bool          smlDoubleToInt64OverFlow(double num);
 int32_t       smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
 bool          smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
@@ -251,7 +251,7 @@ int32_t           smlClearForRerun(SSmlHandle *info);
 int32_t           smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg);
 uint8_t           smlGetTimestampLen(int64_t num);
 void              clearColValArray(SArray* pCols);
-void              smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag);
+void              smlDestroyTableInfo(void *para);
 
 void freeSSmlKv(void* data);
 int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);

source/client/src/clientImpl.c

@@ -1757,6 +1757,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
     return TSDB_CODE_TSC_INTERNAL_ERROR;
   }
 
+  taosMemoryFreeClear(pResultInfo->convertJson);
   pResultInfo->convertJson = taosMemoryCalloc(1, dataLen);
   if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
   char* p1 = pResultInfo->convertJson;

source/client/src/clientSml.c

@@ -24,72 +24,91 @@ int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL};
 int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
 int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL};
 
-void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
+//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
-  NodeList *tmp = list;
+//  NodeList *tmp = list;
-  while (tmp) {
+//  while (tmp) {
-    if (fn == NULL) {
+//    if (fn == NULL) {
-      if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
+//      if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-        return tmp->data.value;
+//        return tmp->data.value;
-      }
+//      }
-    } else {
+//    } else {
-      if (tmp->data.used && fn(tmp->data.key, key) == 0) {
+//      if (tmp->data.used && fn(tmp->data.key, key) == 0) {
-        return tmp->data.value;
+//        return tmp->data.value;
-      }
+//      }
-    }
+//    }
+//
+//    tmp = tmp->next;
+//  }
+//  return NULL;
+//}
+//
+//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
+//  NodeList *tmp = *list;
+//  while (tmp) {
+//    if (!tmp->data.used) break;
+//    if (fn == NULL) {
+//      if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
+//        return -1;
+//      }
+//    } else {
+//      if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
+//        return -1;
+//      }
+//    }
+//
+//    tmp = tmp->next;
+//  }
+//  if (tmp) {
+//    tmp->data.key = key;
+//    tmp->data.keyLen = len;
+//    tmp->data.value = value;
+//    tmp->data.used = true;
+//  } else {
+//    NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
+//    if (newNode == NULL) {
+//      return -1;
+//    }
+//    newNode->data.key = key;
+//    newNode->data.keyLen = len;
+//    newNode->data.value = value;
+//    newNode->data.used = true;
+//    newNode->next = *list;
+//    *list = newNode;
+//  }
+//  return 0;
+//}
+//
+//int nodeListSize(NodeList *list) {
+//  int cnt = 0;
+//  while (list) {
+//    if (list->data.used)
+//      cnt++;
+//    else
+//      break;
+//    list = list->next;
+//  }
+//  return cnt;
+//}
 
-    tmp = tmp->next;
+static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){
-  }
+  SUserAuthInfo pAuth = {0};
-  return NULL;
+  snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user);
-}
+  if (NULL == pTabName) {
-
+    tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb));
-int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
-  NodeList *tmp = *list;
-  while (tmp) {
-    if (!tmp->data.used) break;
-    if (fn == NULL) {
-      if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-        return -1;
-      }
-    } else {
-      if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
-        return -1;
-      }
-    }
-
-    tmp = tmp->next;
-  }
-  if (tmp) {
-    tmp->data.key = key;
-    tmp->data.keyLen = len;
-    tmp->data.value = value;
-    tmp->data.used = true;
   } else {
-    NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
+    toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName);
-    if (newNode == NULL) {
-      return -1;
-    }
-    newNode->data.key = key;
-    newNode->data.keyLen = len;
-    newNode->data.value = value;
-    newNode->data.used = true;
-    newNode->next = *list;
-    *list = newNode;
   }
-  return 0;
+  pAuth.type = type;
-}
 
-int nodeListSize(NodeList *list) {
+  int32_t      code = TSDB_CODE_SUCCESS;
-  int cnt = 0;
+  SUserAuthRes authRes = {0};
-  while (list) {
-    if (list->data.used)
-      cnt++;
-    else
-      break;
-    list = list->next;
-  }
-  return cnt;
-}
 
+  code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes);
+
+
+  return (code == TSDB_CODE_SUCCESS) ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
+
+}
 inline bool smlDoubleToInt64OverFlow(double num) {
   if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
   return false;
@@ -211,6 +230,16 @@ void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tin
   }
 }
 
+static void smlDestroySTableMeta(void *para) {
+  SSmlSTableMeta *meta = *(SSmlSTableMeta**)para;
+  taosHashCleanup(meta->tagHash);
+  taosHashCleanup(meta->colHash);
+  taosArrayDestroy(meta->tags);
+  taosArrayDestroy(meta->cols);
+  taosMemoryFreeClear(meta->tableMeta);
+  taosMemoryFree(meta);
+}
+
 SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) {
   SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1);
   if (!meta) {
@@ -245,7 +274,7 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) {
   return meta;
 
 cleanup:
-  taosMemoryFree(meta);
+  smlDestroySTableMeta(meta);
   return NULL;
 }
 
@@ -813,6 +842,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
     code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
 
     if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) {
+      code = smlCheckAuth(info, &conn, NULL, AUTH_TYPE_WRITE);
+      if(code != TSDB_CODE_SUCCESS){
+        goto end;
+      }
       uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname);
       SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField));
       SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField));
@@ -857,6 +890,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
         goto end;
       }
       if (action != SCHEMA_ACTION_NULL) {
+        code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
+        if(code != TSDB_CODE_SUCCESS){
+          goto end;
+        }
         uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname,
                action);
         SArray *pColumns =
@@ -927,6 +964,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
         goto end;
       }
       if (action != SCHEMA_ACTION_NULL) {
+        code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
+        if(code != TSDB_CODE_SUCCESS){
+          goto end;
+        }
         uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname,
                action);
         SArray *pColumns =
@@ -1004,6 +1045,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
       }
     }
 
+    taosMemoryFreeClear(sTableData->tableMeta);
     sTableData->tableMeta = pTableMeta;
     uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid,
            pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
@@ -1062,15 +1104,6 @@ static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) {
   }
 }
 
-static void smlDestroySTableMeta(SSmlSTableMeta *meta) {
-  taosHashCleanup(meta->tagHash);
-  taosHashCleanup(meta->colHash);
-  taosArrayDestroy(meta->tags);
-  taosArrayDestroy(meta->cols);
-  taosMemoryFree(meta->tableMeta);
-  taosMemoryFree(meta);
-}
-
 static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) {
   for (int i = 0; i < taosArrayGetSize(cols); ++i) {
     SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
@@ -1110,7 +1143,8 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols
   return TSDB_CODE_SUCCESS;
 }
 
-void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
+void smlDestroyTableInfo(void *para) {
+  SSmlTableInfo *tag = *(SSmlTableInfo**)para;
   for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) {
     SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i);
     taosHashCleanup(kvHash);
@@ -1147,18 +1181,18 @@ void smlDestroyInfo(SSmlHandle *info) {
   qDestroyQuery(info->pQuery);
 
   // destroy info->childTables
-  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
+//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-  while (oneTable) {
+//  while (oneTable) {
-    smlDestroyTableInfo(info, *oneTable);
+//    smlDestroyTableInfo(oneTable);
-    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
+//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-  }
+//  }
 
   // destroy info->superTables
-  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
+//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-  while (oneSTable) {
+//  while (oneSTable) {
-    smlDestroySTableMeta(*oneSTable);
+//    smlDestroySTableMeta(*oneSTable);
-    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
+//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-  }
+//  }
 
   // destroy info->pVgHash
   taosHashCleanup(info->pVgHash);
@@ -1217,6 +1251,8 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
   info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
   info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
   info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  taosHashSetFreeFp(info->superTables, smlDestroySTableMeta);
+  taosHashSetFreeFp(info->childTables, smlDestroyTableInfo);
 
   info->id = smlGenId();
   info->pQuery = smlInitHandle();
@@ -1323,6 +1359,9 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
       uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
              info->lineNum);
       SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
+      if(meta == NULL){
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
       taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
       terrno = 0;
       smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
@@ -1367,6 +1406,11 @@ static int32_t smlInsertData(SSmlHandle *info) {
     conn.requestObjRefId = info->pRequest->self;
     conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
 
+    code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
+    if(code != TSDB_CODE_SUCCESS){
+      return code;
+    }
+
     SVgroupInfo vg;
     code = catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg);
     if (code != TSDB_CODE_SUCCESS) {
@@ -1437,18 +1481,18 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
 int32_t smlClearForRerun(SSmlHandle *info) {
   info->reRun = false;
   // clear info->childTables
-  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
+//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-  while (oneTable) {
+//  while (oneTable) {
-    smlDestroyTableInfo(info, *oneTable);
+//    smlDestroyTableInfo(info, *oneTable);
-    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
+//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-  }
+//  }
 
   // clear info->superTables
-  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
+//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-  while (oneSTable) {
+//  while (oneSTable) {
-    smlDestroySTableMeta(*oneSTable);
+//    smlDestroySTableMeta(*oneSTable);
-    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
+//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-  }
+//  }
 
   taosHashClear(info->childTables);
   taosHashClear(info->superTables);
@@ -1586,9 +1630,7 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
 
   do {
     code = smlModifyDBSchemas(info);
-    if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA || code == TSDB_CODE_PAR_TOO_MANY_COLUMNS
+    if (code != TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER && code != TSDB_CODE_SDB_OBJ_CREATING && code != TSDB_CODE_MND_TRANS_CONFLICT) {
-        || code == TSDB_CODE_PAR_INVALID_TAGS_NUM || code == TSDB_CODE_PAR_INVALID_TAGS_LENGTH
-        || code == TSDB_CODE_PAR_INVALID_ROW_LENGTH || code == TSDB_CODE_MND_FIELD_VALUE_OVERFLOW) {
       break;
     }
     taosMsleep(100);

source/client/src/clientSmlJson.c

@@ -695,6 +695,10 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
           return TSDB_CODE_SUCCESS;
         }
         sMeta = smlBuildSTableMeta(info->dataFormat);
+        if(sMeta == NULL){
+          taosMemoryFreeClear(pTableMeta);
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
         sMeta->tableMeta = pTableMeta;
         taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
         for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
@@ -784,7 +788,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
       tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
       if (tinfo->tableDataCtx == NULL) {
         smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(info, tinfo);
+        smlDestroyTableInfo(&tinfo);
         return TSDB_CODE_SML_INVALID_DATA;
       }
     }
@@ -1048,12 +1052,18 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
     return TSDB_CODE_TSC_INVALID_JSON;
   }
 
-  info->lineNum = payloadNum;
+
-  info->dataFormat = true;
   if (unlikely(info->lines != NULL)) {
+    for (int i = 0; i < info->lineNum; i++) {
+      taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
+      if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
+    }
     taosMemoryFree(info->lines);
     info->lines = NULL;
   }
+  info->lineNum = payloadNum;
+  info->dataFormat = true;
+
   ret = smlClearForRerun(info);
   if (ret != TSDB_CODE_SUCCESS) {
     return ret;

source/client/src/clientSmlLine.c

@@ -168,6 +168,10 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
           return TSDB_CODE_SUCCESS;
         }
         sMeta = smlBuildSTableMeta(info->dataFormat);
+        if(sMeta == NULL){
+          taosMemoryFreeClear(pTableMeta);
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
         sMeta->tableMeta = pTableMeta;
         taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
         for (int i = pTableMeta->tableInfo.numOfColumns;
@@ -326,7 +330,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     info->currSTableMeta->uid = tinfo->uid;
     tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
     if (tinfo->tableDataCtx == NULL) {
-      smlDestroyTableInfo(info, tinfo);
+      smlDestroyTableInfo(&tinfo);
       smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
       return TSDB_CODE_SML_INVALID_DATA;
     }
@@ -372,6 +376,10 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
           return TSDB_CODE_SUCCESS;
         }
         *tmp = smlBuildSTableMeta(info->dataFormat);
+        if(*tmp == NULL){
+          taosMemoryFreeClear(pTableMeta);
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
         (*tmp)->tableMeta = pTableMeta;
         taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES);
 

source/client/src/clientSmlTelnet.c

@@ -91,6 +91,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
           return TSDB_CODE_SUCCESS;
         }
         sMeta = smlBuildSTableMeta(info->dataFormat);
+        if(sMeta == NULL){
+          taosMemoryFreeClear(pTableMeta);
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
         sMeta->tableMeta = pTableMeta;
         taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
         for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
@@ -212,7 +216,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
       tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
       if (tinfo->tableDataCtx == NULL) {
         smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(info, tinfo);
+        smlDestroyTableInfo(&tinfo);
         return TSDB_CODE_SML_INVALID_DATA;
       }
     }

source/client/src/clientTmq.c

@@ -27,6 +27,8 @@
 #define EMPTY_BLOCK_POLL_IDLE_DURATION 10
 #define DEFAULT_AUTO_COMMIT_INTERVAL   5000
 
+#define OFFSET_IS_RESET_OFFSET(_of)  ((_of) < 0)
+
 typedef void (*__tmq_askep_fn_t)(tmq_t* pTmq, int32_t code, SDataBuf* pBuf, void* pParam);
 
 struct SMqMgmt {
@@ -2626,12 +2628,12 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
   SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo;
 
   int32_t type = pOffsetInfo->currentOffset.type;
-  if (type != TMQ_OFFSET__LOG) {
+  if (type != TMQ_OFFSET__LOG && !OFFSET_IS_RESET_OFFSET(type)) {
     tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type);
     return TSDB_CODE_INVALID_PARA;
   }
 
-  if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) {
+  if (type == TMQ_OFFSET__LOG && (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd)) {
     tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]",
              tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd);
     return TSDB_CODE_INVALID_PARA;

source/client/test/clientTests.cpp

@@ -1100,7 +1100,7 @@ TEST(clientCase, sub_tb_test) {
 
   // 创建订阅 topics 列表
   tmq_list_t* topicList = tmq_list_new();
-  tmq_list_append(topicList, "topic_t1");
+  tmq_list_append(topicList, "t1");
 
   // 启动订阅
   tmq_subscribe(tmq, topicList);
@@ -1118,7 +1118,7 @@ TEST(clientCase, sub_tb_test) {
   tmq_topic_assignment* pAssign = NULL;
   int32_t numOfAssign = 0;
 
-  int32_t code = tmq_get_topic_assignment(tmq, "topic_t1", &pAssign, &numOfAssign);
+  int32_t code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign);
   if (code != 0) {
     printf("error occurs:%s\n", tmq_err2str(code));
     tmq_consumer_close(tmq);
@@ -1127,7 +1127,16 @@ TEST(clientCase, sub_tb_test) {
     return;
   }
 
-  tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgId, 0);
+  tmq_offset_seek(tmq, "t1", pAssign[0].vgId, 4);
+
+  code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign);
+  if (code != 0) {
+    printf("error occurs:%s\n", tmq_err2str(code));
+    tmq_consumer_close(tmq);
+    taos_close(pConn);
+    fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
+    return;
+  }
 
   while (1) {
     TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);

source/dnode/mnode/impl/src/mndDnode.c

@@ -986,8 +986,15 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
   }
 
   int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id);
-  if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !force) {
+  if (numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) {
-    if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
+    bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs());
+    if (isonline && force) {
+      terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE;
+      mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(),
+             numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
+      goto _OVER;
+    }
+    if (!isonline && !force) {
       terrno = TSDB_CODE_DNODE_OFFLINE;
       mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(),
              numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);

source/dnode/mnode/impl/src/mndMnode.c

@@ -695,7 +695,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
   if (totalMnodes == 2) {
     if (force) {
       mError("cant't force drop dnode, since a mnode on it and replica is 2");
-      terrno = TSDB_CODE_DNODE_OFFLINE;
+      terrno = TSDB_CODE_MNODE_ONLY_TWO_MNODE;
       return -1;
     }
     mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);

source/dnode/mnode/impl/src/mndQuery.c

@@ -33,6 +33,7 @@ void mndPostProcessQueryMsg(SRpcMsg *pMsg) {
 int32_t mndProcessQueryMsg(SRpcMsg *pMsg) {
   int32_t     code = -1;
   SMnode     *pMnode = pMsg->info.node;
+
   SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb};
 
   mTrace("msg:%p, in query queue is processing", pMsg);

source/dnode/qnode/CMakeLists.txt

@@ -14,4 +14,7 @@ target_link_libraries(
     PRIVATE qworker
     PRIVATE qcom
     PRIVATE executor
+    PRIVATE tdb
+    PRIVATE wal
+    PRIVATE index
 )

source/dnode/qnode/src/qnode.c

@@ -14,7 +14,6 @@
  */
 
 #include "executor.h"
-#include "libs/function/function.h"
 #include "qndInt.h"
 #include "query.h"
 #include "qworker.h"

source/dnode/snode/CMakeLists.txt

@@ -14,4 +14,6 @@ target_link_libraries(
     PRIVATE util
     PRIVATE qcom
     PRIVATE stream
+    PRIVATE wal
+    PRIVATE index
 )

source/dnode/snode/inc/sndInt.h

@@ -53,6 +53,8 @@ int32_t sndStopTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
 int32_t sndResumeTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
 #endif
 
+void initStreamStateAPI(SStorageAPI* pAPI);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/snode/src/snode.c

@@ -62,8 +62,7 @@ FAIL:
 }
 
 int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
-  ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
+  ASSERT(pTask->taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->childEpInfo) != 0);
-  ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0);
 
   pTask->refCnt = 1;
   pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
@@ -87,9 +86,10 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
   }
 
   int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo);
-  SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState };
+  SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState };
+  initStreamStateAPI(&handle.api);
 
-  pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0);
+  pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0);
   ASSERT(pTask->exec.pExecutor);
 
   streamSetupTrigger(pTask);

source/dnode/snode/src/snodeInitApi.c

@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "storageapi.h"
+#include "tstreamUpdate.h"
+#include "streamState.h"
+
+static void initStateStoreAPI(SStateStore* pStore);
+static void initFunctionStateStore(SFunctionStateStore* pStore);
+
+void initStreamStateAPI(SStorageAPI* pAPI) {
+  initStateStoreAPI(&pAPI->stateStore);
+  initFunctionStateStore(&pAPI->functionStore);
+}
+
+void initStateStoreAPI(SStateStore* pStore) {
+  pStore->streamFileStateInit = streamFileStateInit;
+  pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF;
+
+  pStore->streamStateGetByPos = streamStateGetByPos;
+
+  pStore->streamStatePutParName = streamStatePutParName;
+  pStore->streamStateGetParName = streamStateGetParName;
+
+  pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
+  pStore->streamStateReleaseBuf = streamStateReleaseBuf;
+  pStore->streamStateFreeVal = streamStateFreeVal;
+
+  pStore->streamStatePut = streamStatePut;
+  pStore->streamStateGet = streamStateGet;
+  pStore->streamStateCheck = streamStateCheck;
+  pStore->streamStateGetByPos = streamStateGetByPos;
+  pStore->streamStateDel = streamStateDel;
+  pStore->streamStateClear = streamStateClear;
+  pStore->streamStateSaveInfo = streamStateSaveInfo;
+  pStore->streamStateGetInfo = streamStateGetInfo;
+  pStore->streamStateSetNumber = streamStateSetNumber;
+
+  pStore->streamStateFillPut = streamStateFillPut;
+  pStore->streamStateFillGet = streamStateFillGet;
+  pStore->streamStateFillDel = streamStateFillDel;
+
+  pStore->streamStateCurNext = streamStateCurNext;
+  pStore->streamStateCurPrev = streamStateCurPrev;
+
+  pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur;
+  pStore->streamStateSeekKeyNext = streamStateSeekKeyNext;
+  pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext;
+  pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev;
+  pStore->streamStateFreeCur = streamStateFreeCur;
+
+  pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur;
+  pStore->streamStateGetKVByCur = streamStateGetKVByCur;
+
+  pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
+  pStore->streamStateSessionPut = streamStateSessionPut;
+  pStore->streamStateSessionGet = streamStateSessionGet;
+  pStore->streamStateSessionDel = streamStateSessionDel;
+  pStore->streamStateSessionClear = streamStateSessionClear;
+  pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur;
+  pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist;
+  pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
+
+  pStore->updateInfoInit = updateInfoInit;
+  pStore->updateInfoFillBlockData = updateInfoFillBlockData;
+  pStore->updateInfoIsUpdated = updateInfoIsUpdated;
+  pStore->updateInfoIsTableInserted = updateInfoIsTableInserted;
+  pStore->updateInfoDestroy = updateInfoDestroy;
+
+  pStore->updateInfoInitP = updateInfoInitP;
+  pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF;
+  pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF;
+  pStore->updateInfoSerialize = updateInfoSerialize;
+  pStore->updateInfoDeserialize = updateInfoDeserialize;
+
+  pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
+  pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
+  pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext;
+
+  pStore->streamFileStateInit = streamFileStateInit;
+
+  pStore->streamFileStateDestroy = streamFileStateDestroy;
+  pStore->streamFileStateClear = streamFileStateClear;
+  pStore->needClearDiskBuff = needClearDiskBuff;
+
+  pStore->streamStateOpen = streamStateOpen;
+  pStore->streamStateClose = streamStateClose;
+  pStore->streamStateBegin = streamStateBegin;
+  pStore->streamStateCommit = streamStateCommit;
+  pStore->streamStateDestroy= streamStateDestroy;
+  pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
+}
+
+void initFunctionStateStore(SFunctionStateStore* pStore) {
+  pStore->streamStateFuncPut = streamStateFuncPut;
+  pStore->streamStateFuncGet = streamStateFuncGet;
+}

source/dnode/vnode/CMakeLists.txt

@@ -15,6 +15,7 @@ target_sources(
     "src/vnd/vnodeSync.c"
     "src/vnd/vnodeSnapshot.c"
     "src/vnd/vnodeRetention.c"
+    "src/vnd/vnodeInitApi.c"
 
     # meta
     "src/meta/metaOpen.c"

source/dnode/vnode/inc/vnode.h

@@ -26,6 +26,7 @@
 #include "tfs.h"
 #include "wal.h"
 
+#include "filter.h"
 #include "tcommon.h"
 #include "tfs.h"
 #include "tgrant.h"
@@ -33,6 +34,7 @@
 #include "trow.h"
 
 #include "tdb.h"
+#include "storageapi.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -65,17 +67,18 @@ int32_t   vnodeStart(SVnode *pVnode);
 void      vnodeStop(SVnode *pVnode);
 int64_t   vnodeGetSyncHandle(SVnode *pVnode);
 void      vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
-void      vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
+void      vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);
 int32_t   vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
+int32_t   vnodeGetTableList(void* pVnode, int8_t type, SArray* pList);
 int32_t   vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
 int32_t   vnodeIsCatchUp(SVnode *pVnode);
 ESyncRole vnodeGetRole(SVnode *pVnode);
 
-int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list);
+int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
 int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
 int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list);
-void   *vnodeGetIdx(SVnode *pVnode);
+void   *vnodeGetIdx(void *pVnode);
-void   *vnodeGetIvtIdx(SVnode *pVnode);
+void   *vnodeGetIvtIdx(void *pVnode);
 
 int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num);
 int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num);
@@ -103,57 +106,40 @@ typedef struct SMetaEntry  SMetaEntry;
 
 #define META_READER_NOLOCK 0x1
 
-void        metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+void        _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI);
 void        metaReaderReleaseLock(SMetaReader *pReader);
 void        metaReaderClear(SMetaReader *pReader);
-int32_t     metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
+int32_t     metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
-int32_t     metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
+int32_t     metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
 int         metaGetTableEntryByName(SMetaReader *pReader, const char *name);
-int32_t     metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList);
+int32_t     metaGetTableTags(void *pVnode, uint64_t suid, SArray *uidList);
-int32_t     metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList);
+int32_t     metaGetTableTagsByUids(void* pVnode, int64_t suid, SArray *uidList);
 int32_t     metaReadNext(SMetaReader *pReader);
-const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
+const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal);
 int         metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
 
 int      metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
-int      metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
+int      metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid);
 int      metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
-bool     metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
+bool     metaIsTableExist(void* pVnode, tb_uid_t uid);
 int32_t  metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
                                    bool *acquired);
 int32_t  metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
                                int32_t payloadLen, double selectivityRatio);
 int32_t  metaUidCacheClear(SMeta *pMeta, uint64_t suid);
 tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
-int32_t  metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid);
+int32_t  metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid);
-int32_t  metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
+int32_t  metaGetCachedTbGroup(SMeta *pMeta, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
-int32_t  metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
+int32_t  metaPutTbGroupToCache(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
-                              int32_t payloadLen);
+                               int32_t payloadLen);
 
 int64_t  metaGetTbNum(SMeta *pMeta);
 int64_t  metaGetNtbNum(SMeta *pMeta);
-typedef struct {
+//typedef struct {
-  int64_t uid;
+//  int64_t uid;
-  int64_t ctbNum;
+//  int64_t ctbNum;
-} SMetaStbStats;
+//} SMetaStbStats;
-int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo);
+int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables);
-
-typedef struct SMetaFltParam {
-  tb_uid_t suid;
-  int16_t  cid;
-  int16_t  type;
-  void    *val;
-  bool     reverse;
-  bool     equal;
-  int (*filterFunc)(void *a, void *b, int16_t type);
-
-} SMetaFltParam;
-
-// TODO, refactor later
-int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results);
-int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids);
-int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids);
-int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids);
 
 #if 1  // refact APIs below (TODO)
 typedef SVCreateTbReq   STbCfg;
@@ -161,7 +147,7 @@ typedef SVCreateTSmaReq SSmaCfg;
 
 typedef struct SMTbCursor SMTbCursor;
 
-SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
+SMTbCursor *metaOpenTbCursor(void *pVnode);
 void        metaCloseTbCursor(SMTbCursor *pTbCur);
 int32_t     metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType);
 int32_t     metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType);
@@ -183,7 +169,8 @@ typedef struct STsdbReader STsdbReader;
 #define CACHESCAN_RETRIEVE_LAST        0x8
 
 int32_t      tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
-                            SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables);
+                            SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly,
+                            SHashObj **pIgnoreTables);
 int32_t      tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
 void         tsdbReaderSetId(STsdbReader *pReader, const char *idstr);
 void         tsdbReaderClose(STsdbReader *pReader);
@@ -198,47 +185,47 @@ void        *tsdbGetIdx(SMeta *pMeta);
 void        *tsdbGetIvtIdx(SMeta *pMeta);
 uint64_t     tsdbGetReaderMaxVersion(STsdbReader *pReader);
 void         tsdbReaderSetCloseFlag(STsdbReader *pReader);
-int64_t      tsdbGetLastTimestamp(SVnode* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr);
+int64_t      tsdbGetLastTimestamp(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
 
-int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables);
+int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
 int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
                                 SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
 int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
                               SArray *pTableUids);
 void   *tsdbCacherowsReaderClose(void *pReader);
-int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
+int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
 
 void    tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
 size_t  tsdbCacheGetCapacity(SVnode *pVnode);
 size_t  tsdbCacheGetUsage(SVnode *pVnode);
 int32_t tsdbCacheGetElems(SVnode *pVnode);
 
-// tq
+//// tq
-typedef struct SMetaTableInfo {
+//typedef struct SMetaTableInfo {
-  int64_t         suid;
+//  int64_t         suid;
-  int64_t         uid;
+//  int64_t         uid;
-  SSchemaWrapper *schema;
+//  SSchemaWrapper *schema;
-  char            tbName[TSDB_TABLE_NAME_LEN];
+//  char            tbName[TSDB_TABLE_NAME_LEN];
-} SMetaTableInfo;
+//} SMetaTableInfo;
 
 typedef struct SIdInfo {
   int64_t version;
   int32_t index;
 } SIdInfo;
 
-typedef struct SSnapContext {
+//typedef struct SSnapContext {
-  SMeta    *pMeta;
+//  SMeta    *pMeta;
-  int64_t   snapVersion;
+//  int64_t   snapVersion;
-  TBC      *pCur;
+//  TBC      *pCur;
-  int64_t   suid;
+//  int64_t   suid;
-  int8_t    subType;
+//  int8_t    subType;
-  SHashObj *idVersion;
+//  SHashObj *idVersion;
-  SHashObj *suidInfo;
+//  SHashObj *suidInfo;
-  SArray   *idList;
+//  SArray   *idList;
-  int32_t   index;
+//  int32_t   index;
-  bool      withMeta;
+//  bool      withMeta;
-  bool      queryMeta;  // true-get meta, false-get data
+//  bool      queryMeta;  // true-get meta, false-get data
-} SSnapContext;
+//} SSnapContext;
 
 typedef struct STqReader {
   SPackedData     msg;
@@ -257,21 +244,26 @@ typedef struct STqReader {
 } STqReader;
 
 STqReader *tqReaderOpen(SVnode *pVnode);
-void       tqCloseReader(STqReader *);
+void       tqReaderClose(STqReader *);
 
 void    tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
 int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
 int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
 int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
 
-int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id);
+bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid);
-bool    tqNextBlockInWal(STqReader* pReader, const char* idstr);
+bool tqCurrentBlockConsumed(const STqReader* pReader);
-bool    tqNextBlockImpl(STqReader *pReader, const char* idstr);
 
-int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id);
+int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id);
+bool    tqNextBlockInWal(STqReader *pReader, const char *idstr);
+bool    tqNextBlockImpl(STqReader *pReader, const char *idstr);
+SWalReader* tqGetWalReader(STqReader* pReader);
+SSDataBlock* tqGetResultBlock (STqReader* pReader);
+
+int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id);
 int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
 bool    tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
-int32_t tqRetrieveDataBlock(STqReader *pReader, const char* idstr);
+int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock** pRes, const char* idstr);
 int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
 
 int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg);
@@ -288,10 +280,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
 int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot);
 int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
 
-int32_t        buildSnapContext(SMeta *pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta,
+int32_t        buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta,
                                 SSnapContext **ctxRet);
-int32_t        getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
+int32_t        getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
-SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx);
+SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx);
 int32_t        setForSnapShot(SSnapContext *ctx, int64_t uid);
 int32_t        destroySnapContext(SSnapContext *ctx);
 
@@ -350,67 +342,62 @@ struct SVnodeCfg {
   int32_t     tsdbPageSize;
 };
 
-typedef struct {
-  uint64_t uid;
-  uint64_t groupId;
-} STableKeyInfo;
-
 #define TABLE_ROLLUP_ON       ((int8_t)0x1)
 #define TABLE_IS_ROLLUP(FLG)  (((FLG) & (TABLE_ROLLUP_ON)) != 0)
 #define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON)
-struct SMetaEntry {
+//struct SMetaEntry {
-  int64_t  version;
+//  int64_t  version;
-  int8_t   type;
+//  int8_t   type;
-  int8_t   flags;  // TODO: need refactor?
+//  int8_t   flags;  // TODO: need refactor?
-  tb_uid_t uid;
+//  tb_uid_t uid;
-  char    *name;
+//  char    *name;
-  union {
+//  union {
-    struct {
+//    struct {
-      SSchemaWrapper schemaRow;
+//      SSchemaWrapper schemaRow;
-      SSchemaWrapper schemaTag;
+//      SSchemaWrapper schemaTag;
-      SRSmaParam     rsmaParam;
+//      SRSmaParam     rsmaParam;
-    } stbEntry;
+//    } stbEntry;
-    struct {
+//    struct {
-      int64_t  ctime;
+//      int64_t  ctime;
-      int32_t  ttlDays;
+//      int32_t  ttlDays;
-      int32_t  commentLen;
+//      int32_t  commentLen;
-      char    *comment;
+//      char    *comment;
-      tb_uid_t suid;
+//      tb_uid_t suid;
-      uint8_t *pTags;
+//      uint8_t *pTags;
-    } ctbEntry;
+//    } ctbEntry;
-    struct {
+//    struct {
-      int64_t        ctime;
+//      int64_t        ctime;
-      int32_t        ttlDays;
+//      int32_t        ttlDays;
-      int32_t        commentLen;
+//      int32_t        commentLen;
-      char          *comment;
+//      char          *comment;
-      int32_t        ncid;  // next column id
+//      int32_t        ncid;  // next column id
-      SSchemaWrapper schemaRow;
+//      SSchemaWrapper schemaRow;
-    } ntbEntry;
+//    } ntbEntry;
-    struct {
+//    struct {
-      STSma *tsma;
+//      STSma *tsma;
-    } smaEntry;
+//    } smaEntry;
-  };
+//  };
+//
+//  uint8_t *pBuf;
+//};
 
-  uint8_t *pBuf;
+//struct SMetaReader {
-};
+//  int32_t    flags;
+//  SMeta     *pMeta;
+//  SDecoder   coder;
+//  SMetaEntry me;
+//  void      *pBuf;
+//  int32_t    szBuf;
+//};
 
-struct SMetaReader {
+//struct SMTbCursor {
-  int32_t    flags;
+//  TBC        *pDbc;
-  SMeta     *pMeta;
+//  void       *pKey;
-  SDecoder   coder;
+//  void       *pVal;
-  SMetaEntry me;
+//  int32_t     kLen;
-  void      *pBuf;
+//  int32_t     vLen;
-  int32_t    szBuf;
+//  SMetaReader mr;
-};
+//};
-
-struct SMTbCursor {
-  TBC        *pDbc;
-  void       *pKey;
-  void       *pVal;
-  int32_t     kLen;
-  int32_t     vLen;
-  SMetaReader mr;
-};
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/meta.h

@@ -162,6 +162,12 @@ typedef struct {
 int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid,
                         STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey);
 
+// TODO, refactor later
+int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *param, SArray *results);
+int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *parm, SArray *pUids);
+int32_t metaFilterTableName(void *pVnode, SMetaFltParam *param, SArray *pUids);
+int32_t metaFilterTtl(void *pVnode, SMetaFltParam *param, SArray *pUids);
+
 #ifndef META_REFACT
 // SMetaDB
 int  metaOpenDB(SMeta* pMeta);

source/dnode/vnode/src/inc/tsdb.h

@@ -847,9 +847,6 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
 int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
 int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
 
-void   tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
-size_t tsdbCacheGetCapacity(SVnode *pVnode);
-
 // int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
 
 // ========== inline functions ==========

source/dnode/vnode/src/inc/vnodeInt.h

@@ -161,6 +161,8 @@ void*         metaGetIdx(SMeta* pMeta);
 void*         metaGetIvtIdx(SMeta* pMeta);
 int           metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList);
 
+void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+
 int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
 int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
 
@@ -473,6 +475,8 @@ struct SCompactInfo {
   STimeWindow tw;
 };
 
+void initStorageAPI(SStorageAPI* pAPI);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/meta/metaQuery.c

@@ -17,10 +17,16 @@
 #include "osMemory.h"
 #include "tencode.h"
 
+void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI) {
+  SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
+  metaReaderInit(pReader, pMeta, flags);
+  pReader->pAPI = pAPI;
+}
+
 void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) {
   memset(pReader, 0, sizeof(*pReader));
-  pReader->flags = flags;
   pReader->pMeta = pMeta;
+  pReader->flags = flags;
   if (pReader->pMeta && !(flags & META_READER_NOLOCK)) {
     metaRLock(pMeta);
   }
@@ -64,96 +70,20 @@ _err:
   return -1;
 }
 
-// int metaGetTableEntryByUidTest(void* meta, SArray *uidList) {
+bool metaIsTableExist(void *pVnode, tb_uid_t uid) {
-//
+  SVnode* pVnodeObj = pVnode;
-//   SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader));
+  metaRLock(pVnodeObj->pMeta);  // query uid.idx
-//   SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey));
-//   SMeta  *pMeta = meta;
-//   int64_t version;
-//   SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
-//
-//   int64_t stt1 = taosGetTimestampUs();
-//   for(int i = 0; i < taosArrayGetSize(uidList); i++) {
-//     void* ppVal = NULL;
-//     int vlen = 0;
-//     uint64_t *  uid = taosArrayGet(uidList, i);
-//     // query uid.idx
-//     if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) {
-//       continue;
-//     }
-//     version = *(int64_t *)ppVal;
-//
-//     STbDbKey tbDbKey = {.version = version, .uid = *uid};
-//     taosArrayPush(uidVersion, &tbDbKey);
-//     taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t));
-//   }
-//   int64_t stt2 = taosGetTimestampUs();
-//   qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1);
-//
-//   TBC        *pCur = NULL;
-//   tdbTbcOpen(pMeta->pTbDb, &pCur, NULL);
-//   tdbTbcMoveToFirst(pCur);
-//   void *pKey = NULL;
-//   int   kLen = 0;
-//
-//   while(1){
-//     SMetaReader pReader = {0};
-//     int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf);
-//     if (ret < 0) break;
-//     STbDbKey *tmp = (STbDbKey*)pKey;
-//     int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t));
-//     if(ver == NULL || *ver != tmp->version) continue;
-//     taosArrayPush(readerList, &pReader);
-//   }
-//   tdbTbcClose(pCur);
-//
-//   taosArrayClear(readerList);
-//   int64_t stt3 = taosGetTimestampUs();
-//   qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2);
-//   for(int i = 0; i < taosArrayGetSize(uidVersion); i++) {
-//     SMetaReader pReader = {0};
-//
-//     STbDbKey *tbDbKey = taosArrayGet(uidVersion, i);
-//     // query table.db
-//     if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) {
-//       continue;
-//     }
-//     taosArrayPush(readerList, &pReader);
-//   }
-//   int64_t stt4 = taosGetTimestampUs();
-//   qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3);
-//
-//   for(int i = 0; i < taosArrayGetSize(readerList); i++){
-//     SMetaReader* pReader  = taosArrayGet(readerList, i);
-//     metaReaderInit(pReader, meta, 0);
-//     // decode the entry
-//     tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf);
-//
-//     if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) {
-//     }
-//     metaReaderClear(pReader);
-//   }
-//   int64_t stt5 = taosGetTimestampUs();
-//   qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4);
-//   return 0;
-// }
-
-bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) {
-  // query uid.idx
-  metaRLock(pMeta);
-
-  if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) {
-    metaULock(pMeta);
 
+  if (tdbTbGet(pVnodeObj->pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) {
+    metaULock(pVnodeObj->pMeta);
     return false;
   }
 
-  metaULock(pMeta);
+  metaULock(pVnodeObj->pMeta);
-
   return true;
 }
 
-int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
+int metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
   SMeta  *pMeta = pReader->pMeta;
   int64_t version1;
 
@@ -167,7 +97,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
   return metaGetTableEntryByVersion(pReader, version1, uid);
 }
 
-int metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) {
+int metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) {
   SMeta *pMeta = pReader->pMeta;
 
   SMetaInfo info;
@@ -190,7 +120,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
   }
 
   uid = *(tb_uid_t *)pReader->pBuf;
-  return metaGetTableEntryByUid(pReader, uid);
+  return metaReaderGetTableEntryByUid(pReader, uid);
 }
 
 tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
@@ -210,11 +140,11 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
   return uid;
 }
 
-int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) {
+int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) {
   int         code = 0;
   SMetaReader mr = {0};
-  metaReaderInit(&mr, (SMeta *)meta, 0);
+  metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0);
-  code = metaGetTableEntryByUid(&mr, uid);
+  code = metaReaderGetTableEntryByUid(&mr, uid);
   if (code < 0) {
     metaReaderClear(&mr);
     return -1;
@@ -230,7 +160,7 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) {
   int         code = 0;
   SMetaReader mr = {0};
   metaReaderInit(&mr, (SMeta *)meta, 0);
-  code = metaGetTableEntryByUid(&mr, uid);
+  code = metaReaderGetTableEntryByUid(&mr, uid);
   if (code < 0) {
     metaReaderClear(&mr);
     return -1;
@@ -241,15 +171,15 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) {
   return 0;
 }
 
-int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) {
+int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) {
   int         code = 0;
   SMetaReader mr = {0};
-  metaReaderInit(&mr, (SMeta *)meta, 0);
+  metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0);
 
   SMetaReader *pReader = &mr;
 
   // query name.idx
-  if (tdbTbGet(pReader->pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
+  if (tdbTbGet(((SMeta*)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
     terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     metaReaderClear(&mr);
     return -1;
@@ -262,10 +192,10 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) {
   return 0;
 }
 
-int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType) {
+int metaGetTableTypeByName(void *pVnode, char *tbName, ETableType *tbType) {
   int         code = 0;
   SMetaReader mr = {0};
-  metaReaderInit(&mr, (SMeta *)meta, 0);
+  metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0);
 
   code = metaGetTableEntryByName(&mr, tbName);
   if (code == 0) *tbType = mr.me.type;
@@ -283,7 +213,7 @@ int metaReadNext(SMetaReader *pReader) {
 }
 
 #if 1  // ===================================================
-SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
+SMTbCursor *metaOpenTbCursor(void *pVnode) {
   SMTbCursor *pTbCur = NULL;
 
   pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur));
@@ -291,12 +221,12 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
     return NULL;
   }
 
-  metaReaderInit(&pTbCur->mr, pMeta, 0);
+  SVnode* pVnodeObj = pVnode;
+  metaReaderInit(&pTbCur->mr, pVnodeObj->pMeta, 0);
 
-  tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL);
+  tdbTbcOpen(pVnodeObj->pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL);
-
-  tdbTbcMoveToFirst(pTbCur->pDbc);
 
+  tdbTbcMoveToFirst((TBC *)pTbCur->pDbc);
   return pTbCur;
 }
 
@@ -306,7 +236,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) {
     tdbFree(pTbCur->pVal);
     metaReaderClear(&pTbCur->mr);
     if (pTbCur->pDbc) {
-      tdbTbcClose(pTbCur->pDbc);
+      tdbTbcClose((TBC *)pTbCur->pDbc);
     }
     taosMemoryFree(pTbCur);
   }
@@ -318,7 +248,7 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) {
   STbCfg tbCfg;
 
   for (;;) {
-    ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
+    ret = tdbTbcNext((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
     if (ret < 0) {
       return -1;
     }
@@ -342,7 +272,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) {
   STbCfg tbCfg;
 
   for (;;) {
-    ret = tdbTbcPrev(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
+    ret = tdbTbcPrev((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
     if (ret < 0) {
       return -1;
     }
@@ -876,7 +806,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
   STSma  *pTSma = NULL;
   for (int i = 0; i < pSW->number; ++i) {
     smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i);
-    if (metaGetTableEntryByUid(&mr, smaId) < 0) {
+    if (metaReaderGetTableEntryByUid(&mr, smaId) < 0) {
       tDecoderClear(&mr.coder);
       metaWarn("vgId:%d, no entry for tbId:%" PRIi64 ", smaId:%" PRIi64, TD_VID(pMeta->pVnode), uid, smaId);
       continue;
@@ -926,7 +856,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
   STSma      *pTSma = NULL;
   SMetaReader mr = {0};
   metaReaderInit(&mr, pMeta, 0);
-  if (metaGetTableEntryByUid(&mr, indexUid) < 0) {
+  if (metaReaderGetTableEntryByUid(&mr, indexUid) < 0) {
     metaWarn("vgId:%d, failed to get table entry for smaId:%" PRIi64, TD_VID(pMeta->pVnode), indexUid);
     metaReaderClear(&mr);
     return NULL;
@@ -1027,7 +957,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) {
 
 #endif
 
-const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) {
+const void *metaGetTableTagVal(const void *pTag, int16_t type, STagVal *val) {
   STag *tag = (STag *)pTag;
   if (type == TSDB_DATA_TYPE_JSON) {
     return tag;
@@ -1083,8 +1013,10 @@ typedef struct {
   int32_t  vLen;
 } SIdxCursor;
 
-int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
+int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
-  int32_t ret = 0;
+  SMeta         *pMeta = ((SVnode*)pVnode)->pMeta;
+  SMetaFltParam *param = arg;
+  int32_t        ret = 0;
 
   SIdxCursor *pCursor = NULL;
   pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor));
@@ -1141,9 +1073,11 @@ END:
   return ret;
 }
 
-int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
+int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
-  int32_t ret = 0;
+  SMeta         *pMeta = ((SVnode*)pVnode)->pMeta;
-  char   *buf = NULL;
+  SMetaFltParam *param = arg;
+  int32_t        ret = 0;
+  char          *buf = NULL;
 
   STagIdxKey *pKey = NULL;
   int32_t     nKey = 0;
@@ -1206,9 +1140,11 @@ END:
 
   return ret;
 }
-int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
+int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
-  int32_t ret = 0;
+  SMeta         *pMeta = ((SVnode*)pVnode)->pMeta;
-  char   *buf = NULL;
+  SMetaFltParam *param = arg;
+  int32_t        ret = 0;
+  char          *buf = NULL;
 
   STtlIdxKey *pKey = NULL;
   int32_t     nKey = 0;
@@ -1235,7 +1171,10 @@ END:
   // impl later
   return 0;
 }
-int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
+int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
+  SMeta         *pMeta = ((SVnode*)pVnode)->pMeta;
+  SMetaFltParam *param = arg;
+
   SMetaEntry oStbEntry = {0};
   int32_t    ret = -1;
   char      *buf = NULL;
@@ -1418,7 +1357,8 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi
   return ret;
 }
 
-int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) {
+int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) {
+  SMeta* pMeta = ((SVnode*) pVnode)->pMeta;
   const int32_t LIMIT = 128;
 
   int32_t isLock = false;
@@ -1450,8 +1390,8 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) {
   return 0;
 }
 
-int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *pUidTagInfo) {
+int32_t metaGetTableTags(void* pVnode, uint64_t suid, SArray *pUidTagInfo) {
-  SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid, 1);
+  SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode*)pVnode)->pMeta, suid, 1);
 
   // If len > 0 means there already have uids, and we only want the
   // tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept
@@ -1556,30 +1496,35 @@ _exit:
   return code;
 }
 
-int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo) {
+int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t* numOfTables) {
   int32_t code = 0;
+  *numOfTables = 0;
 
-  metaRLock(pMeta);
+  SVnode* pVnodeObj = pVnode;
+  metaRLock(pVnodeObj->pMeta);
 
   // fast path: search cache
-  if (metaStatsCacheGet(pMeta, uid, pInfo) == TSDB_CODE_SUCCESS) {
+  SMetaStbStats state = {0};
-    metaULock(pMeta);
+  if (metaStatsCacheGet(pVnodeObj->pMeta, uid, &state) == TSDB_CODE_SUCCESS) {
+    metaULock(pVnodeObj->pMeta);
+    *numOfTables = state.ctbNum;
     goto _exit;
   }
 
   // slow path: search TDB
   int64_t ctbNum = 0;
-  vnodeGetCtbNum(pMeta->pVnode, uid, &ctbNum);
+  vnodeGetCtbNum(pVnode, uid, &ctbNum);
 
-  metaULock(pMeta);
+  metaULock(pVnodeObj->pMeta);
+  *numOfTables = ctbNum;
 
-  pInfo->uid = uid;
+  state.uid = uid;
-  pInfo->ctbNum = ctbNum;
+  state.ctbNum = ctbNum;
 
   // upsert the cache
-  metaWLock(pMeta);
+  metaWLock(pVnodeObj->pMeta);
-  metaStatsCacheUpsert(pMeta, pInfo);
+  metaStatsCacheUpsert(pVnodeObj->pMeta, &state);
-  metaULock(pMeta);
+  metaULock(pVnodeObj->pMeta);
 
 _exit:
   return code;

source/dnode/vnode/src/meta/metaSma.c

@@ -36,7 +36,7 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
   // validate req
   // save smaIndex
   metaReaderInit(&mr, pMeta, 0);
-  if (metaGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) {
+  if (metaReaderGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) {
 #if 1
     terrno = TSDB_CODE_TSMA_ALREADY_EXIST;
     metaReaderClear(&mr);

source/dnode/vnode/src/meta/metaSnapshot.c

@@ -222,29 +222,29 @@ static void destroySTableInfoForChildTable(void* data) {
 }
 
 static void MoveToSnapShotVersion(SSnapContext* ctx) {
-  tdbTbcClose(ctx->pCur);
+  tdbTbcClose((TBC*)ctx->pCur);
-  tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL);
+  tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL);
   STbDbKey key = {.version = ctx->snapVersion, .uid = INT64_MAX};
   int      c = 0;
-  tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c);
+  tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c);
   if (c < 0) {
-    tdbTbcMoveToPrev(ctx->pCur);
+    tdbTbcMoveToPrev((TBC*)ctx->pCur);
   }
 }
 
 static int32_t MoveToPosition(SSnapContext* ctx, int64_t ver, int64_t uid) {
-  tdbTbcClose(ctx->pCur);
+  tdbTbcClose((TBC*)ctx->pCur);
-  tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL);
+  tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL);
   STbDbKey key = {.version = ver, .uid = uid};
   int      c = 0;
-  tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c);
+  tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c);
   return c;
 }
 
 static void MoveToFirst(SSnapContext* ctx) {
-  tdbTbcClose(ctx->pCur);
+  tdbTbcClose((TBC*)ctx->pCur);
-  tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL);
+  tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL);
-  tdbTbcMoveToFirst(ctx->pCur);
+  tdbTbcMoveToFirst((TBC*)ctx->pCur);
 }
 
 static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) {
@@ -260,12 +260,12 @@ static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo)
   taosHashPut(suidInfo, &me->uid, sizeof(tb_uid_t), &dataTmp, sizeof(STableInfoForChildTable));
 }
 
-int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta,
+int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta,
                          SSnapContext** ctxRet) {
   SSnapContext* ctx = taosMemoryCalloc(1, sizeof(SSnapContext));
   if (ctx == NULL) return -1;
   *ctxRet = ctx;
-  ctx->pMeta = pMeta;
+  ctx->pMeta = pVnode->pMeta;
   ctx->snapVersion = snapVersion;
   ctx->suid = suid;
   ctx->subType = subType;
@@ -291,7 +291,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
   metaDebug("tmqsnap init snapVersion:%" PRIi64, ctx->snapVersion);
   MoveToFirst(ctx);
   while (1) {
-    int32_t ret = tdbTbcNext(ctx->pCur, &pKey, &kLen, &pVal, &vLen);
+    int32_t ret = tdbTbcNext((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen);
     if (ret < 0) break;
     STbDbKey* tmp = (STbDbKey*)pKey;
     if (tmp->version > ctx->snapVersion) break;
@@ -301,7 +301,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
       continue;
     }
 
-    if (tdbTbGet(pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) <
+    if (tdbTbGet(ctx->pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) <
         0) {  // check if table exist for now, need optimize later
       continue;
     }
@@ -329,7 +329,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
 
   MoveToSnapShotVersion(ctx);
   while (1) {
-    int32_t ret = tdbTbcPrev(ctx->pCur, &pKey, &kLen, &pVal, &vLen);
+    int32_t ret = tdbTbcPrev((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen);
     if (ret < 0) break;
 
     STbDbKey* tmp = (STbDbKey*)pKey;
@@ -378,7 +378,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
 }
 
 int32_t destroySnapContext(SSnapContext* ctx) {
-  tdbTbcClose(ctx->pCur);
+  tdbTbcClose((TBC*)ctx->pCur);
   taosArrayDestroy(ctx->idList);
   taosHashCleanup(ctx->idVersion);
   taosHashCleanup(ctx->suidInfo);
@@ -466,7 +466,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
   return c;
 }
 
-int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) {
+int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) {
   int32_t ret = 0;
   void*   pKey = NULL;
   void*   pVal = NULL;
@@ -496,7 +496,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in
     metaDebug("tmqsnap get meta not exist uid:%" PRIi64 " version:%" PRIi64, *uid, idInfo->version);
   }
 
-  tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
+  tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
   SDecoder   dc = {0};
   SMetaEntry me = {0};
   tDecoderInit(&dc, pVal, vLen);
@@ -598,7 +598,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in
   return ret;
 }
 
-SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) {
+SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext* ctx) {
   SMetaTableInfo result = {0};
   void*          pKey = NULL;
   void*          pVal = NULL;
@@ -619,10 +619,10 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) {
 
     int32_t ret = MoveToPosition(ctx, idInfo->version, *uidTmp);
     if (ret != 0) {
-      metaDebug("tmqsnap getUidfromSnapShot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version);
+      metaDebug("tmqsnap getMetaTableInfoFromSnapshot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version);
       continue;
     }
-    tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
+    tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
     SDecoder   dc = {0};
     SMetaEntry me = {0};
     tDecoderInit(&dc, pVal, vLen);

source/dnode/vnode/src/meta/metaTable.c

@@ -690,7 +690,7 @@ _err:
   return -1;
 }
 
-int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) {
+int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) {
   SMetaEntry  me = {0};
   SMetaReader mr = {0};
 
@@ -729,7 +729,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe
   metaReaderClear(&mr);
 
   // build SMetaEntry
-  me.version = version;
+  me.version = ver;
   me.type = pReq->type;
   me.uid = pReq->uid;
   me.name = pReq->name;

source/dnode/vnode/src/sma/smaRollup.c

@@ -276,12 +276,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
       return TSDB_CODE_FAILED;
     }
 
-    SReadHandle handle = {
+    SReadHandle handle = { .vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState };
-        .meta = pVnode->pMeta,
+    initStorageAPI(&handle.api);
-        .vnode = pVnode,
+
-        .initTqReader = 1,
-        .pStateBackend = pStreamState,
-    };
     pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode));
     if (!pRSmaInfo->taskInfo[idx]) {
       terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
@@ -853,11 +850,8 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t
   code = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  SReadHandle handle = {
+  SReadHandle handle = { .vnode = pVnode, .initTqReader = 1 };
-      .meta = pVnode->pMeta,
+  initStorageAPI(&handle.api);
-      .vnode = pVnode,
-      .initTqReader = 1,
-  };
 
   if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) {
     code = TSDB_CODE_APP_ERROR;
@@ -904,7 +898,7 @@ static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) {
 
   metaReaderInit(&mr, SMA_META(pSma), 0);
   smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid);
-  if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) {
+  if (metaReaderGetTableEntryByUidCache(&mr, pInfo->suid) < 0) {
     code = terrno;
     TSDB_CHECK_CODE(code, lino, _exit);
   }
@@ -1131,7 +1125,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
   for (int64_t i = 0; i < arrSize; ++i) {
     suid = *(tb_uid_t *)taosArrayGet(suidList, i);
     smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid);
-    if (metaGetTableEntryByUidCache(&mr, suid) < 0) {
+    if (metaReaderGetTableEntryByUidCache(&mr, suid) < 0) {
       code = terrno;
       TSDB_CHECK_CODE(code, lino, _exit);
     }

source/dnode/vnode/src/tq/tq.c

@@ -69,12 +69,12 @@ static void destroyTqHandle(void* data) {
   if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
     taosMemoryFreeClear(pData->execHandle.execCol.qmsg);
   } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) {
-    tqCloseReader(pData->execHandle.pTqReader);
+    tqReaderClose(pData->execHandle.pTqReader);
     walCloseReader(pData->pWalReader);
     taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid);
   } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
     walCloseReader(pData->pWalReader);
-    tqCloseReader(pData->execHandle.pTqReader);
+    tqReaderClose(pData->execHandle.pTqReader);
   }
   if(pData->msg != NULL) {
     rpcFreeCont(pData->msg->pCont);
@@ -510,8 +510,6 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
   int64_t sver = 0, ever = 0;
   walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever);
 
-  int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader);
-
   SMqDataRsp dataRsp = {0};
   tqInitDataRsp(&dataRsp, &req);
 
@@ -537,7 +535,12 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
     dataRsp.rspOffset.type = TMQ_OFFSET__LOG;
 
     if (reqOffset.type == TMQ_OFFSET__LOG) {
-      dataRsp.rspOffset.version = currentVer; // return current consume offset value
+      int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader);
+      if (currentVer == -1) { // not start to read data from wal yet, return req offset directly
+        dataRsp.rspOffset.version = reqOffset.version;
+      } else {
+        dataRsp.rspOffset.version = currentVer;  // return current consume offset value
+      }
     } else if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
       dataRsp.rspOffset.version = sver;  // not consume yet, set the earliest position
     } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
@@ -670,8 +673,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
     int64_t ver = pRef->refVer;
     pHandle->pRef = pRef;
 
-    SReadHandle handle = {
+    SReadHandle handle = {.vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver};
-        .meta = pVnode->pMeta, .vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver};
+    initStorageAPI(&handle.api);
+
     pHandle->snapshotVer = ver;
 
     if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
@@ -689,7 +693,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
 
       pHandle->execHandle.execDb.pFilterOutTbUid =
           taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
-      buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
+      buildSnapContext(handle.vnode, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
                        (SSnapContext**)(&handle.sContext));
 
       pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId);
@@ -708,7 +712,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
       tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList);
       taosArrayDestroy(tbUidList);
 
-      buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta,
+      buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta,
                        (SSnapContext**)(&handle.sContext));
       pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId);
     }
@@ -787,8 +791,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
       return -1;
     }
 
-    SReadHandle handle = {
+    SReadHandle handle = {.vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState};
-        .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState};
+    initStorageAPI(&handle.api);
 
     pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId);
     if (pTask->exec.pExecutor == NULL) {
@@ -803,9 +807,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
     }
 
     int32_t     numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo);
-    SReadHandle mgHandle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState};
+    SReadHandle handle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState};
+    initStorageAPI(&handle.api);
 
-    pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId);
+    pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId);
     if (pTask->exec.pExecutor == NULL) {
       return -1;
     }
@@ -1083,6 +1088,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t
 
   qDebug("s-task:%s set the start wal offset to be:%"PRId64, pTask->id.idStr, sversion);
   walReaderSeekVer(pTask->exec.pWalReader, sversion);
+  pTask->chkInfo.currentVer = sversion;
 
   if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@@ -1278,6 +1284,8 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
 
 int32_t tqProcessTaskDropReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
   SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
+  tqDebug("vgId:%d receive msg to drop stream task:0x%x", TD_VID(pTq->pVnode), pReq->taskId);
+
   streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId);
   return 0;
 }

source/dnode/vnode/src/tq/tqMeta.c

@@ -298,13 +298,14 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
     walSetRefVer(handle.pRef, handle.snapshotVer);
 
     SReadHandle reader = {
-        .meta = pTq->pVnode->pMeta,
         .vnode = pTq->pVnode,
         .initTableReader = true,
         .initTqReader = true,
-        .version = handle.snapshotVer,
+        .version = handle.snapshotVer
     };
 
+    initStorageAPI(&reader.api);
+
     if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
       handle.execHandle.task =
           qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0);
@@ -330,7 +331,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
       handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
       handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode);
 
-      buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta,
+      buildSnapContext(reader.vnode, reader.version, 0, handle.execHandle.subType, handle.fetchMeta,
                        (SSnapContext**)(&reader.sContext));
       handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0);
     } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
@@ -347,7 +348,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
       tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList);
       taosArrayDestroy(tbUidList);
 
-      buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType,
+      buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType,
                        handle.fetchMeta, (SSnapContext**)(&reader.sContext));
       handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0);
     }

source/dnode/vnode/src/tq/tqRead.c

@@ -273,7 +273,7 @@ STqReader* tqReaderOpen(SVnode* pVnode) {
   return pReader;
 }
 
-void tqCloseReader(STqReader* pReader) {
+void tqReaderClose(STqReader* pReader) {
   // close wal reader
   if (pReader->pWalReader) {
     walCloseReader(pReader->pWalReader);
@@ -294,7 +294,7 @@ void tqCloseReader(STqReader* pReader) {
   taosMemoryFree(pReader);
 }
 
-int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
+int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) {
   if (walReaderSeekVer(pReader->pWalReader, ver) < 0) {
     return -1;
   }
@@ -394,8 +394,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) {
       SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
 
       if (pReader->tbIdHash == NULL) {
-        int32_t code = tqRetrieveDataBlock(pReader, NULL);
+          SSDataBlock* pRes = NULL;
-        if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) {
+        int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL);
+        if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
           return true;
         }
       }
@@ -404,8 +405,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) {
       if (ret != NULL) {
         tqDebug("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver);
 
-        int32_t code = tqRetrieveDataBlock(pReader, NULL);
+        SSDataBlock* pRes = NULL;
-        if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) {
+        int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL);
+        if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
           return true;
         }
       } else {
@@ -440,6 +442,14 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i
   return 0;
 }
 
+SWalReader* tqGetWalReader(STqReader* pReader) {
+  return pReader->pWalReader;
+}
+
+SSDataBlock* tqGetResultBlock (STqReader* pReader) {
+  return pReader->pResBlock;
+}
+
 bool tqNextBlockImpl(STqReader* pReader, const char* idstr) {
   if (pReader->msg.msgStr == NULL) {
     return false;
@@ -592,12 +602,14 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol
   return code;
 }
 
-int32_t tqRetrieveDataBlock(STqReader* pReader, const char* id) {
+int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) {
   tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk);
 
   SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++);
 
   SSDataBlock* pBlock = pReader->pResBlock;
+  *pRes = pBlock;
+
   blockDataCleanup(pBlock);
 
   int32_t sversion = pSubmitTbData->sver;
@@ -1024,6 +1036,14 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) {
   return 0;
 }
 
+bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid) {
+  return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t));
+}
+
+bool tqCurrentBlockConsumed(const STqReader* pReader) {
+    return pReader->msg.msgStr == NULL;
+}
+
 int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
   for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
     int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
@@ -1069,7 +1089,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
         for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) {
           uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i);
 
-          int32_t code = metaGetTableEntryByUidCache(&mr, *id);
+          int32_t code = metaReaderGetTableEntryByUidCache(&mr, *id);
           if (code != TSDB_CODE_SUCCESS) {
             tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
             continue;

source/dnode/vnode/src/tq/tqRestore.c

@@ -61,9 +61,10 @@ static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) {
   // seek the stored version and extract data from WAL
   int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader);
   if (pTask->chkInfo.currentVer < firstVer) {
+    tqWarn("vgId:%d s-task:%s ver:%"PRId64" earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId,
+           pTask->id.idStr, pTask->chkInfo.currentVer, firstVer, firstVer);
+
     pTask->chkInfo.currentVer = firstVer;
-    tqWarn("vgId:%d s-task:%s ver earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId,
-           pTask->id.idStr, firstVer, pTask->chkInfo.currentVer);
 
     // todo need retry if failed
     int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer);

source/dnode/vnode/src/tq/tqScan.c

@@ -51,7 +51,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in
   metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
 
   // TODO add reference to gurantee success
-  if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
+  if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) {
     metaReaderClear(&mr);
     return -1;
   }

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -703,6 +703,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
     *pTmpLastCol = *pLastCol;
     pLastCol = pTmpLastCol;
 
+    reallocVarData(&pLastCol->colVal);
     size_t charge = sizeof(*pLastCol);
     if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) {
       charge += pLastCol->colVal.value.nData;
@@ -789,7 +790,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA
         code = -1;
       }
 
-      taosArraySet(pLastArray, idxKey->idx, pLastCol);
+      SLastCol lastCol = *pLastCol;
+      reallocVarData(&lastCol.colVal);
+      taosArraySet(pLastArray, idxKey->idx, &lastCol);
       taosArrayRemove(remainCols, j);
 
       taosMemoryFree(values_list[i]);
@@ -825,7 +828,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
       SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h);
 
       SLastCol lastCol = *pLastCol;
-      // reallocVarData(&lastCol.colVal);
+      reallocVarData(&lastCol.colVal);
       taosArrayPush(pLastArray, &lastCol);
 
       if (h) {
@@ -853,8 +856,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
 
         SLastCol lastCol = *pLastCol;
         reallocVarData(&lastCol.colVal);
-
         taosArraySet(pLastArray, idxKey->idx, &lastCol);
+
         if (h) {
           taosLRUCacheRelease(pCache, h, false);
         }
@@ -937,14 +940,14 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
   }
 
   // build keys & multi get from rocks
-  int     num_keys = pTSchema->numOfCols;
+  int          num_keys = pTSchema->numOfCols;
-  char  **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *));
+  char       **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *));
-  size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t));
+  size_t      *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t));
+  const size_t klen = ROCKS_KEY_LEN;
   for (int i = 0; i < num_keys; ++i) {
     int16_t cid = pTSchema->columns[i].colId;
 
-    size_t klen = ROCKS_KEY_LEN;
+    char *keys = taosMemoryCalloc(2, sizeof(SLastKey));
-    char  *keys = taosMemoryCalloc(2, sizeof(SLastKey));
     ((SLastKey *)keys)[0] = (SLastKey){.ltype = 1, .uid = uid, .cid = cid};
     ((SLastKey *)keys)[1] = (SLastKey){.ltype = 0, .uid = uid, .cid = cid};
 
@@ -960,39 +963,35 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
   rocksMayWrite(pTsdb, true, false, false);
   rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list,
                     keys_list_sizes, values_list, values_list_sizes, errs);
-  for (int i = 0; i < num_keys; ++i) {
-    taosMemoryFree(keys_list[i]);
-  }
   for (int i = 0; i < num_keys * 2; ++i) {
-    rocksdb_free(errs[i]);
+    if (errs[i]) {
+      rocksdb_free(errs[i]);
+    }
   }
-  taosMemoryFree(keys_list);
-  taosMemoryFree(keys_list_sizes);
   taosMemoryFree(errs);
 
   rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
   for (int i = 0; i < num_keys; ++i) {
     SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]);
     if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) {
-      SLastKey *key = &(SLastKey){.ltype = 1, .uid = uid, .cid = pLastCol->colVal.cid};
+      rocksdb_writebatch_delete(wb, keys_list[i], klen);
-      size_t    klen = ROCKS_KEY_LEN;
-
-      rocksdb_writebatch_delete(wb, (char *)key, klen);
-      taosLRUCacheErase(pTsdb->lruCache, key, klen);
     }
+    taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen);
 
     pLastCol = tsdbCacheDeserialize(values_list[i + num_keys]);
     if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) {
-      SLastKey *key = &(SLastKey){.ltype = 0, .uid = uid, .cid = pLastCol->colVal.cid};
+      rocksdb_writebatch_delete(wb, keys_list[num_keys + i], klen);
-      size_t    klen = ROCKS_KEY_LEN;
-
-      rocksdb_writebatch_delete(wb, (char *)key, klen);
-      taosLRUCacheErase(pTsdb->lruCache, key, klen);
     }
+    taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen);
 
     rocksdb_free(values_list[i]);
     rocksdb_free(values_list[i + num_keys]);
   }
+  for (int i = 0; i < num_keys; ++i) {
+    taosMemoryFree(keys_list[i]);
+  }
+  taosMemoryFree(keys_list);
+  taosMemoryFree(keys_list_sizes);
   taosMemoryFree(values_list);
   taosMemoryFree(values_list_sizes);
 
@@ -1432,7 +1431,7 @@ static tb_uid_t getTableSuidByUid(tb_uid_t uid, STsdb *pTsdb) {
 
   SMetaReader mr = {0};
   metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
-  if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
+  if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) {
     metaReaderClear(&mr);  // table not esist
     return 0;
   }
@@ -1871,10 +1870,14 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
           if (isLast && (pColData->flag & HAS_VALUE)) {
             skipBlock = false;
             break;
-          } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+          } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
             skipBlock = false;
             break;
-          }
+            }*/
+        }
+
+        if (!isLast) {
+          skipBlock = false;
         }
 
         if (skipBlock) {
@@ -1908,6 +1911,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
       if (checkRemainingRow) {
         bool skipBlock = true;
         int  inputColIndex = 0;
+        if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) {
+          ++inputColIndex;
+        }
         for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
           SColData *pColData = &state->pBlockData->aColData[colIndex];
           int16_t   cid = pColData->cid;
@@ -1916,15 +1922,19 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
             if (isLast && (pColData->flag & HAS_VALUE)) {
               skipBlock = false;
               break;
-            } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+            } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
               skipBlock = false;
               break;
-            }
+              }*/
 
             ++inputColIndex;
           }
         }
 
+        if (!isLast) {
+          skipBlock = false;
+        }
+
         if (skipBlock) {
           if (--state->iBlock < 0) {
             tsdbDataFReaderClose(state->pDataFReader);
@@ -2145,9 +2155,14 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) {
       return false;
     } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) {
       if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) {
+        // if (key->version <= pItemFront->version || key->version <= pItemBack->version) {
         return true;
       } else {
-        return false;
+        if (*iSkyline > 1) {
+          --*iSkyline;
+        } else {
+          return false;
+        }
       }
     } else {
       if (*iSkyline > 1) {
@@ -2959,7 +2974,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray,
 
   do {
     TSDBROW *pRow = NULL;
-    nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray));
+    nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray));
 
     if (!pRow) {
       break;

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -315,14 +315,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
       tsdbCacheGetBatch(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype);
       // tsdbCacheGet(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype);
       if (TARRAY_SIZE(pRow) <= 0) {
-        // taosArrayClearEx(pRow, freeItem);
+        taosArrayClearEx(pRow, freeItem);
-        taosArrayClear(pRow);
+        // taosArrayClear(pRow);
         continue;
       }
       SLastCol* pColVal = taosArrayGet(pRow, 0);
       if (COL_VAL_IS_NONE(&pColVal->colVal)) {
-        // taosArrayClearEx(pRow, freeItem);
+        taosArrayClearEx(pRow, freeItem);
-        taosArrayClear(pRow);
+        // taosArrayClear(pRow);
         continue;
       }
 
@@ -381,8 +381,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
         }
       }
 
-      // taosArrayClearEx(pRow, freeItem);
+      taosArrayClearEx(pRow, freeItem);
-      taosArrayClear(pRow);
+      // taosArrayClear(pRow);
     }
 
     if (hasRes) {
@@ -394,20 +394,20 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
 
       tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
       if (TARRAY_SIZE(pRow) <= 0) {
-        // taosArrayClearEx(pRow, freeItem);
+        taosArrayClearEx(pRow, freeItem);
-        taosArrayClear(pRow);
+        // taosArrayClear(pRow);
         continue;
       }
       SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0);
       if (COL_VAL_IS_NONE(&pColVal->colVal)) {
-        // taosArrayClearEx(pRow, freeItem);
+        taosArrayClearEx(pRow, freeItem);
-        taosArrayClear(pRow);
+        // taosArrayClear(pRow);
         continue;
       }
 
       saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
-      // taosArrayClearEx(pRow, freeItem);
+      taosArrayClearEx(pRow, freeItem);
-      taosArrayClear(pRow);
+      // taosArrayClear(pRow);
 
       taosArrayPush(pTableUidList, &uid);
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -190,9 +190,9 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
     tsdbCacheDeleteLast(pTsdb->lruCache, pTbData->uid, eKey);
   }
   */
-  if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) {
+  // if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) {
-    tsdbCacheDel(pTsdb, suid, uid, sKey, eKey);
+  tsdbCacheDel(pTsdb, suid, uid, sKey, eKey);
-  }
+  //}
 
   tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
             " at version %" PRId64,

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -1848,7 +1848,7 @@ static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBloc
   SDataBlockToLoadInfo info = {0};
   getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader);
   bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf ||
-                            info.overlapWithDelInfo || info.overlapWithLastBlock);
+                            info.overlapWithDelInfo || info.overlapWithLastBlock || info.partiallyRequired);
   return isCleanFileBlock;
 }
 
@@ -3440,6 +3440,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) {
       if (!hasNexTable) {
         return TSDB_CODE_SUCCESS;
       }
+      pBlockScanInfo = pStatus->pTableIter;
     }
     
     initMemDataIterator(*pBlockScanInfo, pReader);
@@ -5380,10 +5381,10 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) {
   return rows;
 }
 
-int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) {
+int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) {
   SMetaReader mr = {0};
-  metaReaderInit(&mr, pVnode->pMeta, 0);
+  metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0);
-  int32_t code = metaGetTableEntryByUidCache(&mr, uid);
+  int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
     metaReaderClear(&mr);
@@ -5396,7 +5397,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
   if (mr.me.type == TSDB_CHILD_TABLE) {
     tDecoderClear(&mr.coder);
     *suid = mr.me.ctbEntry.suid;
-    code = metaGetTableEntryByUidCache(&mr, *suid);
+    code = metaReaderGetTableEntryByUidCache(&mr, *suid);
     if (code != TSDB_CODE_SUCCESS) {
       terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
       metaReaderClear(&mr);
@@ -5412,7 +5413,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
   metaReaderClear(&mr);
 
   // get the newest table schema version
-  code = metaGetTbTSchemaEx(pVnode->pMeta, *suid, uid, -1, pSchema);
+  code = metaGetTbTSchemaEx(((SVnode*)pVnode)->pMeta, *suid, uid, -1, pSchema);
   return code;
 }
 

source/dnode/vnode/src/vnd/vnodeInitApi.c

@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "storageapi.h"
+#include "vnodeInt.h"
+#include "tstreamUpdate.h"
+#include "meta.h"
+
+static void initTsdbReaderAPI(TsdReader* pReader);
+static void initMetadataAPI(SStoreMeta* pMeta);
+static void initTqAPI(SStoreTqReader* pTq);
+static void initStateStoreAPI(SStateStore* pStore);
+static void initMetaReaderAPI(SStoreMetaReader* pMetaReader);
+static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter);
+static void initFunctionStateStore(SFunctionStateStore* pStore);
+static void initCacheFn(SStoreCacheReader* pCache);
+static void initSnapshotFn(SStoreSnapshotFn* pSnapshot);
+
+void initStorageAPI(SStorageAPI* pAPI) {
+  initTsdbReaderAPI(&pAPI->tsdReader);
+  initMetadataAPI(&pAPI->metaFn);
+  initTqAPI(&pAPI->tqReaderFn);
+  initStateStoreAPI(&pAPI->stateStore);
+  initMetaReaderAPI(&pAPI->metaReaderFn);
+  initMetaFilterAPI(&pAPI->metaFilter);
+  initFunctionStateStore(&pAPI->functionStore);
+  initCacheFn(&pAPI->cacheFn);
+  initSnapshotFn(&pAPI->snapshotFn);
+}
+
+void initTsdbReaderAPI(TsdReader* pReader) {
+  pReader->tsdReaderOpen = (__store_reader_open_fn_t)tsdbReaderOpen;
+  pReader->tsdReaderClose = tsdbReaderClose;
+
+  pReader->tsdNextDataBlock = tsdbNextDataBlock;
+
+  pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock;
+  pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock;
+
+  pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA;
+
+  pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag;
+  pReader->tsdReaderResetStatus = tsdbReaderReset;
+
+  pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo;
+  pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable;    // todo this function should be moved away
+
+  pReader->tsdSetQueryTableList = tsdbSetTableList;
+  pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId;
+}
+
+void initMetadataAPI(SStoreMeta* pMeta) {
+  pMeta->isTableExisted = metaIsTableExist;
+
+  pMeta->openTableMetaCursor = metaOpenTbCursor;
+  pMeta->closeTableMetaCursor = metaCloseTbCursor;
+  pMeta->cursorNext = metaTbCursorNext;
+  pMeta->cursorPrev = metaTbCursorPrev;
+
+  pMeta->getBasicInfo = vnodeGetInfo;
+  pMeta->getNumOfChildTables = metaGetStbStats;
+
+  pMeta->getChildTableList = vnodeGetCtbIdList;
+
+  pMeta->storeGetIndexInfo = vnodeGetIdx;
+  pMeta->getInvertIndex = vnodeGetIvtIdx;
+
+  pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal;
+  pMeta->getTableTags = metaGetTableTags;
+  pMeta->getTableTagsByUid = metaGetTableTagsByUids;
+
+  pMeta->getTableUidByName = metaGetTableUidByName;
+  pMeta->getTableTypeByName = metaGetTableTypeByName;
+  pMeta->getTableNameByUid = metaGetTableNameByUid;
+
+  pMeta->getTableSchema = tsdbGetTableSchema;   // todo refactor
+  pMeta->storeGetTableList = vnodeGetTableList;
+}
+
+void initTqAPI(SStoreTqReader* pTq) {
+  pTq->tqReaderOpen = tqReaderOpen;
+  pTq->tqReaderSetColIdList = tqReaderSetColIdList;
+
+  pTq->tqReaderClose = tqReaderClose;
+  pTq->tqReaderSeek = tqReaderSeek;
+  pTq->tqRetrieveBlock = tqRetrieveDataBlock;
+
+  pTq->tqReaderNextBlockInWal = tqNextBlockInWal;
+
+  pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it
+
+  pTq->tqReaderAddTables = tqReaderAddTbUidList;
+  pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList;
+
+  pTq->tqReaderRemoveTables = tqReaderRemoveTbUidList;
+
+  pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable;
+  pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed;
+
+  pTq->tqReaderGetWalReader = tqGetWalReader;  // todo remove it
+  pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock;          // todo remove it
+
+  pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it
+  pTq->tqGetResultBlock = tqGetResultBlock;
+
+  pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
+}
+
+void initStateStoreAPI(SStateStore* pStore) {
+  pStore->streamFileStateInit = streamFileStateInit;
+  pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF;
+
+  pStore->streamStateGetByPos = streamStateGetByPos;
+
+  pStore->streamStatePutParName = streamStatePutParName;
+  pStore->streamStateGetParName = streamStateGetParName;
+
+  pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
+  pStore->streamStateReleaseBuf = streamStateReleaseBuf;
+  pStore->streamStateFreeVal = streamStateFreeVal;
+
+  pStore->streamStatePut = streamStatePut;
+  pStore->streamStateGet = streamStateGet;
+  pStore->streamStateCheck = streamStateCheck;
+  pStore->streamStateGetByPos = streamStateGetByPos;
+  pStore->streamStateDel = streamStateDel;
+  pStore->streamStateClear = streamStateClear;
+  pStore->streamStateSaveInfo = streamStateSaveInfo;
+  pStore->streamStateGetInfo = streamStateGetInfo;
+  pStore->streamStateSetNumber = streamStateSetNumber;
+
+  pStore->streamStateFillPut = streamStateFillPut;
+  pStore->streamStateFillGet = streamStateFillGet;
+  pStore->streamStateFillDel = streamStateFillDel;
+
+  pStore->streamStateCurNext = streamStateCurNext;
+  pStore->streamStateCurPrev = streamStateCurPrev;
+
+  pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur;
+  pStore->streamStateSeekKeyNext = streamStateSeekKeyNext;
+  pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext;
+  pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev;
+  pStore->streamStateFreeCur = streamStateFreeCur;
+
+  pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur;
+  pStore->streamStateGetKVByCur = streamStateGetKVByCur;
+
+  pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
+  pStore->streamStateSessionPut = streamStateSessionPut;
+  pStore->streamStateSessionGet = streamStateSessionGet;
+  pStore->streamStateSessionDel = streamStateSessionDel;
+  pStore->streamStateSessionClear = streamStateSessionClear;
+  pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur;
+  pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist;
+  pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
+
+  pStore->updateInfoInit = updateInfoInit;
+  pStore->updateInfoFillBlockData = updateInfoFillBlockData;
+  pStore->updateInfoIsUpdated = updateInfoIsUpdated;
+  pStore->updateInfoIsTableInserted = updateInfoIsTableInserted;
+  pStore->updateInfoDestroy = updateInfoDestroy;
+
+  pStore->updateInfoInitP = updateInfoInitP;
+  pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF;
+  pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF;
+  pStore->updateInfoSerialize = updateInfoSerialize;
+  pStore->updateInfoDeserialize = updateInfoDeserialize;
+
+  pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
+  pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
+  pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext;
+
+  pStore->streamFileStateInit = streamFileStateInit;
+
+  pStore->streamFileStateDestroy = streamFileStateDestroy;
+  pStore->streamFileStateClear = streamFileStateClear;
+  pStore->needClearDiskBuff = needClearDiskBuff;
+
+  pStore->streamStateOpen = streamStateOpen;
+  pStore->streamStateClose = streamStateClose;
+  pStore->streamStateBegin = streamStateBegin;
+  pStore->streamStateCommit = streamStateCommit;
+  pStore->streamStateDestroy= streamStateDestroy;
+  pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
+}
+
+void initMetaReaderAPI(SStoreMetaReader* pMetaReader) {
+  pMetaReader->initReader = _metaReaderInit;
+  pMetaReader->clearReader = metaReaderClear;
+
+  pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid;
+
+  pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache;
+  pMetaReader->getTableEntryByName = metaGetTableEntryByName;
+
+  pMetaReader->readerReleaseLock = metaReaderReleaseLock;
+}
+
+void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) {
+  pFilter->metaFilterCreateTime = metaFilterCreateTime;
+  pFilter->metaFilterTableIds = metaFilterTableIds;
+  pFilter->metaFilterTableName = metaFilterTableName;
+  pFilter->metaFilterTtl = metaFilterTtl;
+}
+
+void initFunctionStateStore(SFunctionStateStore* pStore) {
+  pStore->streamStateFuncPut = streamStateFuncPut;
+  pStore->streamStateFuncGet = streamStateFuncGet;
+}
+
+void initCacheFn(SStoreCacheReader* pCache) {
+  pCache->openReader = tsdbCacherowsReaderOpen;
+  pCache->closeReader = tsdbCacherowsReaderClose;
+  pCache->retrieveRows = tsdbRetrieveCacheRows;
+  pCache->reuseReader = tsdbReuseCacherowsReader;
+}
+
+void initSnapshotFn(SStoreSnapshotFn* pSnapshot) {
+  pSnapshot->createSnapshot = setForSnapShot;
+  pSnapshot->destroySnapshot = destroySnapContext;
+  pSnapshot->getMetaTableInfoFromSnapshot = getMetaTableInfoFromSnapshot;
+  pSnapshot->getTableInfoFromSnapshot = getTableInfoFromSnapshot;
+}

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -80,7 +80,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
     metaRsp.suid = mer1.me.uid;
   } else if (mer1.me.type == TSDB_CHILD_TABLE) {
     metaReaderInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
-    if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
+    if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
 
     strcpy(metaRsp.stbName, mer2.me.name);
     metaRsp.suid = mer2.me.uid;
@@ -189,7 +189,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
     goto _exit;
   } else if (mer1.me.type == TSDB_CHILD_TABLE) {
     metaReaderInit(&mer2, pVnode->pMeta, 0);
-    if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
+    if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
 
     strcpy(cfgRsp.stbName, mer2.me.name);
     schema = mer2.me.stbEntry.schemaRow;
@@ -410,13 +410,32 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
                             "nBatchInsertSuccess");
 }
 
-void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
+void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables) {
+  SVnode* pVnodeObj = pVnode;
+  SVnodeCfg* pConf = &pVnodeObj->config;
+
   if (dbname) {
-    *dbname = pVnode->config.dbname;
+    *dbname = pConf->dbname;
   }
 
   if (vgId) {
-    *vgId = TD_VID(pVnode);
+    *vgId = TD_VID(pVnodeObj);
+  }
+
+  if (numOfTables) {
+    *numOfTables = pConf->vndStats.numOfNTables + pConf->vndStats.numOfCTables;
+  }
+
+  if (numOfNormalTables) {
+    *numOfNormalTables = pConf->vndStats.numOfNTables;
+  }
+}
+
+int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) {
+  if (type == TSDB_SUPER_TABLE) {
+    return vnodeGetStbIdList(pVnode, 0, pList);
+  } else {
+    return TSDB_CODE_INVALID_PARA;
   }
 }
 
@@ -440,8 +459,10 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
 int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) {
   return 0;
 }
-int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
+
-  SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 1);
+int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
+  SVnode      *pVnodeObj = pVnode;
+  SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj->pMeta, suid, 1);
 
   while (1) {
     tb_uid_t id = metaCtbCursorNext(pCur);
@@ -529,10 +550,8 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
   for (int64_t i = 0; i < arrSize; ++i) {
     tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
 
-    SMetaStbStats stats = {0};
+    int64_t ctbNum = 0;
-    metaGetStbStats(pVnode->pMeta, suid, &stats);
+    metaGetStbStats(pVnode, suid, &ctbNum);
-    int64_t ctbNum = stats.ctbNum;
-    // vnodeGetCtbNum(pVnode, id, &ctbNum);
 
     int numOfCols = 0;
     vnodeGetStbColumnNum(pVnode, suid, &numOfCols);
@@ -567,16 +586,17 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) {
   return TSDB_CODE_SUCCESS;
 }
 
-void *vnodeGetIdx(SVnode *pVnode) {
+void *vnodeGetIdx(void *pVnode) {
   if (pVnode == NULL) {
     return NULL;
   }
-  return metaGetIdx(pVnode->pMeta);
+
+  return metaGetIdx(((SVnode*)pVnode)->pMeta);
 }
 
-void *vnodeGetIvtIdx(SVnode *pVnode) {
+void *vnodeGetIvtIdx(void *pVnode) {
   if (pVnode == NULL) {
     return NULL;
   }
-  return metaGetIvtIdx(pVnode->pMeta);
+  return metaGetIvtIdx(((SVnode*)pVnode)->pMeta);
 }

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -19,23 +19,23 @@
 #include "vnode.h"
 #include "vnodeInt.h"
 
-static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 
 static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime, int64_t *pUid) {
   int32_t code = 0;
@@ -250,7 +250,9 @@ static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   uint8_t    *pCont;
   SEncoder   *pCoder = &(SEncoder){0};
   SDeleteRes  res = {0};
-  SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
+
+  SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
+  initStorageAPI(&handle.api);
 
   code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res);
   if (code) goto _exit;
@@ -301,32 +303,32 @@ _exit:
   return code;
 }
 
-int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) {
+int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg *pRsp) {
   void   *ptr = NULL;
   void   *pReq;
   int32_t len;
   int32_t ret;
 
-  if (version <= pVnode->state.applied) {
+  if (ver <= pVnode->state.applied) {
-    vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version,
+    vError("vgId:%d, duplicate write request. ver: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), ver,
            pVnode->state.applied);
     terrno = TSDB_CODE_VND_DUP_REQUEST;
     return -1;
   }
 
   vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
-         version);
+         ver);
 
   ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm);
-  ASSERT(pVnode->state.applied + 1 == version);
+  ASSERT(pVnode->state.applied + 1 == ver);
 
-  atomic_store_64(&pVnode->state.applied, version);
+  atomic_store_64(&pVnode->state.applied, ver);
   atomic_store_64(&pVnode->state.applyTerm, pMsg->info.conn.applyTerm);
 
   if (!syncUtilUserCommit(pMsg->msgType)) goto _exit;
 
   if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE || pMsg->msgType == TDMT_STREAM_TASK_CHECK_RSP) {
-    if (tqCheckLogInWal(pVnode->pTq, version)) return 0;
+    if (tqCheckLogInWal(pVnode->pTq, ver)) return 0;
   }
 
   // skip header
@@ -337,123 +339,123 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
   switch (pMsg->msgType) {
     /* META */
     case TDMT_VND_CREATE_STB:
-      if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessCreateStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_ALTER_STB:
-      if (vnodeProcessAlterStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessAlterStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_STB:
-      if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessDropStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_CREATE_TABLE:
-      if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessCreateTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_ALTER_TABLE:
-      if (vnodeProcessAlterTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessAlterTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_TABLE:
-      if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessDropTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_TTL_TABLE:
-      if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessDropTtlTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_TRIM:
-      if (vnodeProcessTrimReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_CREATE_SMA:
-      if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     /* TSDB */
     case TDMT_VND_SUBMIT:
-      if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err;
+      if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DELETE:
-      if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_BATCH_DEL:
-      if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
       break;
     /* TQ */
     case TDMT_VND_TMQ_SUBSCRIBE:
-      if (tqProcessSubscribeReq(pVnode->pTq, version, pReq, len) < 0) {
+      if (tqProcessSubscribeReq(pVnode->pTq, ver, pReq, len) < 0) {
         goto _err;
       }
       break;
     case TDMT_VND_TMQ_DELETE_SUB:
-      if (tqProcessDeleteSubReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+      if (tqProcessDeleteSubReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
         goto _err;
       }
       break;
     case TDMT_VND_TMQ_COMMIT_OFFSET:
-      if (tqProcessOffsetCommitReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
+      if (tqProcessOffsetCommitReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
         goto _err;
       }
       break;
     case TDMT_VND_TMQ_SEEK_TO_OFFSET:
-      if (tqProcessSeekReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
+      if (tqProcessSeekReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
         goto _err;
       }
       break;
     case TDMT_VND_TMQ_ADD_CHECKINFO:
-      if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) {
+      if (tqProcessAddCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) {
         goto _err;
       }
       break;
     case TDMT_VND_TMQ_DEL_CHECKINFO:
-      if (tqProcessDelCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) {
+      if (tqProcessDelCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) {
         goto _err;
       }
       break;
     case TDMT_STREAM_TASK_DEPLOY: {
-      if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) {
+      if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, ver, pReq, len) < 0) {
         goto _err;
       }
     } break;
     case TDMT_STREAM_TASK_DROP: {
-      if (tqProcessTaskDropReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+      if (tqProcessTaskDropReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
         goto _err;
       }
     } break;
     case TDMT_STREAM_TASK_PAUSE: {
-      if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+      if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
         goto _err;
       }
     } break;
     case TDMT_STREAM_TASK_RESUME: {
-      if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+      if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
         goto _err;
       }
     } break;
     case TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE: {
-      if (tqProcessTaskRecover2Req(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+      if (tqProcessTaskRecover2Req(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
         goto _err;
       }
     } break;
     case TDMT_STREAM_TASK_CHECK_RSP: {
-      if (tqProcessStreamTaskCheckRsp(pVnode->pTq, version, pReq, len) < 0) {
+      if (tqProcessStreamTaskCheckRsp(pVnode->pTq, ver, pReq, len) < 0) {
         goto _err;
       }
     } break;
     case TDMT_VND_ALTER_CONFIRM:
       needCommit = pVnode->config.hashChange;
-      if (vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp) < 0) {
+      if (vnodeProcessAlterConfirmReq(pVnode, ver, pReq, len, pRsp) < 0) {
         goto _err;
       }
       break;
     case TDMT_VND_ALTER_CONFIG:
-      vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp);
+      vnodeProcessAlterConfigReq(pVnode, ver, pReq, len, pRsp);
       break;
     case TDMT_VND_COMMIT:
       needCommit = true;
       break;
     case TDMT_VND_CREATE_INDEX:
-      vnodeProcessCreateIndexReq(pVnode, version, pReq, len, pRsp);
+      vnodeProcessCreateIndexReq(pVnode, ver, pReq, len, pRsp);
       break;
     case TDMT_VND_DROP_INDEX:
-      vnodeProcessDropIndexReq(pVnode, version, pReq, len, pRsp);
+      vnodeProcessDropIndexReq(pVnode, ver, pReq, len, pRsp);
       break;
     case TDMT_VND_COMPACT:
-      vnodeProcessCompactVnodeReq(pVnode, version, pReq, len, pRsp);
+      vnodeProcessCompactVnodeReq(pVnode, ver, pReq, len, pRsp);
       goto _exit;
     default:
       vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType);
@@ -461,18 +463,18 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
   }
 
   vTrace("vgId:%d, process %s request, code:0x%x index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), pRsp->code,
-         version);
+         ver);
 
-  walApplyVer(pVnode->pWal, version);
+  walApplyVer(pVnode->pWal, ver);
 
-  if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
+  if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) {
     vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;
   }
 
   // commit if need
   if (needCommit) {
-    vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version);
+    vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), ver);
     if (vnodeAsyncCommit(pVnode) < 0) {
       vError("vgId:%d, failed to vnode async commit since %s.", TD_VID(pVnode), tstrerror(terrno));
       goto _err;
@@ -489,8 +491,8 @@ _exit:
   return 0;
 
 _err:
-  vError("vgId:%d, process %s request failed since %s, version:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
+  vError("vgId:%d, process %s request failed since %s, ver:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
-         tstrerror(terrno), version);
+         tstrerror(terrno), ver);
   return -1;
 }
 
@@ -514,7 +516,9 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
     return 0;
   }
 
-  SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
+  SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
+  initStorageAPI(&handle.api);
+
   switch (pMsg->msgType) {
     case TDMT_SCH_QUERY:
     case TDMT_SCH_MERGE_QUERY:
@@ -603,7 +607,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) {
 }
 
 extern int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now);
-static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   int32_t     code = 0;
   SVTrimDbReq trimReq = {0};
 
@@ -624,7 +628,7 @@ _exit:
   return code;
 }
 
-static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SArray *tbUids = taosArrayInit(8, sizeof(int64_t));
   if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY;
 
@@ -650,7 +654,7 @@ end:
   return ret;
 }
 
-static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SVCreateStbReq req = {0};
   SDecoder       coder;
 
@@ -667,7 +671,7 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p
     goto _err;
   }
 
-  if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) {
+  if (metaCreateSTable(pVnode->pMeta, ver, &req) < 0) {
     pRsp->code = terrno;
     goto _err;
   }
@@ -685,7 +689,7 @@ _err:
   return -1;
 }
 
-static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SDecoder           decoder = {0};
   SEncoder           encoder = {0};
   int32_t            rcode = 0;
@@ -742,7 +746,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR
     }
 
     // do create table
-    if (metaCreateTable(pVnode->pMeta, version, pCreateReq, &cRsp.pMeta) < 0) {
+    if (metaCreateTable(pVnode->pMeta, ver, pCreateReq, &cRsp.pMeta) < 0) {
       if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
         cRsp.code = TSDB_CODE_SUCCESS;
       } else {
@@ -790,7 +794,7 @@ _exit:
   return rcode;
 }
 
-static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SVCreateStbReq req = {0};
   SDecoder       dc = {0};
 
@@ -808,7 +812,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR
     return -1;
   }
 
-  if (metaAlterSTable(pVnode->pMeta, version, &req) < 0) {
+  if (metaAlterSTable(pVnode->pMeta, ver, &req) < 0) {
     pRsp->code = terrno;
     tDecoderClear(&dc);
     return -1;
@@ -819,7 +823,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR
   return 0;
 }
 
-static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SVDropStbReq req = {0};
   int32_t      rcode = TSDB_CODE_SUCCESS;
   SDecoder     decoder = {0};
@@ -839,7 +843,7 @@ static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pRe
   // process request
   tbUidList = taosArrayInit(8, sizeof(int64_t));
   if (tbUidList == NULL) goto _exit;
-  if (metaDropSTable(pVnode->pMeta, version, &req, tbUidList) < 0) {
+  if (metaDropSTable(pVnode->pMeta, ver, &req, tbUidList) < 0) {
     rcode = terrno;
     goto _exit;
   }
@@ -862,7 +866,7 @@ _exit:
   return 0;
 }
 
-static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SVAlterTbReq  vAlterTbReq = {0};
   SVAlterTbRsp  vAlterTbRsp = {0};
   SDecoder      dc = {0};
@@ -887,7 +891,7 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pRe
   }
 
   // process
-  if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq, &vMetaRsp) < 0) {
+  if (metaAlterTable(pVnode->pMeta, ver, &vAlterTbReq, &vMetaRsp) < 0) {
     vAlterTbRsp.code = terrno;
     tDecoderClear(&dc);
     rcode = -1;
@@ -912,7 +916,7 @@ _exit:
   return 0;
 }
 
-static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SVDropTbBatchReq req = {0};
   SVDropTbBatchRsp rsp = {0};
   SDecoder         decoder = {0};
@@ -946,7 +950,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq
     tb_uid_t     tbUid = 0;
 
     /* code */
-    ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, &tbUid);
+    ret = metaDropTable(pVnode->pMeta, ver, pDropTbReq, tbUids, &tbUid);
     if (ret < 0) {
       if (pDropTbReq->igNotExists && terrno == TSDB_CODE_TDB_TABLE_NOT_EXIST) {
         dropTbRsp.code = TSDB_CODE_SUCCESS;
@@ -1189,7 +1193,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) {
   return code;
 }
 
-static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   int32_t code = 0;
   terrno = 0;
 
@@ -1251,7 +1255,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
       for (int32_t iRow = 0; iRow < pColData->nVal; iRow++) {
         if (aKey[iRow] < minKey || aKey[iRow] > maxKey || (iRow > 0 && aKey[iRow] <= aKey[iRow - 1])) {
           code = TSDB_CODE_INVALID_MSG;
-          vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version);
+          vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver);
           goto _exit;
         }
       }
@@ -1263,7 +1267,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
       for (int32_t iRow = 0; iRow < nRow; ++iRow) {
         if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) {
           code = TSDB_CODE_INVALID_MSG;
-          vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version);
+          vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver);
           goto _exit;
         }
       }
@@ -1350,7 +1354,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
       SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1);
 
       // create table
-      if (metaCreateTable(pVnode->pMeta, version, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) {
+      if (metaCreateTable(pVnode->pMeta, ver, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) {
         // create table success
 
         if (newTbUids == NULL &&
@@ -1376,7 +1380,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
 
     // insert data
     int32_t affectedRows;
-    code = tsdbInsertTableData(pVnode->pTsdb, version, pSubmitTbData, &affectedRows);
+    code = tsdbInsertTableData(pVnode->pTsdb, ver, pSubmitTbData, &affectedRows);
     if (code) goto _exit;
 
     pSubmitRsp->affectedRows += affectedRows;
@@ -1404,7 +1408,7 @@ _exit:
   atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1);
   if (code == 0) {
     atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1);
-    tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT);
+    tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT);
   }
 
   // clear
@@ -1419,7 +1423,7 @@ _exit:
   return code;
 }
 
-static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SVCreateTSmaReq req = {0};
   SDecoder        coder = {0};
 
@@ -1439,20 +1443,20 @@ static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *
     goto _err;
   }
 
-  if (tdProcessTSmaCreate(pVnode->pSma, version, (const char *)&req) < 0) {
+  if (tdProcessTSmaCreate(pVnode->pSma, ver, (const char *)&req) < 0) {
     if (pRsp) pRsp->code = terrno;
     goto _err;
   }
 
   tDecoderClear(&coder);
   vDebug("vgId:%d, success to create tsma %s:%" PRIi64 " version %" PRIi64 " for table %" PRIi64, TD_VID(pVnode),
-         req.indexName, req.indexUid, version, req.tableUid);
+         req.indexName, req.indexUid, ver, req.tableUid);
   return 0;
 
 _err:
   tDecoderClear(&coder);
   vError("vgId:%d, failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s",
-         TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr());
+         TD_VID(pVnode), req.indexName, req.indexUid, ver, req.tableUid, terrstr());
   return -1;
 }
 
@@ -1468,28 +1472,28 @@ int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) {
   return vnodeProcessCreateTSmaReq(pVnode, 1, pCont, contLen, NULL);
 }
 
-static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t version) {
+static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   vInfo("vgId:%d, trim meta of tables per hash range [%" PRIu32 ", %" PRIu32 "]. apply-index:%" PRId64, TD_VID(pVnode),
-        pVnode->config.hashBegin, pVnode->config.hashEnd, version);
+        pVnode->config.hashBegin, pVnode->config.hashEnd, ver);
 
   // TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd]
 
   return code;
 }
 
-static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   vInfo("vgId:%d, vnode handle msgType:alter-confirm, alter confim msg is processed", TD_VID(pVnode));
   int32_t code = TSDB_CODE_SUCCESS;
   if (!pVnode->config.hashChange) {
     goto _exit;
   }
 
-  code = vnodeConsolidateAlterHashRange(pVnode, version);
+  code = vnodeConsolidateAlterHashRange(pVnode, ver);
   if (code < 0) {
     vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(),
-           version);
+           ver);
     goto _exit;
   }
   pVnode->config.hashChange = false;
@@ -1503,7 +1507,7 @@ _exit:
   return code;
 }
 
-static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   bool walChanged = false;
   bool tsdbChanged = false;
 
@@ -1606,7 +1610,7 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void
   return 0;
 }
 
-static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SBatchDeleteReq deleteReq;
   SDecoder        decoder;
   tDecoderInit(&decoder, pReq, len);
@@ -1626,7 +1630,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void
 
     int64_t uid = mr.me.uid;
 
-    int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs);
+    int32_t code = tsdbDeleteTableData(pVnode->pTsdb, ver, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs);
     if (code < 0) {
       terrno = code;
       vError("vgId:%d, delete error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 ", end ts:%" PRId64,
@@ -1640,7 +1644,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void
   return 0;
 }
 
-static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   int32_t     code = 0;
   SDecoder   *pCoder = &(SDecoder){0};
   SDeleteRes *pRes = &(SDeleteRes){0};
@@ -1661,7 +1665,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq
   ASSERT(taosArrayGetSize(pRes->uidList) == 0 || (pRes->skey != 0 && pRes->ekey != 0));
 
   for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) {
-    code = tsdbDeleteTableData(pVnode->pTsdb, version, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid),
+    code = tsdbDeleteTableData(pVnode->pTsdb, ver, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid),
                                pRes->skey, pRes->ekey);
     if (code) goto _err;
   }
@@ -1682,7 +1686,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq
 _err:
   return code;
 }
-static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SVCreateStbReq req = {0};
   SDecoder       dc = {0};
 
@@ -1698,7 +1702,7 @@ static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void
     tDecoderClear(&dc);
     return -1;
   }
-  if (metaAddIndexToSTable(pVnode->pMeta, version, &req) < 0) {
+  if (metaAddIndexToSTable(pVnode->pMeta, ver, &req) < 0) {
     pRsp->code = terrno;
     goto _err;
   }
@@ -1708,7 +1712,7 @@ _err:
   tDecoderClear(&dc);
   return -1;
 }
-static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   SDropIndexReq req = {0};
   pRsp->msgType = TDMT_VND_DROP_INDEX_RSP;
   pRsp->code = TSDB_CODE_SUCCESS;
@@ -1720,21 +1724,21 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *p
     return -1;
   }
 
-  if (metaDropIndexFromSTable(pVnode->pMeta, version, &req) < 0) {
+  if (metaDropIndexFromSTable(pVnode->pMeta, ver, &req) < 0) {
     pRsp->code = terrno;
     return -1;
   }
   return TSDB_CODE_SUCCESS;
 }
 
-extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 
-static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  return vnodeProcessCompactVnodeReqImpl(pVnode, version, pReq, len, pRsp);
+  return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp);
 }
 
 #ifndef TD_ENTERPRISE
-int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   return 0;
 }
 #endif

source/libs/executor/CMakeLists.txt

@@ -1,10 +1,9 @@
 aux_source_directory(src EXECUTOR_SRC)
-#add_library(executor ${EXECUTOR_SRC})
 
 add_library(executor STATIC ${EXECUTOR_SRC})
 
 target_link_libraries(executor
-        PRIVATE os util common function parser planner qcom vnode scalar nodes index stream
+        PRIVATE os util common function parser planner qcom scalar nodes index wal tdb
         )
 
 target_include_directories(

source/libs/executor/inc/dataSinkInt.h

@@ -22,6 +22,7 @@ extern "C" {
 
 #include "dataSinkMgt.h"
 #include "plannodes.h"
+#include "storageapi.h"
 #include "tcommon.h"
 
 struct SDataSink;
@@ -29,6 +30,7 @@ struct SDataSinkHandle;
 
 typedef struct SDataSinkManager {
   SDataSinkMgtCfg cfg;
+  SStorageAPI*    pAPI;
 } SDataSinkManager;
 
 typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue);

source/libs/executor/inc/executil.h

@@ -12,17 +12,17 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TDENGINE_QUERYUTIL_H
+#ifndef TDENGINE_EXECUTIL_H
-#define TDENGINE_QUERYUTIL_H
+#define TDENGINE_EXECUTIL_H
 
 #include "executor.h"
 #include "function.h"
 #include "nodes.h"
 #include "plannodes.h"
+#include "storageapi.h"
 #include "tcommon.h"
 #include "tpagedbuf.h"
 #include "tsimplehash.h"
-#include "vnode.h"
 
 #define T_LONG_JMP(_obj, _c) \
   do {                       \
@@ -154,7 +154,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
 SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode);
 
 EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
-int32_t  getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId);
+int32_t  getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId, SStorageAPI* pAPI);
 size_t   getTableTagsBufLen(const SNodeList* pGroups);
 
 SArray* createSortInfo(SNodeList* pNodeList);
@@ -166,7 +166,7 @@ void       createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId);
 void       createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode);
 SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs);
 
-SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset);
+SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore);
 void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn);
 void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
 
@@ -178,8 +178,8 @@ void    cleanupQueryTableDataCond(SQueryTableDataCond* pCond);
 
 int32_t convertFillType(int32_t mode);
 int32_t resultrowComparAsc(const void* p1, const void* p2);
-int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified);
+int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI);
 
 void printDataBlock(SSDataBlock* pBlock, const char* flag);
 
-#endif  // TDENGINE_QUERYUTIL_H
+#endif  // TDENGINE_EXECUTIL_H

source/libs/executor/inc/executorInt.h

@@ -38,13 +38,15 @@ extern "C" {
 #include "tlockfree.h"
 #include "tmsg.h"
 #include "tpagedbuf.h"
-#include "tstream.h"
+//#include "tstream.h"
-#include "tstreamUpdate.h"
+//#include "tstreamUpdate.h"
-
+#include "tlrucache.h"
-#include "vnode.h"
 
 typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
 
+typedef struct STsdbReader STsdbReader;
+typedef struct STqReader  STqReader;
+
 #define IS_VALID_SESSION_WIN(winInfo)        ((winInfo).sessionWin.win.skey > 0)
 #define SET_SESSION_WIN_INVALID(winInfo)     ((winInfo).sessionWin.win.skey = INT64_MIN)
 #define IS_INVALID_SESSION_WIN_KEY(winKey)   ((winKey).win.skey <= 0)
@@ -206,6 +208,7 @@ typedef struct STableScanBase {
   SLimitInfo             limitInfo;
   // there are more than one table list exists in one task, if only one vnode exists.
   STableListInfo* pTableListInfo;
+  TsdReader     readerAPI;
 } STableScanBase;
 
 typedef struct STableScanInfo {
@@ -221,6 +224,7 @@ typedef struct STableScanInfo {
   int8_t          assignBlockUid;
   bool            hasGroupByTag;
   bool            countOnly;
+//  TsdReader    readerAPI;
 } STableScanInfo;
 
 typedef struct STableMergeScanInfo {
@@ -280,6 +284,7 @@ typedef struct SStreamAggSupporter {
   int32_t         stateKeySize;
   int16_t         stateKeyType;
   SDiskbasedBuf*  pResultBuf;
+  SStateStore     stateStore;
 } SStreamAggSupporter;
 
 typedef struct SWindowSupporter {
@@ -335,7 +340,7 @@ typedef struct SStreamScanInfo {
   STqReader*   tqReader;
 
   uint64_t     groupId;
-  SUpdateInfo* pUpdateInfo;
+  struct SUpdateInfo* pUpdateInfo;
 
   EStreamScanMode       scanMode;
   struct SOperatorInfo*        pStreamScanOp;
@@ -366,15 +371,18 @@ typedef struct SStreamScanInfo {
   SSDataBlock*  pCreateTbRes;
   int8_t        igCheckUpdate;
   int8_t        igExpired;
-  SStreamState* pState;
+  void*         pState;         //void
+  SStoreTqReader readerFn;
+  SStateStore stateStore;
 } SStreamScanInfo;
 
 typedef struct {
-  SVnode*         vnode;
+  struct SVnode*       vnode;  // todo remove this
-  SSDataBlock     pRes;  // result SSDataBlock
+  SSDataBlock          pRes;   // result SSDataBlock
-  STsdbReader*    dataReader;
+  STsdbReader*         dataReader;
-  SSnapContext*   sContext;
+  struct SSnapContext* sContext;
-  STableListInfo* pTableListInfo;
+  SStorageAPI*         pAPI;
+  STableListInfo*      pTableListInfo;
 } SStreamRawScanInfo;
 
 typedef struct STableCountScanSupp {
@@ -441,12 +449,13 @@ typedef struct SStreamIntervalOperatorInfo {
   bool               isFinal;
   SArray*            pChildren;
   int32_t            numOfChild;
-  SStreamState*      pState;
+  SStreamState*      pState;        // void
   SWinKey            delKey;
   uint64_t           numOfDatapack;
   SArray*            pUpdated;
   SSHashObj*         pUpdatedMap;
   int64_t            dataVersion;
+  SStateStore        statestore;
 } SStreamIntervalOperatorInfo;
 
 typedef struct SDataGroupInfo {
@@ -543,6 +552,7 @@ typedef struct SStreamFillSupporter {
   int32_t        rowSize;
   SSHashObj*     pResMap;
   bool           hasDelete;
+  SStorageAPI*   pAPI;
 } SStreamFillSupporter;
 
 typedef struct SStreamFillOperatorInfo {
@@ -569,12 +579,11 @@ void    cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo);
 void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
 void cleanupBasicInfo(SOptrBasicInfo* pInfo);
 
-int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr);
+int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore);
 void    cleanupExprSupp(SExprSupp* pSup);
 
-
 int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
-                   const char* pkey, void* pState);
+                   const char* pkey, void* pState, SFunctionStateStore* pStore);
 void    cleanupAggSup(SAggSupporter* pAggSup);
 
 void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
@@ -636,7 +645,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap);
 bool functionNeedToExecute(SqlFunctionCtx* pCtx);
 bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
 bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
-bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup);
+bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore);
 void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
                                       uint64_t* pGp, void* pTbName);
 uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
@@ -645,20 +654,17 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos
                            SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
 
 bool    groupbyTbname(SNodeList* pGroupList);
-int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
+int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
                                    SGroupResInfo* pGroupResInfo);
-int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size);
+int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI);
-int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock,
+int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
                                     SExprSupp* pSup, SGroupResInfo* pGroupResInfo);
-int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId,
+int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI);
-                     SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup);
-int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult);
-int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
 void    getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order);
 int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order,
                                int64_t* pData);
-void    appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
+void    appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
-                             SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock);
+                             SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI);
 
 SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag);
 SExprInfo*   createExpr(SNodeList* pNodeList, int32_t* numOfExprs);

source/libs/executor/inc/operator.h

@@ -156,7 +156,7 @@ void           destroyOperator(SOperatorInfo* pOperator);
 
 SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id);
 int32_t        getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder);
-int32_t        stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr);
+int32_t        stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI);
 int32_t        getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList);
 
 #ifdef __cplusplus

source/libs/executor/inc/querytask.h

@@ -56,20 +56,19 @@ typedef struct STaskStopInfo {
 } STaskStopInfo;
 
 typedef struct {
-  STqOffsetVal        currentOffset;  // for tmq
+  STqOffsetVal         currentOffset;  // for tmq
-  SMqMetaRsp          metaRsp;        // for tmq fetching meta
+  SMqMetaRsp           metaRsp;        // for tmq fetching meta
-  int64_t             snapshotVer;
+  int64_t              snapshotVer;
-//  SPackedData         submit;  // todo remove it
+  SSchemaWrapper*      schema;
-  SSchemaWrapper*     schema;
+  char                 tbName[TSDB_TABLE_NAME_LEN];  // this is the current scan table: todo refactor
-  char                tbName[TSDB_TABLE_NAME_LEN];   // this is the current scan table: todo refactor
+  int8_t               recoverStep;
-  int8_t              recoverStep;
+  int8_t               recoverScanFinished;
-  int8_t              recoverScanFinished;
+  SQueryTableDataCond  tableCond;
-  SQueryTableDataCond tableCond;
+  int64_t              fillHistoryVer1;
-  int64_t             fillHistoryVer1;
+  int64_t              fillHistoryVer2;
-  int64_t             fillHistoryVer2;
+  SStreamState*        pState;
-  SStreamState*       pState;
+  int64_t              dataVersion;
-  int64_t             dataVersion;
+  int64_t              checkPointId;
-  int64_t             checkPointId;
 } SStreamTaskInfo;
 
 struct SExecTaskInfo {
@@ -92,10 +91,11 @@ struct SExecTaskInfo {
   SArray*               pResultBlockList;  // result block list
   STaskStopInfo         stopInfo;
   SRWLatch              lock;  // secure the access of STableListInfo
+  SStorageAPI           storageAPI;
 };
 
 void           buildTaskId(uint64_t taskId, uint64_t queryId, char* dst);
-SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model);
+SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI);
 void           doDestroyTask(SExecTaskInfo* pTaskInfo);
 bool           isTaskKilled(SExecTaskInfo* pTaskInfo);
 void           setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode);

source/libs/executor/src/aggregateoperator.c

@@ -21,7 +21,6 @@
 #include "tname.h"
 
 #include "executorInt.h"
-#include "index.h"
 #include "operator.h"
 #include "query.h"
 #include "querytask.h"
@@ -30,6 +29,7 @@
 #include "tglobal.h"
 #include "thash.h"
 #include "ttypes.h"
+#include "index.h"
 
 typedef struct {
   bool    hasAgg;
@@ -84,7 +84,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
   int32_t    code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
-                               pTaskInfo->streamInfo.pState);
+                               pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -95,7 +95,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
     pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr);
   }
 
-  code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr);
+  code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -485,8 +485,8 @@ void cleanupAggSup(SAggSupporter* pAggSup) {
 }
 
 int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
-                   const char* pkey, void* pState) {
+                   const char* pkey, void* pState, SFunctionStateStore* pStore) {
-  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
+  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }

source/libs/executor/src/cachescanoperator.c

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "function.h"
 #include "os.h"
+#include "function.h"
 #include "tname.h"
 
 #include "tdatablock.h"
@@ -27,6 +27,8 @@
 #include "thash.h"
 #include "ttypes.h"
 
+#include "storageapi.h"
+
 typedef struct SCacheRowsScanInfo {
   SSDataBlock*    pRes;
   SReadHandle     readHandle;
@@ -102,9 +104,9 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
     STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
 
     uint64_t suid = tableListGetSuid(pTableListInfo);
-    code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
+    code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
-                                   taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds, suid,
+                                                    taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds,
-                                   &pInfo->pLastrowReader, pTaskInfo->id.str);
+                                                    suid, &pInfo->pLastrowReader, pTaskInfo->id.str);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -124,7 +126,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
   if (pScanNode->scan.pScanPseudoCols != NULL) {
     SExprSupp* p = &pInfo->pseudoExprSup;
     p->pExprInfo = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &p->numOfExprs);
-    p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset);
+    p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore);
   }
 
   setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED,
@@ -172,7 +174,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
       blockDataCleanup(pInfo->pBufferredRes);
       taosArrayClear(pInfo->pUidList);
 
-      int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds,
+      int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds,
                                            pInfo->pDstSlotIds, pInfo->pUidList);
       if (code != TSDB_CODE_SUCCESS) {
         T_LONG_JMP(pTaskInfo->env, code);
@@ -239,7 +241,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
       }
 
       if (NULL == pInfo->pLastrowReader) {
-        code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
+        code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
                                        taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader,
                                        pTaskInfo->id.str);
         if (code != TSDB_CODE_SUCCESS) {
@@ -248,12 +250,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
           continue;
         }
       } else {
-        tsdbReuseCacherowsReader(pInfo->pLastrowReader, pList, num);
+        pInfo->readHandle.api.cacheFn.reuseReader(pInfo->pLastrowReader, pList, num);
       }
 
       taosArrayClear(pInfo->pUidList);
 
-      code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds,
+      code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds,
                                    pInfo->pUidList);
       if (code != TSDB_CODE_SUCCESS) {
         T_LONG_JMP(pTaskInfo->env, code);
@@ -287,7 +289,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
       }
     }
 
-    pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
+    pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader);
     setOperatorCompleted(pOperator);
     return NULL;
   }
@@ -305,7 +307,7 @@ void destroyCacheScanOperator(void* param) {
   tableListDestroy(pInfo->pTableList);
 
   if (pInfo->pLastrowReader != NULL) {
-    pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
+    pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader);
   }
 
   cleanupExprSupp(&pInfo->pseudoExprSup);

source/libs/executor/src/dataInserter.c

@@ -17,6 +17,7 @@
 #include "dataSinkMgt.h"
 #include "executorInt.h"
 #include "planner.h"
+#include "storageapi.h"
 #include "tcompression.h"
 #include "tdatablock.h"
 #include "tglobal.h"
@@ -428,8 +429,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
   inserter->explain = pInserterNode->explain;
 
   int64_t suid = 0;
-  int32_t code =
+  int32_t code = pManager->pAPI->metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid);
-      tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid);
   if (code) {
     destroyDataSinker((SDataSinkHandle*)inserter);
     taosMemoryFree(inserter);

source/libs/executor/src/dataSinkMgt.c

@@ -21,8 +21,9 @@
 static SDataSinkManager gDataSinkManager = {0};
 SDataSinkStat           gDataSinkStat = {0};
 
-int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg) {
+int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI) {
   gDataSinkManager.cfg = *cfg;
+  gDataSinkManager.pAPI = pAPI;
   return 0;  // to avoid compiler eror
 }
 

source/libs/executor/src/eventwindowoperator.c

@@ -92,7 +92,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
   if (pEventWindowNode->window.pExprs != NULL) {
     int32_t    numOfScalarExpr = 0;
     SExprInfo* pScalarExprInfo = createExprInfo(pEventWindowNode->window.pExprs, NULL, &numOfScalarExpr);
-    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
+    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -110,7 +110,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
   initResultSizeInfo(&pOperator->resultInfo, 4096);
 
   code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
-                    pTaskInfo->streamInfo.pState);
+                    pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }

source/libs/executor/src/executil.c

@@ -27,6 +27,7 @@
 #include "executorInt.h"
 #include "querytask.h"
 #include "tcompression.h"
+#include "storageapi.h"
 
 typedef struct tagFilterAssist {
   SHashObj* colHash;
@@ -41,13 +42,13 @@ typedef enum {
 } FilterCondType;
 
 static FilterCondType checkTagCond(SNode* cond);
-static int32_t        optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond);
+static int32_t        optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SStorageAPI* pAPI);
-static int32_t        optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond);
+static int32_t        optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond, SStorageAPI* pStoreAPI);
 
-static int32_t      getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond,
+static int32_t      getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond,
-                                 SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr);
+                                 SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI);
 static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList,
-                                               void* metaHandle);
+                                               void* pVnode, SStorageAPI* pStorageAPI);
 
 static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; }
 static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; }
@@ -262,7 +263,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
 
     STagVal tagVal = {0};
     tagVal.cid = pSColumnNode->colId;
-    const char* p = metaGetTableTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal);
+    const char* p = mr->pAPI->extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal);
     if (p == NULL) {
       res->node.resType.type = TSDB_DATA_TYPE_NULL;
     } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) {
@@ -301,14 +302,14 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) {
+int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI) {
   int32_t     code = TSDB_CODE_SUCCESS;
   SMetaReader mr = {0};
 
-  metaReaderInit(&mr, metaHandle, 0);
+  pAPI->metaReaderFn.initReader(&mr, metaHandle, 0, &pAPI->metaFn);
-  code = metaGetTableEntryByUidCache(&mr, info->uid);
+  code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, info->uid);
   if (TSDB_CODE_SUCCESS != code) {
-    metaReaderClear(&mr);
+    pAPI->metaReaderFn.clearReader(&mr);
     *pQualified = false;
 
     return TSDB_CODE_SUCCESS;
@@ -317,7 +318,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle,
   SNode* pTagCondTmp = nodesCloneNode(pTagCond);
 
   nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr);
-  metaReaderClear(&mr);
+  pAPI->metaReaderFn.clearReader(&mr);
 
   SNode* pNew = NULL;
   code = scalarCalculateConstants(pTagCondTmp, &pNew);
@@ -435,7 +436,6 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) {
   taosMemoryFree(payload);
 }
 
-
 static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_CTX* pContext) {
   char*   payload = NULL;
   int32_t len = 0;
@@ -453,8 +453,8 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C
   taosMemoryFree(payload);
 }
 
-
+int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInfo* pTableListInfo, uint8_t* digest,
-int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, uint8_t *digest) {
+                                   SStorageAPI* pAPI) {
   int32_t      code = TSDB_CODE_SUCCESS;
   SArray*      pBlockList = NULL;
   SSDataBlock* pResBlock = NULL;
@@ -465,7 +465,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
 
   int32_t rows = taosArrayGetSize(pTableListInfo->pTableList);
   if (rows == 0) {
-    return TDB_CODE_SUCCESS;
+    return TSDB_CODE_SUCCESS;
   }
 
   tagFilterAssist ctx = {0};
@@ -492,18 +492,19 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
   if (tsTagFilterCache) {
     SNodeListNode* listNode = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST);
     listNode->pNodeList = group;
-    genTbGroupDigest((SNode *)listNode, digest, &context);
+    genTbGroupDigest((SNode*)listNode, digest, &context);
     nodesFree(listNode);
-    
+
-    metaGetCachedTbGroup(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList);
+    pAPI->metaFn.getCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList);
     if (tableList) {
       taosArrayDestroy(pTableListInfo->pTableList);
       pTableListInfo->pTableList = tableList;
-      qDebug("retrieve tb group list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pTableListInfo->pTableList));
+      qDebug("retrieve tb group list from cache, numOfTables:%d",
+             (int32_t)taosArrayGetSize(pTableListInfo->pTableList));
       goto end;
     }
   }
-  
+
   pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo));
   for (int32_t i = 0; i < rows; ++i) {
     STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i);
@@ -511,14 +512,13 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
     taosArrayPush(pUidTagList, &info);
   }
 
-  //  int64_t stt = taosGetTimestampUs();
+  code = pAPI->metaFn.getTableTags(pVnode, pTableListInfo->idInfo.suid, pUidTagList);
-  code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList);
   if (code != TSDB_CODE_SUCCESS) {
     goto end;
   }
 
   int32_t numOfTables = taosArrayGetSize(pUidTagList);
-  pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle);
+  pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI);
   if (pResBlock == NULL) {
     code = terrno;
     goto end;
@@ -632,9 +632,9 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
 
   if (tsTagFilterCache) {
     tableList = taosArrayDup(pTableListInfo->pTableList, NULL);
-    metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo));
+    pAPI->metaFn.putTableListIntoCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo));
   }
-  
+
   //  int64_t st2 = taosGetTimestampUs();
   //  qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1);
 
@@ -734,12 +734,12 @@ static FilterCondType checkTagCond(SNode* cond) {
   return FILTER_OTHER;
 }
 
-static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond) {
+static int32_t optimizeTbnameInCond(void* pVnode, int64_t suid, SArray* list, SNode* cond, SStorageAPI* pAPI) {
   int32_t ret = -1;
   int32_t ntype = nodeType(cond);
 
   if (ntype == QUERY_NODE_OPERATOR) {
-    ret = optimizeTbnameInCondImpl(metaHandle, list, cond);
+    ret = optimizeTbnameInCondImpl(pVnode, list, cond, pAPI);
   }
 
   if (ntype != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) {
@@ -758,7 +758,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
   SListCell* cell = pList->pHead;
   for (int i = 0; i < len; i++) {
     if (cell == NULL) break;
-    if (optimizeTbnameInCondImpl(metaHandle, list, cell->pNode) == 0) {
+    if (optimizeTbnameInCondImpl(pVnode, list, cell->pNode, pAPI) == 0) {
       hasTbnameCond = true;
       break;
     }
@@ -769,14 +769,14 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
   taosArrayRemoveDuplicate(list, filterTableInfoCompare, NULL);
 
   if (hasTbnameCond) {
-    ret = metaGetTableTagsByUids(metaHandle, suid, list);
+    ret = pAPI->metaFn.getTableTagsByUid(pVnode, suid, list);
   }
 
   return ret;
 }
 
 // only return uid that does not contained in pExistedUidList
-static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) {
+static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, SNode* pTagCond, SStorageAPI* pStoreAPI) {
   if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) {
     return -1;
   }
@@ -813,9 +813,9 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis
       char* name = taosArrayGetP(pTbList, i);
 
       uint64_t uid = 0;
-      if (metaGetTableUidByName(metaHandle, name, &uid) == 0) {
+      if (pStoreAPI->metaFn.getTableUidByName(pVnode, name, &uid) == 0) {
         ETableType tbType = TSDB_TABLE_MAX;
-        if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
+        if (pStoreAPI->metaFn.getTableTypeByName(pVnode, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
           if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) {
             STUidTagInfo s = {.uid = uid, .name = name, .pTagVal = NULL};
             taosArrayPush(pExistedUidList, &s);
@@ -839,9 +839,8 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis
   return -1;
 }
 
-
 static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList,
-                                               void* metaHandle) {
+                                               void* pVnode, SStorageAPI* pStorageAPI) {
   SSDataBlock* pResBlock = createDataBlock();
   if (pResBlock == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -876,7 +875,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
         if (p1->name != NULL) {
           STR_TO_VARSTR(str, p1->name);
         } else {  // name is not retrieved during filter
-          metaGetTableNameByUid(metaHandle, p1->uid, str);
+          pStorageAPI->metaFn.getTableNameByUid(pVnode, p1->uid, str);
         }
 
         colDataSetVal(pColInfo, i, str, false);
@@ -889,7 +888,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
         if (p1->pTagVal == NULL) {
           colDataSetNULL(pColInfo, i);
         } else {
-          const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal);
+          const char* p = pStorageAPI->metaFn.extractTagVal(p1->pTagVal, pColInfo->info.type, &tagVal);
 
           if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
             colDataSetNULL(pColInfo, i);
@@ -949,13 +948,13 @@ static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) {
   }
 }
 
-static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle,
+static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* pVnode,
-                                 SIdxFltStatus status) {
+                                 SIdxFltStatus status, SStorageAPI* pAPI) {
   if (pTagCond == NULL) {
     return TSDB_CODE_SUCCESS;
   }
 
-  terrno = TDB_CODE_SUCCESS;
+  terrno = TSDB_CODE_SUCCESS;
 
   int32_t      code = TSDB_CODE_SUCCESS;
   SArray*      pBlockList = NULL;
@@ -985,7 +984,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
 
   FilterCondType condType = checkTagCond(pTagCond);
 
-  int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond);
+  int32_t filter = optimizeTbnameInCond(pVnode, pListInfo->idInfo.suid, pUidTagList, pTagCond, pAPI);
   if (filter == 0) {  // tbname in filter is activated, do nothing and return
     taosArrayClear(pUidList);
 
@@ -998,9 +997,9 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
     terrno = 0;
   } else {
     if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) {
-      code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList);
+      code = pAPI->metaFn.getTableTagsByUid(pVnode, pListInfo->idInfo.suid, pUidTagList);
     } else {
-      code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList);
+      code = pAPI->metaFn.getTableTags(pVnode, pListInfo->idInfo.suid, pUidTagList);
     }
     if (code != TSDB_CODE_SUCCESS) {
       qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid);
@@ -1014,7 +1013,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
     goto end;
   }
 
-  pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle);
+  pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI);
   if (pResBlock == NULL) {
     code = terrno;
     goto end;
@@ -1052,8 +1051,8 @@ end:
   return code;
 }
 
-int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
+int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
-                     STableListInfo* pListInfo, uint8_t* digest, const char* idstr) {
+                     STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI) {
   int32_t code = TSDB_CODE_SUCCESS;
   size_t  numOfTables = 0;
 
@@ -1065,10 +1064,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
   SIdxFltStatus status = SFLT_NOT_INDEX;
   if (pScanNode->tableType != TSDB_SUPER_TABLE) {
     pListInfo->idInfo.uid = pScanNode->uid;
-    if (metaIsTableExist(metaHandle, pScanNode->uid)) {
+    if (pStorageAPI->metaFn.isTableExisted(pVnode, pScanNode->uid)) {
       taosArrayPush(pUidList, &pScanNode->uid);
     }
-    code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status);
+    code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI);
     if (code != TSDB_CODE_SUCCESS) {
       goto _end;
     }
@@ -1080,7 +1079,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
       genTagFilterDigest(pTagCond, &context);
 
       bool acquired = false;
-      metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList,
+      pStorageAPI->metaFn.getCachedTableList(pVnode, pScanNode->suid, context.digest, tListLen(context.digest), pUidList,
                                 &acquired);
       if (acquired) {
         digest[0] = 1;
@@ -1091,26 +1090,27 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
     }
 
     if (!pTagCond) {  // no tag filter condition exists, let's fetch all tables of this super table
-      vnodeGetCtbIdList(pVnode, pScanNode->suid, pUidList);
+      pStorageAPI->metaFn.getChildTableList(pVnode, pScanNode->suid, pUidList);
     } else {
       // failed to find the result in the cache, let try to calculate the results
       if (pTagIndexCond) {
-        void*         pIndex = tsdbGetIvtIdx(metaHandle);
+        void* pIndex = pStorageAPI->metaFn.getInvertIndex(pVnode);
+
         SIndexMetaArg metaArg = {
-            .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid};
+            .metaEx = pVnode, .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), .ivtIdx = pIndex, .suid = pScanNode->uid};
 
         status = SFLT_NOT_INDEX;
-        code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status);
+        code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status, &pStorageAPI->metaFilter);
         if (code != 0 || status == SFLT_NOT_INDEX) {  // temporarily disable it for performance sake
           qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid);
-          code = TDB_CODE_SUCCESS;
+          code = TSDB_CODE_SUCCESS;
         } else {
           qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList));
         }
       }
     }
 
-    code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status);
+    code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI);
     if (code != TSDB_CODE_SUCCESS) {
       goto _end;
     }
@@ -1127,7 +1127,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
         memcpy(pPayload + sizeof(int32_t), taosArrayGet(pUidList, 0), numOfTables * sizeof(uint64_t));
       }
 
-      metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1);
+//      metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1);
       digest[0] = 1;
       memcpy(digest + 1, context.digest, tListLen(context.digest));
     }
@@ -1164,11 +1164,13 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) {
   return keyLen;
 }
 
-int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) {
+int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId,
+    SStorageAPI* pAPI) {
   SMetaReader mr = {0};
-  metaReaderInit(&mr, pMeta, 0);
+
-  if (metaGetTableEntryByUidCache(&mr, uid) != 0) {  // table not exist
+  pAPI->metaReaderFn.initReader(&mr, pVnode, 0, &pAPI->metaFn);
-    metaReaderClear(&mr);
+  if (pAPI->metaReaderFn.getEntryGetUidCache(&mr, uid) != 0) {  // table not exist
+    pAPI->metaReaderFn.clearReader(&mr);
     return TSDB_CODE_PAR_TABLE_NOT_EXIST;
   }
 
@@ -1187,7 +1189,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode,
       REPLACE_NODE(pNew);
     } else {
       nodesDestroyList(groupNew);
-      metaReaderClear(&mr);
+      pAPI->metaReaderFn.clearReader(&mr);
       return code;
     }
 
@@ -1204,7 +1206,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode,
         if (tTagIsJson(data)) {
           terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
           nodesDestroyList(groupNew);
-          metaReaderClear(&mr);
+          pAPI->metaReaderFn.clearReader(&mr);
           return terrno;
         }
         int32_t len = getJsonValueLen(data);
@@ -1224,7 +1226,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode,
   *pGroupId = calcGroupId(keyBuf, len);
 
   nodesDestroyList(groupNew);
-  metaReaderClear(&mr);
+  pAPI->metaReaderFn.clearReader(&mr);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1514,7 +1516,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  SHashObj *pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+  SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
   for (int32_t i = 0; i < numOfOutput; ++i) {
     const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
     if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) {
@@ -1542,7 +1544,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu
   return TSDB_CODE_SUCCESS;
 }
 
-SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset) {
+SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore) {
   SqlFunctionCtx* pFuncCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));
   if (pFuncCtx == NULL) {
     return NULL;
@@ -1605,6 +1607,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
     pCtx->numOfParams = pExpr->base.numOfParams;
     pCtx->param = pFunct->pParam;
     pCtx->saveHandle.currentPage = -1;
+    pCtx->pStore = pStore;
   }
 
   for (int32_t i = 1; i < numOfOutput; ++i) {
@@ -1841,9 +1844,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) {
   return taosArrayGetSize(pTableList->pTableList);
 }
 
-uint64_t tableListGetSuid(const STableListInfo* pTableList) {
+uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->idInfo.suid; }
-  return pTableList->idInfo.suid;
-}
 
 STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) {
   if (taosArrayGetSize(pTableList->pTableList) == 0) {
@@ -2030,11 +2031,11 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
 
   memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups);
   taosArrayDestroy(pList);
-  return TDB_CODE_SUCCESS;
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group,
-                                    bool groupSort, uint8_t *digest) {
+                                    bool groupSort, uint8_t *digest, SStorageAPI* pAPI) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   bool   groupByTbname = groupbyTbname(group);
@@ -2054,7 +2055,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
       pTableListInfo->numOfOuputGroups = 1;
     }
   } else {
-    code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest);
+    code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -2086,7 +2087,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
   }
 
   uint8_t digest[17] = {0};
-  int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr);
+  int32_t code = getTableList(pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr, &pTaskInfo->storageAPI);
   if (code != TSDB_CODE_SUCCESS) {
     qError("failed to getTableList, code: %s", tstrerror(code));
     return code;
@@ -2104,7 +2105,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
     return TSDB_CODE_SUCCESS;
   }
 
-  code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest);
+  code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest, &pTaskInfo->storageAPI);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }

source/libs/executor/src/executor.c

@@ -14,6 +14,8 @@
  */
 
 #include "executor.h"
+#include <libs/transport/trpc.h>
+#include <libs/wal/wal.h>
 #include "executorInt.h"
 #include "operator.h"
 #include "planner.h"
@@ -21,7 +23,8 @@
 #include "tdatablock.h"
 #include "tref.h"
 #include "tudf.h"
-#include "vnode.h"
+
+#include "storageapi.h"
 
 static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT;
 int32_t             exchangeObjRefPool = -1;
@@ -156,18 +159,18 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
   }
 }
 
-void doSetTaskId(SOperatorInfo* pOperator) {
+void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) {
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
   if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     SStreamScanInfo* pStreamScanInfo = pOperator->info;
     if (pStreamScanInfo->pTableScanOp != NULL) {
       STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info;
       if (pScanInfo->base.dataReader != NULL) {
-        tsdbReaderSetId(pScanInfo->base.dataReader, pTaskInfo->id.str);
+        pAPI->tsdReader.tsdSetReaderTaskId(pScanInfo->base.dataReader, pTaskInfo->id.str);
       }
     }
   } else {
-    doSetTaskId(pOperator->pDownstream[0]);
+    doSetTaskId(pOperator->pDownstream[0], pAPI);
   }
 }
 
@@ -177,7 +180,7 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) {
   buildTaskId(taskId, queryId, pTaskInfo->id.str);
 
   // set the idstr for tsdbReader
-  doSetTaskId(pTaskInfo->pRoot);
+  doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI);
 }
 
 int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
@@ -249,7 +252,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
 qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols,
                                      uint64_t id) {
   if (msg == NULL) {  // create raw scan
-    SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE);
+    SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, &pReaderHandle->api);
     if (NULL == pTaskInfo) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
@@ -261,6 +264,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3
       return NULL;
     }
 
+    pTaskInfo->storageAPI = pReaderHandle->api;
     qDebug("create raw scan task info completed, vgId:%d, %s", vgId, GET_TASKID(pTaskInfo));
     return pTaskInfo;
   }
@@ -320,7 +324,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v
   return pTaskInfo;
 }
 
-static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) {
+static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr,
+    SStorageAPI* pAPI) {
   SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
   int32_t numOfUids = taosArrayGetSize(tableIdList);
   if (numOfUids == 0) {
@@ -336,11 +341,11 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
 
   // let's discard the tables those are not created according to the queried super table.
   SMetaReader mr = {0};
-  metaReaderInit(&mr, pScanInfo->readHandle.meta, 0);
+  pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, 0, &pAPI->metaFn);
   for (int32_t i = 0; i < numOfUids; ++i) {
     uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i);
 
-    int32_t code = metaGetTableEntryByUid(&mr, *id);
+    int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, *id);
     if (code != TSDB_CODE_SUCCESS) {
       qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr);
       continue;
@@ -368,7 +373,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
     if (pScanInfo->pTagCond != NULL) {
       bool          qualified = false;
       STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid};
-      code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified);
+      code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.vnode, &qualified, pAPI);
       if (code != TSDB_CODE_SUCCESS) {
         qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr);
         continue;
@@ -383,7 +388,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
     taosArrayPush(qa, id);
   }
 
-  metaReaderClear(&mr);
+  pAPI->metaReaderFn.clearReader(&mr);
   return qa;
 }
 
@@ -401,10 +406,10 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
   SStreamScanInfo* pScanInfo = pInfo->info;
 
   if (isAdd) {  // add new table id
-    SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id);
+    SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id, &pTaskInfo->storageAPI);
     int32_t numOfQualifiedTables = taosArrayGetSize(qa);
     qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id);
-    code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
+    code = pTaskInfo->storageAPI.tqReaderFn.tqReaderAddTables(pScanInfo->tqReader, qa);
     if (code != TSDB_CODE_SUCCESS) {
       taosArrayDestroy(qa);
       return code;
@@ -433,8 +438,8 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
         if (assignUid) {
           keyInfo.groupId = keyInfo.uid;
         } else {
-          code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf,
+          code = getGroupIdFromTagsVal(pScanInfo->readHandle.vnode, keyInfo.uid, pScanInfo->pGroupTags, keyBuf,
-                                       &keyInfo.groupId);
+                                       &keyInfo.groupId, &pTaskInfo->storageAPI);
           if (code != TSDB_CODE_SUCCESS) {
             taosMemoryFree(keyBuf);
             taosArrayDestroy(qa);
@@ -456,7 +461,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
   } else {  // remove the table id in current list
     qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id);
     taosWLockLatch(&pTaskInfo->lock);
-    code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList);
+    code = pTaskInfo->storageAPI.tqReaderFn.tqReaderRemoveTables(pScanInfo->tqReader, tableIdList);
     taosWUnLockLatch(&pTaskInfo->lock);
   }
 
@@ -502,7 +507,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
   }
 
   SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50};
-  code = dsDataSinkMgtInit(&cfg);
+  code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI);
   if (code != TSDB_CODE_SUCCESS) {
     qError("failed to dsDataSinkMgtInit, code:%s, %s", tstrerror(code), (*pTask)->id.str);
     goto _error;
@@ -1060,6 +1065,8 @@ void qStreamSetOpen(qTaskInfo_t tinfo) {
 
 int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
   SOperatorInfo* pOperator = pTaskInfo->pRoot;
   const char*    id = GET_TASKID(pTaskInfo);
 
@@ -1081,12 +1088,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
 
     if (pOffset->type == TMQ_OFFSET__LOG) {
       // todo refactor: move away
-      tsdbReaderClose(pScanBaseInfo->dataReader);
+      pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pScanBaseInfo->dataReader);
       pScanBaseInfo->dataReader = NULL;
 
-      walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset);
+      SStoreTqReader* pReaderAPI = &pTaskInfo->storageAPI.tqReaderFn;
-      if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) {
+      SWalReader* pWalReader = pReaderAPI->tqReaderGetWalReader(pInfo->tqReader);
-        qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
+      walReaderVerifyOffset(pWalReader, pOffset);
+      if (pReaderAPI->tqReaderSeek(pInfo->tqReader, pOffset->version + 1, id) < 0) {
+        qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
         return -1;
       }
     } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
@@ -1141,8 +1150,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       pScanInfo->scanTimes = 0;
 
       if (pScanBaseInfo->dataReader == NULL) {
-        int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1,
+        int32_t code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1,
-                                      pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false, NULL);
+                                      pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL);
         if (code != TSDB_CODE_SUCCESS) {
           qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id);
           terrno = code;
@@ -1152,8 +1161,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
         qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s",
                uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
       } else {
-        tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
+        pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
-        tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
+        pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
         qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 "  table index:%d numOfTable:%d, %s",
                uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
       }
@@ -1175,14 +1184,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       SOperatorInfo*  p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id);
       STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo;
 
-      if (setForSnapShot(sContext, pOffset->uid) != 0) {
+      if (pAPI->snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) {
         qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id);
         terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
         return -1;
       }
 
-      SMetaTableInfo mtInfo = getUidfromSnapShot(sContext);
+      SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getMetaTableInfoFromSnapshot(sContext);
-      tsdbReaderClose(pInfo->dataReader);
+      pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader);
       pInfo->dataReader = NULL;
 
       cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
@@ -1200,7 +1209,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
       int32_t        size = tableListGetSize(pTableListInfo);
 
-      tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL,
+      pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL,
                      false, NULL);
 
       cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
@@ -1212,7 +1221,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
       SStreamRawScanInfo* pInfo = pOperator->info;
       SSnapContext*       sContext = pInfo->sContext;
-      if (setForSnapShot(sContext, pOffset->uid) != 0) {
+      if (pTaskInfo->storageAPI.snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) {
         qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
         terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
         return -1;
@@ -1221,7 +1230,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
              id);
     } else if (pOffset->type == TMQ_OFFSET__LOG) {
       SStreamRawScanInfo* pInfo = pOperator->info;
-      tsdbReaderClose(pInfo->dataReader);
+      pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader);
       pInfo->dataReader = NULL;
       qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id);
     }

source/libs/executor/src/executorInt.c

@@ -33,7 +33,7 @@
 #include "tcompare.h"
 #include "thash.h"
 #include "ttypes.h"
-#include "vnode.h"
+#include "storageapi.h"
 
 #define SET_REVERSE_SCAN_FLAG(runtime)    ((runtime)->scanFlag = REVERSE_SCAN)
 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
@@ -844,6 +844,8 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
 void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
                            SDiskbasedBuf* pBuf) {
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
   SSDataBlock*   pBlock = pbInfo->pRes;
 
   // set output datablock version
@@ -860,12 +862,13 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr
   doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false);
 
   void* tbname = NULL;
-  if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
+  if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
     pBlock->info.parTbName[0] = 0;
   } else {
     memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
   }
-  streamFreeVal(tbname);
+
+  pAPI->stateStore.streamStateFreeVal(tbname);
 }
 
 void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
@@ -976,11 +979,11 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   return NULL;
 }
 
-int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr) {
+int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore) {
   pSup->pExprInfo = pExprInfo;
   pSup->numOfExprs = numOfExpr;
   if (pSup->pExprInfo != NULL) {
-    pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset);
+    pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset, pStore);
     if (pSup->pCtx == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
@@ -1066,122 +1069,22 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo*
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t resultRowEncode(void* k, int32_t* size, char* buf) {
+int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI) {
-  // SResultRow* key = k;
+  pAPI->streamStateReleaseBuf(pState, pKey, pResult);
-  // int         len = 0;
-  // int         struLen = *size;
-  // len += taosEncodeFixedI32((void**)&buf, key->pageId);
-
-  // uint32_t offset = key->offset;
-  // len += taosEncodeFixedU32((void**)&buf, offset);
-
-  // len += taosEncodeFixedI8((void**)&buf, key->startInterp);
-  // len += taosEncodeFixedI8((void**)&buf, key->endInterp);
-  // len += taosEncodeFixedI8((void**)&buf, key->closed);
-  // len += taosEncodeFixedU32((void**)&buf, key->numOfRows);
-
-  // len += taosEncodeFixedI64((void**)&buf, key->win.skey);
-  // len += taosEncodeFixedI64((void**)&buf, key->win.ekey);
-
-  // int32_t numOfEntryInfo = (struLen - sizeof(SResultRow)) / sizeof(struct SResultRowEntryInfo);
-  // len += taosEncodeFixedI32((void**)&buf, numOfEntryInfo);
-  // for (int i = 0; i < numOfEntryInfo; i++) {
-  //   SResultRowEntryInfo* p = &key->pEntryInfo[i];
-
-  //   uint8_t value = p->initialized ? 1 : 0;
-  //   len += taosEncodeFixedU8((void**)&buf, value);
-
-  //   value = p->complete ? 1 : 0;
-  //   len += taosEncodeFixedU8((void**)&buf, value);
-
-  //   value = p->isNullRes;
-  //   len += taosEncodeFixedU8((void**)&buf, value);
-
-  //   len += taosEncodeFixedU16((void**)&buf, p->numOfRes);
-  // }
-  // {
-  //   char* strBuf = taosMemoryCalloc(1, *size * 100);
-  //   resultRowToString(key, *size, strBuf);
-  //   qWarn("encode result row:%s", strBuf);
-  // }
-
-  // return len;
-  return 0;
-}
-
-int32_t resultRowDecode(void** k, size_t size, char* buf) {
-  // char*    p1 = buf;
-  // int32_t  numOfEntryInfo = 0;
-  // uint32_t entryOffset = sizeof(int32_t) + sizeof(uint32_t) + sizeof(int8_t) + sizeof(int8_t) + sizeof(int8_t) +
-  //                        sizeof(uint32_t) + sizeof(int64_t) + sizeof(int64_t);
-  // taosDecodeFixedI32(p1 + entryOffset, &numOfEntryInfo);
-
-  // char* p = buf;
-  // size = sizeof(SResultRow) + numOfEntryInfo * sizeof(SResultRowEntryInfo);
-  // SResultRow* key = taosMemoryCalloc(1, size);
-
-  // p = taosDecodeFixedI32(p, (int32_t*)&key->pageId);
-  // uint32_t offset = 0;
-  // p = taosDecodeFixedU32(p, &offset);
-  // key->offset = offset;
-
-  // p = taosDecodeFixedI8(p, (int8_t*)(&key->startInterp));
-  // p = taosDecodeFixedI8(p, (int8_t*)(&key->endInterp));
-  // p = taosDecodeFixedI8(p, (int8_t*)&key->closed);
-  // p = taosDecodeFixedU32(p, &key->numOfRows);
-
-  // p = taosDecodeFixedI64(p, &key->win.skey);
-  // p = taosDecodeFixedI64(p, &key->win.ekey);
-  // p = taosDecodeFixedI32(p, &numOfEntryInfo);
-  // for (int i = 0; i < numOfEntryInfo; i++) {
-  //   SResultRowEntryInfo* pInfo = &key->pEntryInfo[i];
-  //   uint8_t              value = 0;
-  //   p = taosDecodeFixedU8(p, &value);
-  //   pInfo->initialized = (value == 1) ? true : false;
-
-  //   p = taosDecodeFixedU8(p, &value);
-  //   pInfo->complete = (value == 1) ? true : false;
-
-  //   p = taosDecodeFixedU8(p, &value);
-  //   pInfo->isNullRes = value;
-
-  //   p = taosDecodeFixedU16(p, &pInfo->numOfRes);
-  // }
-  // *k = key;
-
-  // {
-  //   char* strBuf = taosMemoryCalloc(1, size * 100);
-  //   resultRowToString(key, size, strBuf);
-  //   qWarn("decode result row:%s", strBuf);
-  // }
-  // return size;
-  return 0;
-}
-
-int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize) {
-  // char* buf = taosMemoryCalloc(1, resSize * 10);
-  // int   len = resultRowEncode((void*)pResult, &resSize, buf);
-  // char* buf = taosMemoryCalloc(1, resSize);
-  // memcpy(buf, pResult, resSize);
-  streamStatePut(pState, pKey, (char*)pResult, resSize);
-  // taosMemoryFree(buf);
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult) {
+int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI) {
-  streamStateReleaseBuf(pState, pKey, pResult);
+  pAPI->streamStateSessionPut(pState, key, (const void*)buf, size);
+  releaseOutputBuf(pState, NULL, (SResultRow*)buf, pAPI);
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size) {
+int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
-  streamStateSessionPut(pState, key, (const void*)buf, size);
-  releaseOutputBuf(pState, NULL, (SResultRow*)buf);
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock,
                                     SExprSupp* pSup, SGroupResInfo* pGroupResInfo) {
   SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
   SExprInfo*      pExprInfo = pSup->pExprInfo;
   int32_t         numOfExprs = pSup->numOfExprs;
   int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;
@@ -1193,7 +1096,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
     SSessionKey* pKey = taosArrayGet(pGroupResInfo->pRows, i);
     int32_t      size = 0;
     void*        pVal = NULL;
-    int32_t      code = streamStateSessionGet(pState, pKey, &pVal, &size);
+    int32_t      code = pAPI->stateStore.streamStateSessionGet(pState, pKey, &pVal, &size);
     ASSERT(code == 0);
     if (code == -1) {
       // coverity scan
@@ -1205,7 +1108,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
     // no results, continue to check the next one
     if (pRow->numOfRows == 0) {
       pGroupResInfo->index += 1;
-      releaseOutputBuf(pState, NULL, pRow);
+      releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
       continue;
     }
 
@@ -1213,23 +1116,23 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
       pBlock->info.id.groupId = pKey->groupId;
 
       void* tbname = NULL;
-      if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
+      if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
         pBlock->info.parTbName[0] = 0;
       } else {
         memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
       }
-      streamFreeVal(tbname);
+      pAPI->stateStore.streamStateFreeVal(tbname);
     } else {
       // current value belongs to different group, it can't be packed into one datablock
       if (pBlock->info.id.groupId != pKey->groupId) {
-        releaseOutputBuf(pState, NULL, pRow);
+        releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
         break;
       }
     }
 
     if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
       ASSERT(pBlock->info.rows > 0);
-      releaseOutputBuf(pState, NULL, pRow);
+      releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
       break;
     }
 
@@ -1260,7 +1163,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
 
     pBlock->info.dataLoad = 1;
     pBlock->info.rows += pRow->numOfRows;
-    releaseOutputBuf(pState, NULL, pRow);
+    releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
   }
   blockDataUpdateTsWindow(pBlock, 0);
   return TSDB_CODE_SUCCESS;
@@ -1292,7 +1195,7 @@ void qStreamCloseTsdbReader(void* task) {
           qDebug("wait for the reader stopping");
         }
 
-        tsdbReaderClose(pTSInfo->base.dataReader);
+        pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pTSInfo->base.dataReader);
         pTSInfo->base.dataReader = NULL;
 
         // restore the status, todo refactor.

source/libs/executor/src/filloperator.c

@@ -338,7 +338,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
     goto _error;
   }
 
-  code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs);
+  code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -355,7 +355,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
 
   initResultSizeInfo(&pOperator->resultInfo, 4096);
   blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
-  code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr);
+  code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -443,7 +443,7 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
   pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols);
   tSimpleHashCleanup(pFillSup->pResMap);
   pFillSup->pResMap = NULL;
-  releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal);
+  releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pFillSup->pAPI->stateStore);   //?????
   pFillSup->cur.pRowVal = NULL;
   cleanupExprSupp(&pFillSup->notFillExprSup);
 
@@ -490,74 +490,78 @@ static void resetFillWindow(SResultRowData* pRowData) {
   pRowData->pRowVal = NULL;
 }
 
-void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) {
+void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) {
   resetFillWindow(&pFillSup->prev);
-  releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal);
+  releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pAPI->stateStore);   //???
   resetFillWindow(&pFillSup->cur);
   resetFillWindow(&pFillSup->next);
   resetFillWindow(&pFillSup->nextNext);
 }
 
 void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
-  SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState;
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
-  resetPrevAndNextWindow(pFillSup, pState);
+
+  void* pState = pOperator->pTaskInfo->streamInfo.pState;
+  resetPrevAndNextWindow(pFillSup, pState, pAPI);
 
   SWinKey key = {.ts = ts, .groupId = groupId};
   int32_t curVLen = 0;
-  int32_t code = streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen);
+
+  int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen);
   ASSERT(code == TSDB_CODE_SUCCESS);
   pFillSup->cur.key = key.ts;
 }
 
 void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
-  SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState;
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
-  resetPrevAndNextWindow(pFillSup, pState);
+  void* pState = pOperator->pTaskInfo->streamInfo.pState;
+  resetPrevAndNextWindow(pFillSup, pState, pAPI);
 
   SWinKey key = {.ts = ts, .groupId = groupId};
   void*   curVal = NULL;
   int32_t curVLen = 0;
-  int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen);
+  int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen);
   ASSERT(code == TSDB_CODE_SUCCESS);
   pFillSup->cur.key = key.ts;
   pFillSup->cur.pRowVal = curVal;
 
-  SStreamStateCur* pCur = streamStateFillSeekKeyPrev(pState, &key);
+  SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key);
   SWinKey          preKey = {.ts = INT64_MIN, .groupId = groupId};
   void*            preVal = NULL;
   int32_t          preVLen = 0;
-  code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen);
+  code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen);
 
   if (code == TSDB_CODE_SUCCESS) {
     pFillSup->prev.key = preKey.ts;
     pFillSup->prev.pRowVal = preVal;
 
-    code = streamStateCurNext(pState, pCur);
+    code = pAPI->stateStore.streamStateCurNext(pState, pCur);
     ASSERT(code == TSDB_CODE_SUCCESS);
 
-    code = streamStateCurNext(pState, pCur);
+    code = pAPI->stateStore.streamStateCurNext(pState, pCur);
     if (code != TSDB_CODE_SUCCESS) {
-      streamStateFreeCur(pCur);
+      pAPI->stateStore.streamStateFreeCur(pCur);
       pCur = NULL;
     }
   } else {
-    streamStateFreeCur(pCur);
+    pAPI->stateStore.streamStateFreeCur(pCur);
-    pCur = streamStateFillSeekKeyNext(pState, &key);
+    pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key);
   }
 
   SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId};
   void*   nextVal = NULL;
   int32_t nextVLen = 0;
-  code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen);
+  code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen);
   if (code == TSDB_CODE_SUCCESS) {
     pFillSup->next.key = nextKey.ts;
     pFillSup->next.pRowVal = nextVal;
     if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) {
-      code = streamStateCurNext(pState, pCur);
+      code = pAPI->stateStore.streamStateCurNext(pState, pCur);
       if (code == TSDB_CODE_SUCCESS) {
         SWinKey nextNextKey = {.groupId = groupId};
         void*   nextNextVal = NULL;
         int32_t nextNextVLen = 0;
-        code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen);
+        code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen);
         if (code == TSDB_CODE_SUCCESS) {
           pFillSup->nextNext.key = nextNextKey.ts;
           pFillSup->nextNext.pRowVal = nextNextVal;
@@ -565,7 +569,7 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId,
       }
     }
   }
-  streamStateFreeCur(pCur);
+  pAPI->stateStore.streamStateFreeCur(pCur);
 }
 
 static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; }
@@ -922,8 +926,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
 }
 
 static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   SWinKey key = {.groupId = groupId, .ts = pRow->key};
-  int32_t code = streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len);
+  int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len);
   qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 "  code:%d", key.ts, key.groupId, code);
   ASSERT(code == TSDB_CODE_SUCCESS);
 }
@@ -1021,7 +1027,8 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) {
 }
 
 static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) {
-  SStreamState* pState = pOp->pTaskInfo->streamInfo.pState;
+  SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;
+  void* pState = pOp->pTaskInfo->streamInfo.pState;
 
   SSDataBlock*     pBlock = delRes;
   SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
@@ -1041,14 +1048,14 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_
   SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
 
   void* tbname = NULL;
-  streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname);
+  pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname);
   if (tbname == NULL) {
     colDataSetNULL(pTableCol, pBlock->info.rows);
   } else {
     char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
     STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
     colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
-    streamFreeVal(tbname);
+    pAPI->stateStore.streamStateFreeVal(tbname);
   }
 
   pBlock->info.rows++;
@@ -1070,12 +1077,13 @@ static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY end
 }
 
 static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
   SStreamFillOperatorInfo* pInfo = pOperator->info;
   getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup);
   setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo);
   SWinKey key = {.ts = startTs, .groupId = groupId};
   if (!pInfo->pFillInfo->needFill) {
-    streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
+    pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
     buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
   } else {
     STimeRange tw = {
@@ -1093,6 +1101,8 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE
 }
 
 static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   SStreamFillOperatorInfo* pInfo = pOperator->info;
   SStreamFillInfo*         pFillInfo = pInfo->pFillInfo;
   int32_t                  size = taosArrayGetSize(pFillInfo->delRanges);
@@ -1109,17 +1119,16 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
       pInfo->pRes->info.id.groupId = range->groupId;
     }
     SWinKey key = {.ts = range->skey, .groupId = range->groupId};
-    streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
+    pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
   }
 }
 
 static void doDeleteFillResult(SOperatorInfo* pOperator) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   SStreamFillOperatorInfo* pInfo = pOperator->info;
-  SStreamFillSupporter*    pFillSup = pInfo->pFillSup;
   SStreamFillInfo*         pFillInfo = pInfo->pFillInfo;
   SSDataBlock*             pBlock = pInfo->pSrcDelBlock;
-  SSDataBlock*             pRes = pInfo->pRes;
-  SSDataBlock*             pDelRes = pInfo->pDelRes;
 
   SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
   TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
@@ -1130,7 +1139,8 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
     TSKEY            endTs = ts;
     uint64_t         groupId = groupIds[pInfo->srcDelRowIndex];
     SWinKey          key = {.ts = ts, .groupId = groupId};
-    SStreamStateCur* pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key);
+    SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key);
+
     if (!pCur) {
       pInfo->srcDelRowIndex++;
       continue;
@@ -1152,15 +1162,15 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
 
       SWinKey delKey = {.groupId = delGroupId, .ts = delTs};
       if (delTs == nextKey.ts) {
-        code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
+        code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
         if (code == TSDB_CODE_SUCCESS) {
-          code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen);
+          code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen);
         }
         // ts will be deleted later
         if (delTs != ts) {
-          streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey);
+          pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey);
-          streamStateFreeCur(pCur);
+          pAPI->stateStore.streamStateFreeCur(pCur);
-          pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
+          pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
         }
         endTs = TMAX(delTs, nextKey.ts - 1);
         if (code != TSDB_CODE_SUCCESS) {
@@ -1169,9 +1179,11 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
       }
       pInfo->srcDelRowIndex++;
     }
-    streamStateFreeCur(pCur);
+
+    pAPI->stateStore.streamStateFreeCur(pCur);
     doDeleteFillResultImpl(pOperator, ts, endTs, groupId);
   }
+
   pFillInfo->current = pFillInfo->end + 1;
 }
 
@@ -1322,7 +1334,7 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) {
 }
 
 static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval,
-                                               SExprInfo* pFillExprInfo, int32_t numOfFillCols) {
+                                               SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) {
   SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter));
   if (!pFillSup) {
     return NULL;
@@ -1335,6 +1347,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
   pFillSup->type = convertFillType(pPhyFillNode->mode);
   pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols;
   pFillSup->interval = *pInterval;
+  pFillSup->pAPI = pAPI;
 
   int32_t code = initResultBuf(pFillSup);
   if (code != TSDB_CODE_SUCCESS) {
@@ -1343,7 +1356,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
   }
 
   SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
-  code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols);
+  code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     destroyStreamFillSupporter(pFillSup);
     return NULL;
@@ -1414,7 +1427,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
   SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval;
   int32_t    numOfFillCols = 0;
   SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols);
-  pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols);
+  pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI);
   if (!pInfo->pFillSup) {
     goto _error;
   }
@@ -1479,7 +1492,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
     goto _error;
   }
 
-  code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
+  code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }

source/libs/executor/src/groupoperator.c

@@ -451,7 +451,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
   }
 
   pInfo->pGroupCols = extractColumnInfo(pAggNode->pGroupKeys);
-  code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
+  code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -467,7 +467,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
   code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str,
-                    pTaskInfo->streamInfo.pState);
+                    pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -849,7 +849,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
   if (pPartNode->pExprs != NULL) {
     int32_t    num = 0;
     SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num);
-    int32_t    code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num);
+    int32_t    code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       terrno = code;
       pTaskInfo->code = terrno;
@@ -951,6 +951,8 @@ static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo
 static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; }
 
 static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   SStreamPartitionOperatorInfo* pInfo = pOperator->info;
   SSDataBlock*                  pDest = pInfo->binfo.pRes;
   ASSERT(hasRemainPartion(pInfo));
@@ -973,9 +975,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
   pDest->info.parTbName[0] = 0;
   if (pInfo->tbnameCalSup.numOfExprs > 0) {
     void* tbname = NULL;
-    if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) {
+    if (pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) {
       memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
-      streamFreeVal(tbname);
+      pAPI->stateStore.streamStateFreeVal(tbname);
     }
   }
   taosArrayDestroy(pParInfo->rowIds);
@@ -991,10 +993,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
   return pDest;
 }
 
-void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
+void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
-                          SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) {
+                          SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) {
   void* pValue = NULL;
-  if (streamStateGetParName(pState, groupId, &pValue) != 0) {
+  if (pAPI->streamStateGetParName(pState, groupId, &pValue) != 0) {
     SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId);
     memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN);
     pTmpBlock->info.id.groupId = groupId;
@@ -1011,7 +1013,7 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
         void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1);
         len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1);
         memcpy(tbName, varDataVal(pData), len);
-        streamStatePutParName(pState, groupId, tbName);
+        pAPI->streamStatePutParName(pState, groupId, tbName);
       }
       memcpy(pTmpBlock->info.parTbName, tbName, len);
       pDestBlock->info.rows--;
@@ -1035,10 +1037,12 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
   } else {
     memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN);
   }
-  streamStateReleaseBuf(pState, NULL, pValue);
+  pAPI->streamStateReleaseBuf(pState, NULL, pValue);
 }
 
 static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) {
+  SExecTaskInfo* pTask = pOperator->pTaskInfo;
+
   SStreamPartitionOperatorInfo* pInfo = pOperator->info;
   if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) ||
       taosHashGetSize(pInfo->pPartitions) == 0) {
@@ -1051,8 +1055,8 @@ static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) {
   if (pInfo->pTbNameIte != NULL) {
     SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte;
     int32_t             rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0);
-    appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup,
+    appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup,
-                         pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes);
+                         pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore);
     pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte);
   }
   return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL;
@@ -1165,14 +1169,17 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
 }
 
 void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) {
+  SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI;
+
   if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     return;
   }
+
   SStreamScanInfo* pScanInfo = downstream->info;
   pScanInfo->partitionSup = *pParSup;
   pScanInfo->pPartScalarSup = pExpr;
   if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) {
-    pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0);
+    pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0);
   }
 }
 
@@ -1236,7 +1243,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
   if (pPartNode->part.pExprs != NULL) {
     int32_t    num = 0;
     SExprInfo* pCalExprInfo = createExprInfo(pPartNode->part.pExprs, NULL, &num);
-    code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num);
+    code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -1251,7 +1258,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
     }
     pInfo->tbnameCalSup.pExprInfo = pSubTableExpr;
     createExprFromOneNode(pSubTableExpr, pPartNode->pSubtable, 0);
-    code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1);
+    code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -1265,7 +1272,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _error;
     }
-    if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) {
+    if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _error;
     }

source/libs/executor/src/operator.c

@@ -25,7 +25,8 @@
 #include "operator.h"
 #include "query.h"
 #include "querytask.h"
-#include "vnode.h"
+
+#include "storageapi.h"
 
 SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
                                    __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn,
@@ -233,11 +234,12 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
 }
 
 static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) {
+  SStorageAPI* pAPI = pParam->pParam;
   if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
     STableScanInfo* pInfo = pOperator->info;
 
     if (pInfo->base.dataReader != NULL) {
-      tsdbReaderSetCloseFlag(pInfo->base.dataReader);
+      pAPI->tsdReader.tsdReaderNotifyClosing(pInfo->base.dataReader);
     }
     return OPTR_FN_RET_ABORT;
   } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
@@ -246,7 +248,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam,
     if (pInfo->pTableScanOp != NULL) {
       STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
       if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) {
-        tsdbReaderSetCloseFlag(pTableScanInfo->base.dataReader);
+        pAPI->tsdReader.tsdReaderNotifyClosing(pTableScanInfo->base.dataReader);
       }
     }
 
@@ -256,8 +258,8 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam,
   return OPTR_FN_RET_CONTINUE;
 }
 
-int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr) {
+int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI) {
-  STraverParam p = {0};
+  STraverParam p = {.pParam = pAPI};
   traverseOperatorTree(pOperator, doStopDataReader, &p, pIdStr);
   return p.code;
 }
@@ -378,17 +380,18 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
       STableListInfo*          pTableListInfo = tableListCreate();
 
       if (pBlockNode->tableType == TSDB_SUPER_TABLE) {
-        SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo));
+        SArray* pList = taosArrayInit(4, sizeof(uint64_t));
-        int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pList);
+        int32_t code = pTaskInfo->storageAPI.metaFn.getChildTableList(pHandle->vnode, pBlockNode->uid, pList);
         if (code != TSDB_CODE_SUCCESS) {
-          pTaskInfo->code = terrno;
+          pTaskInfo->code = code;
+          taosArrayDestroy(pList);
           return NULL;
         }
 
         size_t num = taosArrayGetSize(pList);
         for (int32_t i = 0; i < num; ++i) {
-          STableKeyInfo* p = taosArrayGet(pList, i);
+          uint64_t* id = taosArrayGet(pList, i);
-          tableListAddTableInfo(pTableListInfo, p->uid, 0);
+          tableListAddTableInfo(pTableListInfo, *id, 0);
         }
 
         taosArrayDestroy(pList);

source/libs/executor/src/projectoperator.c

@@ -18,6 +18,7 @@
 #include "functionMgt.h"
 #include "operator.h"
 #include "querytask.h"
+#include "tdatablock.h"
 
 typedef struct SProjectOperatorInfo {
   SOptrBasicInfo binfo;
@@ -114,7 +115,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
 
   initResultSizeInfo(&pOperator->resultInfo, numOfRows);
   code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
-                    pTaskInfo->streamInfo.pState);
+                    pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -369,7 +370,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
   if (pPhyNode->pExprs != NULL) {
     int32_t    num = 0;
     SExprInfo* pSExpr = createExprInfo(pPhyNode->pExprs, NULL, &num);
-    int32_t    code = initExprSupp(&pInfo->scalarSup, pSExpr, num);
+    int32_t    code = initExprSupp(&pInfo->scalarSup, pSExpr, num, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -391,7 +392,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
   blockDataEnsureCapacity(pResBlock, numOfRows);
 
   int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str,
-                            pTaskInfo->streamInfo.pState);
+                            pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }

source/libs/executor/src/querytask.c

@@ -29,13 +29,13 @@
 #include "operator.h"
 #include "query.h"
 #include "querytask.h"
+#include "storageapi.h"
 #include "thash.h"
 #include "ttypes.h"
-#include "vnode.h"
 
 #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st)))
 
-SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model) {
+SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI) {
   SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
   if (pTaskInfo == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -48,6 +48,7 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP
   pTaskInfo->execModel = model;
   pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo));
   pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES);
+  pTaskInfo->storageAPI = *pAPI;
 
   taosInitRWLatch(&pTaskInfo->lock);
 
@@ -55,7 +56,6 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP
   pTaskInfo->id.queryId = queryId;
   pTaskInfo->id.str = taosMemoryMalloc(64);
   buildTaskId(taskId, queryId, pTaskInfo->id.str);
-
   return pTaskInfo;
 }
 
@@ -63,7 +63,7 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); }
 
 void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) {
   pTaskInfo->code = rspCode;
-  stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str);
+  stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI);
 }
 
 void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
@@ -78,7 +78,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
 
 int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
                            int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) {
-  *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model);
+  *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model, &pHandle->api);
   if (*pTaskInfo == NULL) {
     taosMemoryFree(sql);
     return terrno;
@@ -120,13 +120,15 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
     return terrno;
   }
 
-  metaReaderInit(&mr, pHandle->meta, 0);
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
-  int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid);
+
+  pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pAPI->metaFn);
+  int32_t code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, pScanNode->uid);
   if (code != TSDB_CODE_SUCCESS) {
     qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid,
            GET_TASKID(pTaskInfo));
 
-    metaReaderClear(&mr);
+    pAPI->metaReaderFn.clearReader(&mr);
     return terrno;
   }
 
@@ -142,9 +144,9 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
     tDecoderClear(&mr.coder);
 
     tb_uid_t suid = mr.me.ctbEntry.suid;
-    code = metaGetTableEntryByUidCache(&mr, suid);
+    code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, suid);
     if (code != TSDB_CODE_SUCCESS) {
-      metaReaderClear(&mr);
+      pAPI->metaReaderFn.clearReader(&mr);
       return terrno;
     }
 
@@ -154,7 +156,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
     pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow);
   }
 
-  metaReaderClear(&mr);
+  pAPI->metaReaderFn.clearReader(&mr);
 
   pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode);
   return TSDB_CODE_SUCCESS;

source/libs/executor/src/sortoperator.c

@@ -60,7 +60,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
   }
 
   pOperator->exprSupp.pCtx =
-      createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
+      createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore);
   initResultSizeInfo(&pOperator->resultInfo, 1024);
   code = filterInitFromNode((SNode*)pSortNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
   if (code != TSDB_CODE_SUCCESS) {
@@ -500,7 +500,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
   pSup->numOfExprs = numOfCols;
 
   initResultSizeInfo(&pOperator->resultInfo, 1024);
-  pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
+  pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore);
 
   pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

source/libs/executor/src/sysscanoperator.c

@@ -21,27 +21,27 @@
 #include "querynodes.h"
 #include "systable.h"
 #include "tname.h"
-#include "ttime.h"
 
 #include "tdatablock.h"
 #include "tmsg.h"
 
+#include "operator.h"
 #include "query.h"
+#include "querytask.h"
+#include "storageapi.h"
 #include "tcompare.h"
 #include "thash.h"
 #include "ttypes.h"
-#include "vnode.h"
+#include "index.h"
-#include "operator.h"
-#include "querytask.h"
-
 
 typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype);
 typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result);
 typedef int32_t (*__sys_check)(SNode* cond);
 
 typedef struct SSTabFltArg {
-  void* pMeta;
+  void*        pMeta;
-  void* pVnode;
+  void*        pVnode;
+  SStorageAPI* pAPI;
 } SSTabFltArg;
 
 typedef struct SSysTableIndex {
@@ -72,6 +72,7 @@ typedef struct SSysTableScanInfo {
   SLoadRemoteDataInfo    loadInfo;
   SLimitInfo             limitInfo;
   int32_t                tbnameSlotId;
+  SStorageAPI*           pAPI;
 } SSysTableScanInfo;
 
 typedef struct {
@@ -153,10 +154,11 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t
                                                SFilterInfo* pFilterInfo);
 
 int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) {
-  void* pVnode = ((SSTabFltArg*)arg)->pVnode;
+  SSTabFltArg* pArg = arg;
+  void*        pVnode = pArg->pVnode;
 
   const char* db = NULL;
-  vnodeGetInfo(pVnode, &db, NULL);
+  pArg->pAPI->metaFn.getBasicInfo(pVnode, &db, NULL, NULL, NULL);
 
   SName sn = {0};
   char  dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -180,10 +182,11 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) {
 }
 
 int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) {
-  void* pVnode = ((SSTabFltArg*)arg)->pVnode;
+  SSTabFltArg* pArg = arg;
+  void*        pVnode = ((SSTabFltArg*)arg)->pVnode;
 
   int64_t vgId = 0;
-  vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId);
+  pArg->pAPI->metaFn.getBasicInfo(pVnode, NULL, (int32_t*)&vgId, NULL, NULL);
 
   SOperatorNode* pOper = (SOperatorNode*)pNode;
   SValueNode*    pVal = (SValueNode*)pOper->pRight;
@@ -200,7 +203,7 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) {
 }
 
 int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) {
-  void* pMeta = ((SSTabFltArg*)arg)->pMeta;
+  SSTabFltArg* pArg = arg;
 
   SOperatorNode* pOper = (SOperatorNode*)pNode;
   SValueNode*    pVal = (SValueNode*)pOper->pRight;
@@ -220,7 +223,8 @@ int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) {
 }
 
 int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) {
-  void* pMeta = ((SSTabFltArg*)arg)->pMeta;
+  SSTabFltArg* pArg = arg;
+  SStorageAPI* pAPI = pArg->pAPI;
 
   SOperatorNode* pOper = (SOperatorNode*)pNode;
   SValueNode*    pVal = (SValueNode*)pOper->pRight;
@@ -237,7 +241,7 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) {
                          .equal = equal,
                          .filterFunc = func};
 
-  int32_t ret = metaFilterCreateTime(pMeta, &param, result);
+  int32_t ret = pAPI->metaFilter.metaFilterCreateTime(pArg->pVnode, &param, result);
   return ret;
 }
 
@@ -431,8 +435,9 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) {
 }
 
 static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
-  qDebug("sysTableScanUserCols get cols start");
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
-  SExecTaskInfo*     pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
+
   SSysTableScanInfo* pInfo = pOperator->info;
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -446,7 +451,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
 
   const char* db = NULL;
   int32_t     vgId = 0;
-  vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+  pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
 
   SName sn = {0};
   char  dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -461,18 +466,18 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
     STR_TO_VARSTR(tableName, pInfo->req.filterTb);
 
     SMetaReader smrTable = {0};
-    metaReaderInit(&smrTable, pInfo->readHandle.meta, 0);
+    pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
-    int32_t code = metaGetTableEntryByName(&smrTable, pInfo->req.filterTb);
+    int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb);
     if (code != TSDB_CODE_SUCCESS) {
-      // terrno has been set by metaGetTableEntryByName, therefore, return directly
+      // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
-      metaReaderClear(&smrTable);
+      pAPI->metaReaderFn.clearReader(&smrTable);
       blockDataDestroy(dataBlock);
       pInfo->loadInfo.totalRows = 0;
       return NULL;
     }
 
     if (smrTable.me.type == TSDB_SUPER_TABLE) {
-      metaReaderClear(&smrTable);
+      pAPI->metaReaderFn.clearReader(&smrTable);
       blockDataDestroy(dataBlock);
       pInfo->loadInfo.totalRows = 0;
       return NULL;
@@ -480,12 +485,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
 
     if (smrTable.me.type == TSDB_CHILD_TABLE) {
       int64_t suid = smrTable.me.ctbEntry.suid;
-      metaReaderClear(&smrTable);
+      pAPI->metaReaderFn.clearReader(&smrTable);
-      metaReaderInit(&smrTable, pInfo->readHandle.meta, 0);
+      pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
-      code = metaGetTableEntryByUid(&smrTable, suid);
+      code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid);
       if (code != TSDB_CODE_SUCCESS) {
-        // terrno has been set by metaGetTableEntryByName, therefore, return directly
+        // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
-        metaReaderClear(&smrTable);
+        pAPI->metaReaderFn.clearReader(&smrTable);
         blockDataDestroy(dataBlock);
         pInfo->loadInfo.totalRows = 0;
         return NULL;
@@ -503,7 +508,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
     }
 
     sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName);
-    metaReaderClear(&smrTable);
+    pAPI->metaReaderFn.clearReader(&smrTable);
 
     if (numOfRows > 0) {
       relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
@@ -517,7 +522,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
 
   int32_t ret = 0;
   if (pInfo->pCur == NULL) {
-    pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta);
+    pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode);
   }
 
   if (pInfo->pSchema == NULL) {
@@ -535,8 +540,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
 
   int32_t restore = pInfo->restore;
   pInfo->restore = false;
-  while (restore || ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) {
+  
-    if (restore) restore = false;
+  while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) {
+    if (restore) {
+      restore = false;
+    }
+    
     char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
     char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
 
@@ -560,12 +569,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
         schemaRow = *(SSchemaWrapper**)schema;
       } else {
         SMetaReader smrSuperTable = {0};
-        metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0);
+        pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
-        int code = metaGetTableEntryByUid(&smrSuperTable, suid);
+        int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid);
         if (code != TSDB_CODE_SUCCESS) {
-          // terrno has been set by metaGetTableEntryByName, therefore, return directly
+          // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
           qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code);
-          metaReaderClear(&smrSuperTable);
+          pAPI->metaReaderFn.clearReader(&smrSuperTable);
           blockDataDestroy(dataBlock);
           pInfo->loadInfo.totalRows = 0;
           return NULL;
@@ -573,7 +582,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
         SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow);
         taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES);
         schemaRow = schemaWrapper;
-        metaReaderClear(&smrSuperTable);
+        pAPI->metaReaderFn.clearReader(&smrSuperTable);
       }
     } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) {
       qDebug("sysTableScanUserCols cursor get normal table");
@@ -605,7 +614,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
 
   blockDataDestroy(dataBlock);
   if (ret != 0) {
-    metaCloseTbCursor(pInfo->pCur);
+    pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
     pInfo->pCur = NULL;
     setOperatorCompleted(pOperator);
   }
@@ -618,6 +627,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
 
 static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
   SExecTaskInfo*     pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
   SSysTableScanInfo* pInfo = pOperator->info;
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -631,7 +642,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
 
   const char* db = NULL;
   int32_t     vgId = 0;
-  vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+  pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
 
   SName sn = {0};
   char  dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -647,37 +658,37 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
     STR_TO_VARSTR(tableName, condTableName);
 
     SMetaReader smrChildTable = {0};
-    metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0);
+    pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
-    int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName);
+    int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName);
     if (code != TSDB_CODE_SUCCESS) {
-      // terrno has been set by metaGetTableEntryByName, therefore, return directly
+      // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
-      metaReaderClear(&smrChildTable);
+      pAPI->metaReaderFn.clearReader(&smrChildTable);
       blockDataDestroy(dataBlock);
       pInfo->loadInfo.totalRows = 0;
       return NULL;
     }
 
     if (smrChildTable.me.type != TSDB_CHILD_TABLE) {
-      metaReaderClear(&smrChildTable);
+      pAPI->metaReaderFn.clearReader(&smrChildTable);
       blockDataDestroy(dataBlock);
       pInfo->loadInfo.totalRows = 0;
       return NULL;
     }
 
     SMetaReader smrSuperTable = {0};
-    metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK);
+    pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn);
-    code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
+    code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
     if (code != TSDB_CODE_SUCCESS) {
-      // terrno has been set by metaGetTableEntryByUid
+      // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid
-      metaReaderClear(&smrSuperTable);
+      pAPI->metaReaderFn.clearReader(&smrSuperTable);
-      metaReaderClear(&smrChildTable);
+      pAPI->metaReaderFn.clearReader(&smrChildTable);
       blockDataDestroy(dataBlock);
       return NULL;
     }
 
     sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
-    metaReaderClear(&smrSuperTable);
+    pAPI->metaReaderFn.clearReader(&smrSuperTable);
-    metaReaderClear(&smrChildTable);
+    pAPI->metaReaderFn.clearReader(&smrChildTable);
 
     if (numOfRows > 0) {
       relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
@@ -691,11 +702,11 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
 
   int32_t ret = 0;
   if (pInfo->pCur == NULL) {
-    pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta);
+    pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode);
   }
 
   bool blockFull = false;
-  while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
+  while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
     if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) {
       continue;
     }
@@ -704,27 +715,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
     STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name);
 
     SMetaReader smrSuperTable = {0};
-    metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0);
+    pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
     uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
-    int32_t  code = metaGetTableEntryByUid(&smrSuperTable, suid);
+    int32_t  code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid);
     if (code != TSDB_CODE_SUCCESS) {
       qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno),
              GET_TASKID(pTaskInfo));
-      metaReaderClear(&smrSuperTable);
+      pAPI->metaReaderFn.clearReader(&smrSuperTable);
-      metaCloseTbCursor(pInfo->pCur);
+      pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
       pInfo->pCur = NULL;
       T_LONG_JMP(pTaskInfo->env, terrno);
     }
 
     if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) {
-      metaTbCursorPrev(pInfo->pCur, TSDB_TABLE_MAX);
+      pAPI->metaFn.cursorPrev(pInfo->pCur, TSDB_TABLE_MAX);
       blockFull = true;
     } else {
       sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows,
                                        dataBlock);
     }
 
-    metaReaderClear(&smrSuperTable);
+    pAPI->metaReaderFn.clearReader(&smrSuperTable);
 
     if (blockFull || numOfRows >= pOperator->resultInfo.capacity) {
       relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
@@ -745,7 +756,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
 
   blockDataDestroy(dataBlock);
   if (ret != 0) {
-    metaCloseTbCursor(pInfo->pCur);
+    pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
     pInfo->pCur = NULL;
     setOperatorCompleted(pOperator);
   }
@@ -1090,6 +1101,8 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) {
 
 static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
   SExecTaskInfo*     pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
   SSysTableScanInfo* pInfo = pOperator->info;
 
   SSysTableIndex* pIdx = pInfo->pIdx;
@@ -1100,7 +1113,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
 
   const char* db = NULL;
   int32_t     vgId = 0;
-  vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+  pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
 
   SName sn = {0};
   char  dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -1118,10 +1131,10 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
     tb_uid_t* uid = taosArrayGet(pIdx->uids, i);
 
     SMetaReader mr = {0};
-    metaReaderInit(&mr, pInfo->readHandle.meta, 0);
+    pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
-    ret = metaGetTableEntryByUid(&mr, *uid);
+    ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid);
     if (ret < 0) {
-      metaReaderClear(&mr);
+      pAPI->metaReaderFn.clearReader(&mr);
       continue;
     }
     STR_TO_VARSTR(n, mr.me.name);
@@ -1146,15 +1159,15 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
       colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
 
       SMetaReader mr1 = {0};
-      metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK);
+      pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn);
 
       int64_t suid = mr.me.ctbEntry.suid;
-      int32_t code = metaGetTableEntryByUid(&mr1, suid);
+      int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid);
       if (code != TSDB_CODE_SUCCESS) {
         qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name,
                suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
-        metaReaderClear(&mr1);
+        pAPI->metaReaderFn.clearReader(&mr1);
-        metaReaderClear(&mr);
+        pAPI->metaReaderFn.clearReader(&mr);
         T_LONG_JMP(pTaskInfo->env, terrno);
       }
       pColInfoData = taosArrayGet(p->pDataBlock, 3);
@@ -1164,7 +1177,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
       STR_TO_VARSTR(n, mr1.me.name);
       pColInfoData = taosArrayGet(p->pDataBlock, 4);
       colDataSetVal(pColInfoData, numOfRows, n, false);
-      metaReaderClear(&mr1);
+      pAPI->metaReaderFn.clearReader(&mr1);
 
       // table comment
       pColInfoData = taosArrayGet(p->pDataBlock, 8);
@@ -1229,7 +1242,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
       // impl later
     }
 
-    metaReaderClear(&mr);
+    pAPI->metaReaderFn.clearReader(&mr);
 
     pColInfoData = taosArrayGet(p->pDataBlock, 9);
     colDataSetVal(pColInfoData, numOfRows, n, false);
@@ -1275,10 +1288,11 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
 
 static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
 
   SSysTableScanInfo* pInfo = pOperator->info;
   if (pInfo->pCur == NULL) {
-    pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta);
+    pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode);
   }
 
   blockDataCleanup(pInfo->pRes);
@@ -1286,7 +1300,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
 
   const char* db = NULL;
   int32_t     vgId = 0;
-  vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+  pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
 
   SName sn = {0};
   char  dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -1301,7 +1315,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
   char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
 
   int32_t ret = 0;
-  while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
+  while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
     STR_TO_VARSTR(n, pInfo->pCur->mr.me.name);
 
     // table name
@@ -1324,15 +1338,15 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
       colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
 
       SMetaReader mr = {0};
-      metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK);
+      pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn);
 
       uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
-      int32_t  code = metaGetTableEntryByUid(&mr, suid);
+      int32_t  code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid);
       if (code != TSDB_CODE_SUCCESS) {
         qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name,
                suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
-        metaReaderClear(&mr);
+        pAPI->metaReaderFn.clearReader(&mr);
-        metaCloseTbCursor(pInfo->pCur);
+        pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
         pInfo->pCur = NULL;
         T_LONG_JMP(pTaskInfo->env, terrno);
       }
@@ -1345,7 +1359,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
       STR_TO_VARSTR(n, mr.me.name);
       pColInfoData = taosArrayGet(p->pDataBlock, 4);
       colDataSetVal(pColInfoData, numOfRows, n, false);
-      metaReaderClear(&mr);
+      pAPI->metaReaderFn.clearReader(&mr);
 
       // table comment
       pColInfoData = taosArrayGet(p->pDataBlock, 8);
@@ -1442,7 +1456,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
 
   // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found
   if (ret != 0) {
-    metaCloseTbCursor(pInfo->pCur);
+    pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
     pInfo->pCur = NULL;
     setOperatorCompleted(pOperator);
   }
@@ -1471,7 +1485,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
   } else {
     if (pInfo->showRewrite == false) {
       if (pCondition != NULL && pInfo->pIdx == NULL) {
-        SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode};
+        SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI};
 
         SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex));
         idx->init = 0;
@@ -1708,7 +1722,7 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca
 
 SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode,
                                               const char* pUser, SExecTaskInfo* pTaskInfo) {
-  int32_t            code = TDB_CODE_SUCCESS;
+  int32_t            code = TSDB_CODE_SUCCESS;
   SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo));
   SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -1726,6 +1740,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
 
   extractTbnameSlotId(pInfo, pScanNode);
 
+  pInfo->pAPI = &pTaskInfo->storageAPI;
+
   pInfo->accountId = pScanPhyNode->accountId;
   pInfo->pUser = taosStrdup((void*)pUser);
   pInfo->sysInfo = pScanPhyNode->sysInfo;
@@ -1798,9 +1814,13 @@ void destroySysScanOperator(void* param) {
   if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 ||
       strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 ||
       strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) {
-    metaCloseTbCursor(pInfo->pCur);
+    if (pInfo->pAPI->metaFn.closeTableMetaCursor != NULL) {
+      pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
+    }
+
     pInfo->pCur = NULL;
   }
+
   if (pInfo->pIdx) {
     taosArrayDestroy(pInfo->pIdx->uids);
     taosMemoryFree(pInfo->pIdx);
@@ -2124,15 +2144,15 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
   return -1;
 }
 
-static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) {
+static int32_t doGetTableRowSize(SReadHandle *pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) {
   *rowLen = 0;
 
   SMetaReader mr = {0};
-  metaReaderInit(&mr, pMeta, 0);
+  pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn);
-  int32_t code = metaGetTableEntryByUid(&mr, uid);
+  int32_t code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, uid);
   if (code != TSDB_CODE_SUCCESS) {
     qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr);
-    metaReaderClear(&mr);
+    pHandle->api.metaReaderFn.clearReader(&mr);
     return terrno;
   }
 
@@ -2144,10 +2164,10 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
   } else if (mr.me.type == TSDB_CHILD_TABLE) {
     uint64_t suid = mr.me.ctbEntry.suid;
     tDecoderClear(&mr.coder);
-    code = metaGetTableEntryByUid(&mr, suid);
+    code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, suid);
     if (code != TSDB_CODE_SUCCESS) {
       qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr);
-      metaReaderClear(&mr);
+      pHandle->api.metaReaderFn.clearReader(&mr);
       return terrno;
     }
 
@@ -2163,7 +2183,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
     }
   }
 
-  metaReaderClear(&mr);
+  pHandle->api.metaReaderFn.clearReader(&mr);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -2174,16 +2194,17 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
 
   SBlockDistInfo* pBlockScanInfo = pOperator->info;
   SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
 
   STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN};
-  int32_t             code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid,
+  int32_t             code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid,
                                                (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo));
   if (code != TSDB_CODE_SUCCESS) {
     T_LONG_JMP(pTaskInfo->env, code);
   }
 
-  tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
+  pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
-  blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle);
+  blockDistInfo.numOfInmemRows = (int32_t) pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle);
 
   SSDataBlock* pBlock = pBlockScanInfo->pResBlock;
 
@@ -2213,7 +2234,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
 static void destroyBlockDistScanOperatorInfo(void* param) {
   SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param;
   blockDataDestroy(pDistInfo->pResBlock);
-  tsdbReaderClose(pDistInfo->pHandle);
+  pDistInfo->readHandle.api.tsdReader.tsdReaderClose(pDistInfo->pHandle);
   tableListDestroy(pDistInfo->pTableListInfo);
   taosMemoryFreeClear(param);
 }
@@ -2268,7 +2289,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
     size_t num = tableListGetSize(pTableListInfo);
     void*  pList = tableListGetInfo(pTableListInfo, 0);
 
-    code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false, NULL);
+    code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL);
     cleanupQueryTableDataCond(&cond);
     if (code != 0) {
       goto _error;
@@ -2280,7 +2301,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
 
   int32_t    numOfCols = 0;
   SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
-  int32_t    code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
+  int32_t    code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }

source/libs/executor/src/timesliceoperator.c

@@ -18,6 +18,7 @@
 #include "functionMgt.h"
 #include "operator.h"
 #include "querytask.h"
+#include "storageapi.h"
 #include "tcommon.h"
 #include "tcompare.h"
 #include "tdatablock.h"
@@ -873,7 +874,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
 
   int32_t    numOfExprs = 0;
   SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
-  int32_t    code = initExprSupp(pSup, pExprInfo, numOfExprs);
+  int32_t    code = initExprSupp(pSup, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -881,7 +882,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
   if (pInterpPhyNode->pExprs != NULL) {
     int32_t    num = 0;
     SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
-    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
+    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }

source/libs/executor/src/timewindowoperator.c

@@ -1334,10 +1334,12 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf,
 }
 
 static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   SStreamIntervalOperatorInfo* pInfo = pOperator->info;
   SWinKey                      key = {.ts = ts, .groupId = groupId};
   tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));
-  streamStateDel(pInfo->pState, &key);
+  pAPI->stateStore.streamStateDel(pInfo->pState, &key);
   return true;
 }
 
@@ -1474,7 +1476,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
   for (int32_t i = *index; i < size; i++) {
     SWinKey* pWin = taosArrayGet(pWins, i);
     void*    tbname = NULL;
-    streamStateGetParName(pInfo->pState, pWin->groupId, &tbname);
+    pInfo->statestore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname);
     if (tbname == NULL) {
       appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);
     } else {
@@ -1482,7 +1484,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
       STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
       appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);
     }
-    streamFreeVal(tbname);
+    pInfo->statestore.streamStateFreeVal(tbname);
     (*index)++;
   }
 }
@@ -1536,7 +1538,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
   blockDataDestroy(pInfo->pPullDataRes);
   taosArrayDestroy(pInfo->pDelWins);
   blockDataDestroy(pInfo->pDelRes);
-  streamFileStateDestroy(pInfo->pState->pFileState);
+  pInfo->statestore.streamFileStateDestroy(pInfo->pState->pFileState);
   taosMemoryFreeClear(pInfo->pState);
 
   nodesDestroyNode((SNode*)pInfo->pPhyNode);
@@ -1611,16 +1613,20 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
 }
 
 void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo) {
+  SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;
+
   if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     initIntervalDownStream(downstream->pDownstream[0], type, pInfo);
     return;
   }
+
   SStreamScanInfo* pScanInfo = downstream->info;
   pScanInfo->windowSup.parentType = type;
   pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;
   if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) {
-    pScanInfo->pUpdateInfo = updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark);
+    pScanInfo->pUpdateInfo = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark);
   }
+
   pScanInfo->interval = pInfo->interval;
   pScanInfo->twAggSup = pInfo->twAggSup;
   pScanInfo->pState = pInfo->pState;
@@ -1653,7 +1659,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
   int32_t    code =
-      initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState);
+      initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -1681,7 +1687,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
   if (pPhyNode->window.pExprs != NULL) {
     int32_t    numOfScalar = 0;
     SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
-    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -1881,7 +1887,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
   if (pStateNode->window.pExprs != NULL) {
     int32_t    numOfScalarExpr = 0;
     SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr);
-    int32_t    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
+    int32_t    code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -1907,7 +1913,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
   initResultSizeInfo(&pOperator->resultInfo, 4096);
 
   code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
-                    pTaskInfo->streamInfo.pState);
+                    pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -1976,7 +1982,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
   initBasicInfo(&pInfo->binfo, pResBlock);
 
   int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
-                            pTaskInfo->streamInfo.pState);
+                            pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -2048,33 +2054,33 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3
   }
 }
 
-bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return streamStateCheck(pState, pKey); }
+bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { return pStore->streamStateCheck(pState, pKey); }
 
-int32_t setIntervalOutputBuf(SStreamState* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,
+int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,
                              SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
-                             SAggSupporter* pAggSup) {
+                             SAggSupporter* pAggSup, SStateStore* pStore) {
-  SWinKey key = {
+
-      .ts = win->skey,
+  SWinKey key = { .ts = win->skey, .groupId = groupId };
-      .groupId = groupId,
-  };
   char*   value = NULL;
   int32_t size = pAggSup->resultRowSize;
 
-  if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) {
+  if (pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+
   *pResult = (SRowBuffPos*)value;
   SResultRow* res = (SResultRow*)((*pResult)->pRowBuff);
+
   // set time window for current result
   res->win = (*win);
   setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset);
   return TSDB_CODE_SUCCESS;
 }
 
-bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) {
+bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore) {
   if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {
     SWinKey key = {.ts = pWin->skey, .groupId = groupId};
-    if (!hasIntervalWindow(pState, &key)) {
+    if (!hasIntervalWindow(pState, &key, pStore)) {
       return true;
     }
     return false;
@@ -2105,7 +2111,7 @@ static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {
   clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
   initResultRowInfo(&pInfo->binfo.resultRowInfo);
   pInfo->aggSup.currentPageId = -1;
-  streamStateClear(pInfo->pState);
+  pInfo->statestore.streamStateClear(pInfo->pState);
 }
 
 static void clearSpecialDataBlock(SSDataBlock* pBlock) {
@@ -2201,13 +2207,15 @@ static void clearFunctionContext(SExprSupp* pSup) {
   }
 }
 
-int32_t getOutputBuf(SStreamState* pState, SRowBuffPos* pPos, SResultRow** pResult) {
+int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) {
-  return streamStateGetByPos(pState, pPos, (void**)pResult);
+  return pStore->streamStateGetByPos(pState, pPos, (void**)pResult);
 }
 
-int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
+int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
                                    SGroupResInfo* pGroupResInfo) {
   SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   SExprInfo*      pExprInfo = pSup->pExprInfo;
   int32_t         numOfExprs = pSup->numOfExprs;
   int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;
@@ -2218,7 +2226,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
   for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {
     SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i);
     SResultRow*  pRow = NULL;
-    int32_t      code = getOutputBuf(pState, pPos, &pRow);
+    int32_t      code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore);
     uint64_t     groupId = ((SWinKey*)pPos->pKey)->groupId;
     ASSERT(code == 0);
     doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
@@ -2230,12 +2238,12 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
     if (pBlock->info.id.groupId == 0) {
       pBlock->info.id.groupId = groupId;
       void* tbname = NULL;
-      if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
+      if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
         pBlock->info.parTbName[0] = 0;
       } else {
         memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
       }
-      streamFreeVal(tbname);
+      pAPI->stateStore.streamStateFreeVal(tbname);
     } else {
       // current value belongs to different group, it can't be packed into one datablock
       if (pBlock->info.id.groupId != groupId) {
@@ -2282,7 +2290,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
   return TSDB_CODE_SUCCESS;
 }
 
-void doBuildStreamIntervalResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock,
+void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
                                  SGroupResInfo* pGroupResInfo) {
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
   // set output datablock version
@@ -2357,7 +2365,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
           .groupId = groupId,
       };
       void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
-      if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && isClosed && !chIds) {
+      if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->statestore) && isClosed && !chIds) {
         SPullWindowInfo pull = {
             .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
         // add pull data request
@@ -2388,7 +2396,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
     }
 
     int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,
-                                        pSup->rowEntryInfoOffset, &pInfo->aggSup);
+                                        pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->statestore);
     pResult = (SResultRow*)pResPos->pRowBuff;
     if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
       T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
@@ -2470,6 +2478,8 @@ static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) {
 static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   SStreamIntervalOperatorInfo* pInfo = pOperator->info;
   SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   SOperatorInfo*               downstream = pOperator->pDownstream[0];
   SExprSupp*                   pSup = &pOperator->exprSupp;
 
@@ -2508,8 +2518,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
     } else {
       if (pInfo->twAggSup.maxTs > 0 &&
           pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) {
-        streamStateCommit(pInfo->pState);
+        pAPI->stateStore.streamStateCommit(pInfo->pState);
-        streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
+        pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
         pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs;
       }
       qDebug("===stream===interval final close");
@@ -2710,6 +2720,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   }
 
   pOperator->pTaskInfo = pTaskInfo;
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
   pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
                                 .sliding = pIntervalPhyNode->sliding,
                                 .intervalUnit = pIntervalPhyNode->intervalUnit,
@@ -2735,7 +2747,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   if (pIntervalPhyNode->window.pExprs != NULL) {
     int32_t    numOfScalar = 0;
     SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
-    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -2748,18 +2760,18 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
 
   pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
   *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
-  streamStateSetNumber(pInfo->pState, -1);
+
+  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
   int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
-                            pInfo->pState);
+                            pInfo->pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
   initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs);
-
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
-
   initResultRowInfo(&pInfo->binfo.resultRowInfo);
+
   pInfo->numOfChild = numOfChild;
 
   pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
@@ -2776,6 +2788,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
     pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
   }
+
   pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
   pInfo->pullIndex = 0;
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
@@ -2792,9 +2805,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   pInfo->pUpdated = NULL;
   pInfo->pUpdatedMap = NULL;
   int32_t funResSize= getMaxFunResSize(&pOperator->exprSupp, numOfCols);
-  pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
+  pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
                                                   compareTs, pInfo->pState, pInfo->twAggSup.deleteMark);
   pInfo->dataVersion = 0;
+  pInfo->statestore = pTaskInfo->storageAPI.stateStore;
 
   pOperator->operatorType = pPhyNode->type;
   pOperator->blocking = true;
@@ -2851,9 +2865,9 @@ void destroyStreamSessionAggOperatorInfo(void* param) {
 }
 
 int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
-                        SSDataBlock* pResultBlock) {
+                        SSDataBlock* pResultBlock, SFunctionStateStore* pStore) {
   initBasicInfo(pBasicInfo, pResultBlock);
-  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
+  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -2890,13 +2904,13 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin
   pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
   pScanInfo->pState = pAggSup->pState;
   if ((!pScanInfo->igCheckUpdate || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) && !pScanInfo->pUpdateInfo) {
-    pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark);
+    pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark);
   }
   pScanInfo->twAggSup = *pTwSup;
 }
 
 int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap,
-                               SStreamState* pState, int32_t keySize, int16_t keyType) {
+                               SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore) {
   pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput);
   pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR);
   pSup->gap = gap;
@@ -2907,10 +2921,12 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx,
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pSup->stateStore = *pStore;
+
   initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput);
   pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState));
   *(pSup->pState) = *pState;
-  streamStateSetNumber(pSup->pState, -1);
+  pSup->stateStore.streamStateSetNumber(pSup->pState, -1);
 
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   pSup->pResultRows = tSimpleHashInit(32, hashFn);
@@ -2924,11 +2940,13 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx,
   if (bufSize <= pageSize) {
     bufSize = pageSize * 4;
   }
+
   if (!osTempSpaceAvailable()) {
     terrno = TSDB_CODE_NO_DISKSPACE;
     qError("Init stream agg supporter failed since %s, tempDir:%s", terrstr(), tsTempDir);
     return terrno;
   }
+
   int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir);
   for (int32_t i = 0; i < numOfOutput; ++i) {
     pCtx[i].saveHandle.pBuf = pSup->pResultBuf;
@@ -2953,7 +2971,7 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT
   pKey->win.skey = startTs;
   pKey->win.ekey = endTs;
   pKey->groupId = groupId;
-  int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);
+  int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);
   if (code != TSDB_CODE_SUCCESS) {
     SET_SESSION_WIN_KEY_INVALID(pKey);
   }
@@ -2967,8 +2985,8 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT
   pCurWin->sessionWin.win.skey = startTs;
   pCurWin->sessionWin.win.ekey = endTs;
   int32_t size = pAggSup->resultRowSize;
-  int32_t code =
+  int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin,
-      streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size);
+                                                                     pAggSup->gap, &pCurWin->pOutputBuf, &size);
   if (code == TSDB_CODE_SUCCESS) {
     pCurWin->isOutput = true;
   } else {
@@ -2979,11 +2997,12 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT
 
 int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) {
   int32_t size = 0;
-  int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size);
+  int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
-  streamStateCurNext(pAggSup->pState, pCur);
+
+  pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);
   return TSDB_CODE_SUCCESS;
 }
 void saveDeleteInfo(SArray* pWins, SSessionKey key) {
@@ -3068,7 +3087,7 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo
 }
 
 static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {
-  streamStateSessionDel(pAggSup->pState, pKey);
+  pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey);
   SSessionKey hashKey = {0};
   getSessionHashKey(pKey, &hashKey);
   tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey));
@@ -3086,12 +3105,12 @@ static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo*
 
 SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
                                        SResultWindowInfo* pNextWin) {
-  SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);
+  SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);
   pNextWin->isOutput = true;
   setSessionWinOutputInfo(pStUpdated, pNextWin);
   int32_t size = 0;
   pNextWin->sessionWin = pCurWin->sessionWin;
-  int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size);
+  int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size);
   if (code != TSDB_CODE_SUCCESS) {
     taosMemoryFreeClear(pNextWin->pOutputBuf);
     SET_SESSION_WIN_INVALID(*pNextWin);
@@ -3103,6 +3122,8 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC
                                  SSHashObj* pStDeleted) {
   SExprSupp*                     pSup = &pOperator->exprSupp;
   SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
   SResultRow*                    pCurResult = NULL;
   int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;
@@ -3114,7 +3135,7 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC
     SStreamStateCur*  pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);
     if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) {
       taosMemoryFree(winInfo.pOutputBuf);
-      streamStateFreeCur(pCur);
+      pAPI->stateStore.streamStateFreeCur(pCur);
       break;
     }
     SResultRow* pWinResult = NULL;
@@ -3128,13 +3149,13 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC
     }
     removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin);
     doDeleteSessionWindow(pAggSup, &winInfo.sessionWin);
-    streamStateFreeCur(pCur);
+    pAPI->stateStore.streamStateFreeCur(pCur);
     taosMemoryFree(winInfo.pOutputBuf);
   }
 }
 
 int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) {
-  saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize);
+  saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize, &pAggSup->stateStore);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -3254,6 +3275,8 @@ static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) {
 }
 
 void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
+  SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;
+
   blockDataCleanup(pBlock);
   int32_t size = tSimpleHashGetSize(pStDeleted);
   if (size == 0) {
@@ -3282,14 +3305,14 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo
     SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
 
     void* tbname = NULL;
-    streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname);
+    pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname);
     if (tbname == NULL) {
       colDataSetNULL(pTableCol, pBlock->info.rows);
     } else {
       char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
       STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
       colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
-      streamFreeVal(tbname);
+      pAPI->stateStore.streamStateFreeVal(tbname);
     }
     pBlock->info.rows += 1;
   }
@@ -3301,6 +3324,8 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo
 static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) {
   SExprSupp*                     pSup = &pOperator->exprSupp;
   SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   int32_t                        size = taosArrayGetSize(pWinArray);
   SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
   SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;
@@ -3317,7 +3342,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS
       SStreamAggSupporter*           pChAggSup = &pChInfo->streamAggSup;
       SSessionKey                    chWinKey = {0};
       getSessionHashKey(pWinKey, &chWinKey);
-      SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);
+      SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);
       SResultRow*      pResult = NULL;
       SResultRow*      pChResult = NULL;
       while (1) {
@@ -3344,7 +3369,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS
           break;
         }
       }
-      streamStateFreeCur(pCur);
+      pAPI->stateStore.streamStateFreeCur(pCur);
     }
     if (num > 0) {
       saveSessionOutputBuf(pAggSup, &parentWin);
@@ -3409,7 +3434,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL
   pGroupResInfo->pBuf = NULL;
 }
 
-void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo,
+void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo,
                           SSDataBlock* pBlock) {
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
   // set output datablock version
@@ -3568,7 +3593,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   if (pSessionNode->window.pExprs != NULL) {
     int32_t    numOfScalar = 0;
     SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
-    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -3577,13 +3602,13 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
 
   SExprInfo*   pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
   SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
-  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
+  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
   code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap,
-                                pTaskInfo->streamInfo.pState, 0, 0);
+                                pTaskInfo->streamInfo.pState, 0, 0, &pTaskInfo->storageAPI.stateStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -3639,7 +3664,7 @@ _error:
 
 static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
   tSimpleHashClear(pInfo->streamAggSup.pResultRows);
-  streamStateSessionClear(pInfo->streamAggSup.pState);
+  pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState);
 }
 
 static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
@@ -3765,6 +3790,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
     goto _error;
   }
 
+  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
   SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
 
   pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION);
@@ -3789,7 +3815,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
       }
       SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
       pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
-      streamStateSetNumber(pChInfo->streamAggSup.pState, i);
+      pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i);
       taosArrayPush(pInfo->pChildren, &pChildOp);
     }
   }
@@ -3852,7 +3878,7 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
   pCurWin->winInfo.sessionWin.win.skey = ts;
   pCurWin->winInfo.sessionWin.win.ekey = ts;
   int32_t code =
-      streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize,
+      pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize,
                                     compareStateKey, &pCurWin->winInfo.pOutputBuf, &size);
   pCurWin->pStateKey =
       (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize));
@@ -3873,12 +3899,12 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
 
   pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
   pNextWin->winInfo.pOutputBuf = NULL;
-  SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
+  SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
-  code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0);
+  code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0);
   if (code != TSDB_CODE_SUCCESS) {
     SET_SESSION_WIN_INVALID(pNextWin->winInfo);
   }
-  streamStateFreeCur(pCur);
+  pAggSup->stateStore.streamStateFreeCur(pCur);
 }
 
 int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId,
@@ -3917,7 +3943,9 @@ int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNex
 
 static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,
                                  SSHashObj* pStDeleted) {
-  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI*   pAPI = &pOperator->pTaskInfo->storageAPI;
+
   SStreamStateAggOperatorInfo* pInfo = pOperator->info;
   int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;
   uint64_t                     groupId = pSDataBlock->info.id.groupId;
@@ -3959,7 +3987,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
                                        &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
       tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
       doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);
-      releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf);
+      releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf, &pAPI->stateStore);
       continue;
     }
     code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
@@ -4103,7 +4131,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   if (pStateNode->window.pExprs != NULL) {
     int32_t    numOfScalar = 0;
     SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
-    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -4122,14 +4150,14 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   int32_t      numOfCols = 0;
   SExprInfo*   pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);
   SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
-  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
+  code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
   int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes;
   int16_t type = pColNode->node.resType.type;
   code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize,
-                                type);
+                                type, &pTaskInfo->storageAPI.stateStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -4410,7 +4438,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
   SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
 
   code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
-                    pTaskInfo->streamInfo.pState);
+                    pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -4696,7 +4724,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
   initResultSizeInfo(&pOperator->resultInfo, 4096);
 
   int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
-                            pTaskInfo->streamInfo.pState);
+                            pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -4739,7 +4767,9 @@ _error:
 static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   SStreamIntervalOperatorInfo* pInfo = pOperator->info;
   SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
-  SExprSupp*                   pSup = &pOperator->exprSupp;
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
+  SExprSupp* pSup = &pOperator->exprSupp;
 
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -4760,8 +4790,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
     setOperatorCompleted(pOperator);
     if (pInfo->twAggSup.maxTs > 0 &&
         pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) {
-      streamStateCommit(pInfo->pState);
+      pAPI->stateStore.streamStateCommit(pInfo->pState);
-      streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
+      pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
       setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId);
       pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs;
     }
@@ -4903,6 +4933,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
   ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
 
   pOperator->pTaskInfo = pTaskInfo;
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
   pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
   pInfo->ignoreExpiredDataSaved = false;
   pInfo->isFinal = false;
@@ -4917,11 +4949,11 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
 
   pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
   *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
-  streamStateSetNumber(pInfo->pState, -1);
+  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
 
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
   code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
-                    pInfo->pState);
+                    pInfo->pState, &pTaskInfo->storageAPI.functionStore);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -4929,7 +4961,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
   if (pIntervalPhyNode->window.pExprs != NULL) {
     int32_t    numOfScalar = 0;
     SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
-    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -4955,7 +4987,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pUpdated = NULL;
   pInfo->pUpdatedMap = NULL;
   int32_t funResSize= getMaxFunResSize(pSup, numOfCols);
-  pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
+  pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
                                                   compareTs, pInfo->pState, pInfo->twAggSup.deleteMark);
 
   setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,
@@ -4963,6 +4995,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL,
                                          destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL);
 
+  pInfo->statestore = pTaskInfo->storageAPI.stateStore;
   initIntervalDownStream(downstream, pPhyNode->type, pInfo);
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {

source/libs/function/CMakeLists.txt

@@ -34,7 +34,6 @@ target_link_libraries(
     PRIVATE scalar 
     PRIVATE geometry 
     PRIVATE transport 
-    PRIVATE stream ${LINK_JEMALLOC}
     PUBLIC uv_a
 )
 

source/libs/function/src/builtinsimpl.c

@@ -18,7 +18,6 @@
 #include "function.h"
 #include "query.h"
 #include "querynodes.h"
-#include "streamState.h"
 #include "tcompare.h"
 #include "tdatablock.h"
 #include "tdigest.h"
@@ -1697,7 +1696,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
 
         GET_TYPED_DATA(v, double, pVal->nType, &pVal->i);
 
-        int32_t code = getPercentile(pMemBucket, v, &ppInfo->result);
+        code = getPercentile(pMemBucket, v, &ppInfo->result);
         if (code != TSDB_CODE_SUCCESS) {
           goto _fin_error;
         }
@@ -3120,7 +3119,7 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid
 }
 
 static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, SWinKey* key,
-                               STuplePos* pPos) {
+                               STuplePos* pPos, SFunctionStateStore* pStore) {
   STuplePos p = {0};
   if (pHandle->pBuf != NULL) {
     SFilePage* pPage = NULL;
@@ -3153,9 +3152,8 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf,
     pPage->num += length;
     setBufPageDirty(pPage, true);
     releaseBufPage(pHandle->pBuf, pPage);
-  } else {
+  } else { // other tuple save policy
-    // other tuple save policy
+    if (pStore->streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) {
-    if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) {
       p.streamTupleKey = *key;
     }
   }
@@ -3179,10 +3177,10 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock*
   }
 
   char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf);
-  return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos);
+  return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos, pCtx->pStore);
 }
 
-static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) {
+static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos, SFunctionStateStore* pStore) {
   if (pHandle->pBuf != NULL) {
     SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId);
     if (pPage == NULL) {
@@ -3192,7 +3190,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf
     setBufPageDirty(pPage, true);
     releaseBufPage(pHandle->pBuf, pPage);
   } else {
-    streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length);
+    pStore->streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -3202,10 +3200,10 @@ int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc
   prepareBuf(pCtx);
 
   char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf);
-  return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos);
+  return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos, pCtx->pStore);
 }
 
-static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos) {
+static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos, SFunctionStateStore* pStore) {
   if (pHandle->pBuf != NULL) {
     SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId);
     if (pPage == NULL) {
@@ -3217,13 +3215,13 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo
   } else {
     void*   value = NULL;
     int32_t vLen;
-    streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen);
+    pStore->streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen);
     return (char*)value;
   }
 }
 
 const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) {
-  return doLoadTupleData(&pCtx->saveHandle, pPos);
+  return doLoadTupleData(&pCtx->saveHandle, pPos, pCtx->pStore);
 }
 
 int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
@@ -4991,7 +4989,7 @@ static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pI
     memcpy(pInfo->buf, data, pInfo->colBytes);
   }
 
-  return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos);
+  return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos, pCtx->pStore);
 }
 
 static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) {
@@ -5020,7 +5018,7 @@ static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCt
   } else {
     pHashItem->count += 1;
     if (pCtx->subsidiaries.num > 0) {
-      int32_t code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos);
+      code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
       }

source/libs/index/CMakeLists.txt

@@ -12,7 +12,6 @@ target_link_libraries(
     PUBLIC os
     PUBLIC util
     PUBLIC common
-    PUBLIC vnode
     PUBLIC nodes
     PUBLIC scalar 
     PUBLIC function 

source/libs/index/src/indexFilter.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "filter.h"
 #include "index.h"
 #include "indexComm.h"
 #include "indexInt.h"
@@ -20,7 +21,6 @@
 #include "querynodes.h"
 #include "scalar.h"
 #include "tdatablock.h"
-#include "vnode.h"
 
 // clang-format off
 #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
@@ -67,9 +67,8 @@ typedef union {
 
 typedef struct SIFParam {
   SHashObj *pFilter;
-
+  SArray   *result;
-  SArray *result;
+  char     *condValue;
-  char   *condValue;
 
   SIdxFltStatus status;
   uint8_t       colValType;
@@ -79,6 +78,7 @@ typedef struct SIFParam {
   char          colName[TSDB_COL_NAME_LEN * 2 + 4];
 
   SIndexMetaArg arg;
+  SMetaDataFilterAPI api;
 } SIFParam;
 
 typedef struct SIFCtx {
@@ -86,6 +86,7 @@ typedef struct SIFCtx {
   SHashObj     *pRes;    /* element is SIFParam */
   bool          noExec;  // true: just iterate condition tree, and add hint to executor plan
   SIndexMetaArg arg;
+  SMetaDataFilterAPI *pAPI;
 } SIFCtx;
 
 static FORCE_INLINE int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
@@ -288,6 +289,8 @@ static int32_t sifInitParamValByCol(SNode *r, SNode *l, SIFParam *param, SIFCtx
 }
 static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
   param->status = SFLT_COARSE_INDEX;
+  param->api = *ctx->pAPI;
+
   switch (nodeType(node)) {
     case QUERY_NODE_VALUE: {
       SValueNode *vn = (SValueNode *)node;
@@ -364,6 +367,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
       SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
     }
   }
+
   SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam));
 
   if (NULL == paramList) {
@@ -659,7 +663,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
     } else {
       if (sifSetFltParam(left, right, &typedata, &param) != 0) return -1;
     }
-    ret = metaFilterTableIds(arg->metaEx, &param, output->result);
+    ret = left->api.metaFilterTableIds(arg->metaEx, &param, output->result);
   }
   return ret;
 }
@@ -972,8 +976,9 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
   if (pNode == NULL || pDst == NULL) {
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
+
   int32_t code = 0;
-  SIFCtx  ctx = {.code = 0, .noExec = false, .arg = pDst->arg};
+  SIFCtx  ctx = {.code = 0, .noExec = false, .arg = pDst->arg, .pAPI = &pDst->api};
   ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
 
   if (NULL == ctx.pRes) {
@@ -1006,13 +1011,13 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
   return code;
 }
 
-static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) {
+static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) {
   int32_t code = TSDB_CODE_SUCCESS;
   if (pNode == NULL) {
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
 
-  SIFCtx ctx = {.code = 0, .noExec = true};
+  SIFCtx ctx = {.code = 0, .noExec = true, .pAPI = pAPI};
   ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
   if (NULL == ctx.pRes) {
     indexError("index-filter failed to taosHashInit");
@@ -1044,8 +1049,8 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) {
   return code;
 }
 
-int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status) {
+int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) {
-  SIdxFltStatus st = idxGetFltStatus(pFilterNode);
+  SIdxFltStatus st = idxGetFltStatus(pFilterNode, pAPI);
   if (st == SFLT_NOT_INDEX) {
     *status = st;
     return 0;
@@ -1054,7 +1059,7 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result,
   SFilterInfo *filter = NULL;
 
   SArray  *output = taosArrayInit(8, sizeof(uint64_t));
-  SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX};
+  SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX, .api = *pAPI};
   int32_t  code = sifCalculate((SNode *)pFilterNode, &param);
   if (code != 0) {
     sifFreeParam(&param);
@@ -1071,13 +1076,13 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result,
   return TSDB_CODE_SUCCESS;
 }
 
-SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) {
+SIdxFltStatus idxGetFltStatus(SNode *pFilterNode, SMetaDataFilterAPI* pAPI) {
   SIdxFltStatus st = SFLT_NOT_INDEX;
   if (pFilterNode == NULL) {
     return SFLT_NOT_INDEX;
   }
 
-  if (sifGetFltHint((SNode *)pFilterNode, &st) != TSDB_CODE_SUCCESS) {
+  if (sifGetFltHint((SNode *)pFilterNode, &st, pAPI) != TSDB_CODE_SUCCESS) {
     st = SFLT_NOT_INDEX;
   }
   return st;

source/libs/parser/src/parTranslater.c

@@ -8008,6 +8008,9 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
       if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
         isJson = true;
         code = buildJsonTagVal(pCxt, pTagSchema, pVal, pTagArray, ppTag);
+        if (TSDB_CODE_SUCCESS != code) {
+          nodesDestroyNode((SNode*)pVal);
+        }
         taosArrayPush(tagName, pTagSchema->name);
       } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) {
         char*   tmpVal = nodesGetValueFromNode(pVal);
@@ -8328,13 +8331,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
     SArray* pTagVals = taosArrayInit(1, sizeof(STagVal));
     int32_t code = TSDB_CODE_SUCCESS;
     STag*   pTag = NULL;
-    do {
+    code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf);
-      code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf);
-      if (TSDB_CODE_SUCCESS != code) {
-        break;
-      }
-    } while (0);
-
     taosArrayDestroy(pTagVals);
     if (code != TSDB_CODE_SUCCESS) {
       return code;

source/libs/parser/src/parUtil.c

@@ -416,7 +416,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi
 end:
   taosHashCleanup(keyHash);
   if (retCode == TSDB_CODE_SUCCESS) {
-    tTagNew(pTagVals, 1, true, ppTag);
+    retCode = tTagNew(pTagVals, 1, true, ppTag);
   }
   for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
     STagVal* p = (STagVal*)taosArrayGet(pTagVals, i);

source/libs/qworker/CMakeLists.txt

@@ -7,9 +7,15 @@ target_include_directories(
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
-target_link_libraries(qworker
+IF (TD_GRANT)
-        PRIVATE os util transport nodes planner qcom executor
+    TARGET_LINK_LIBRARIES(qworker
-        )
+            PRIVATE os util transport nodes planner qcom executor index grant
+            )
+ELSE ()
+    TARGET_LINK_LIBRARIES(qworker
+            PRIVATE os util transport nodes planner qcom executor index
+            )
+ENDIF()
 
 if(${BUILD_TEST})
     ADD_SUBDIRECTORY(test)

source/libs/qworker/src/qwMsg.c

@@ -440,11 +440,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
   int64_t  rId = msg.refId;
   int32_t  eId = msg.execId;
 
-  SQWMsg qwMsg = {.node = node,
+  SQWMsg qwMsg = {.node = node, .msg = msg.msg, .msgLen = msg.msgLen, .connInfo = pMsg->info, .msgType = pMsg->msgType};
-                  .msg = msg.msg,
-                  .msgLen = msg.msgLen,
-                  .connInfo = pMsg->info,
-                  .msgType = pMsg->msgType};
   qwMsg.msgInfo.explain = msg.explain;
   qwMsg.msgInfo.taskType = msg.taskType;
   qwMsg.msgInfo.needFetch = msg.needFetch;

source/libs/qworker/test/CMakeLists.txt

@@ -8,7 +8,7 @@ IF(NOT TD_DARWIN)
         ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST})
         TARGET_LINK_LIBRARIES(
                 qworkerTest
-                PUBLIC os util common transport gtest qcom nodes planner qworker executor
+                PUBLIC os util common transport gtest qcom nodes planner qworker executor index
         )
 
         TARGET_INCLUDE_DIRECTORIES(

source/libs/scalar/CMakeLists.txt

@@ -14,6 +14,7 @@ target_link_libraries(scalar
     PRIVATE nodes
     PRIVATE function
     PRIVATE qcom
+    PRIVATE parser
 )
 
 if(${BUILD_TEST})

source/libs/scheduler/CMakeLists.txt

@@ -9,7 +9,7 @@ target_include_directories(
 
 target_link_libraries(
         scheduler
-        PUBLIC os util nodes planner qcom common catalog transport command qworker executor
+        PUBLIC os util nodes planner qcom common catalog transport command qworker executor index
 )
 
 if(${BUILD_TEST})

source/libs/stream/CMakeLists.txt

@@ -10,7 +10,7 @@ if(${BUILD_WITH_ROCKSDB})
   target_link_libraries(
     stream
     PUBLIC rocksdb tdb
-    PRIVATE  os util transport qcom executor wal
+    PRIVATE  os util transport qcom executor wal index
   )
 
   target_include_directories(

source/libs/stream/inc/streamBackendRocksdb.h

@@ -17,7 +17,6 @@
 #define _STREAM_BACKEDN_ROCKSDB_H_
 
 #include "rocksdb/c.h"
-// #include "streamInc.h"
 #include "streamState.h"
 #include "tcoding.h"
 #include "tcommon.h"

source/libs/stream/src/streamBackendRocksdb.c

@@ -767,10 +767,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
   if (ppInst != NULL && *ppInst != NULL) {
     RocksdbCfInst* inst = *ppInst;
     pState->pTdbState->rocksdb = inst->db;
-    pState->pTdbState->pHandle = inst->pHandle;
+    pState->pTdbState->pHandle = (void**)inst->pHandle;
     pState->pTdbState->writeOpts = inst->wOpt;
     pState->pTdbState->readOpts = inst->rOpt;
-    pState->pTdbState->cfOpts = inst->cfOpt;
+    pState->pTdbState->cfOpts = (void**)(inst->cfOpt);
     pState->pTdbState->dbOpt = handle->dbOpt;
     pState->pTdbState->param = inst->param;
     pState->pTdbState->pBackend = handle;
@@ -809,10 +809,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
   }
   rocksdb_column_family_handle_t** cfHandle = taosMemoryMalloc(cfLen * sizeof(rocksdb_column_family_handle_t*));
   pState->pTdbState->rocksdb = handle->db;
-  pState->pTdbState->pHandle = cfHandle;
+  pState->pTdbState->pHandle = (void**)cfHandle;
   pState->pTdbState->writeOpts = rocksdb_writeoptions_create();
   pState->pTdbState->readOpts = rocksdb_readoptions_create();
-  pState->pTdbState->cfOpts = (rocksdb_options_t**)cfOpt;
+  pState->pTdbState->cfOpts = (void**)cfOpt;
   pState->pTdbState->dbOpt = handle->dbOpt;
   pState->pTdbState->param = param;
   pState->pTdbState->pBackend = handle;
@@ -1205,7 +1205,8 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin
   }
   pCur->number = pState->number;
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "state", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
 
   SStateKey sKey = {.key = *key, .opNum = pState->number};
   char      buf[128] = {0};
@@ -1245,7 +1246,8 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK
   SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
   if (pCur == NULL) return NULL;
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "state", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
   rocksdb_iter_seek(pCur->iter, buf, (size_t)klen);
 
   rocksdb_iter_prev(pCur->iter);
@@ -1267,7 +1269,8 @@ SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey*
 
   if (pCur == NULL) return NULL;
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "state", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
 
   SStateKey sKey = {.key = *key, .opNum = pState->number};
   char      buf[128] = {0};
@@ -1359,7 +1362,8 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta
   }
   pCur->number = pState->number;
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
 
   char             buf[128] = {0};
   SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
@@ -1398,7 +1402,8 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta
     return NULL;
   }
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
   pCur->number = pState->number;
 
   char buf[128] = {0};
@@ -1434,7 +1439,8 @@ SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, con
     return NULL;
   }
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
   pCur->number = pState->number;
 
   SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
@@ -1526,7 +1532,8 @@ SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinK
   if (pCur == NULL) return NULL;
 
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "fill", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
 
   char buf[128] = {0};
   int  len = winKeyEncode((void*)key, buf);
@@ -1585,7 +1592,8 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const
   }
 
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "fill", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
 
   char buf[128] = {0};
   int  len = winKeyEncode((void*)key, buf);
@@ -1620,7 +1628,8 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const
   }
 
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "fill", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
 
   char buf[128] = {0};
   int  len = winKeyEncode((void*)key, buf);
@@ -1655,7 +1664,8 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes
   }
   pCur->number = pState->number;
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
 
   SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
   int32_t          c = 0;
@@ -1925,7 +1935,8 @@ void* streamDefaultIterCreate_rocksdb(SStreamState* pState) {
   SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
 
   pCur->db = pState->pTdbState->rocksdb;
-  pCur->iter = streamStateIterCreate(pState, "default", &pCur->snapshot, &pCur->readOpt);
+  pCur->iter = streamStateIterCreate(pState, "default", (rocksdb_snapshot_t**)&pCur->snapshot,
+                                     (rocksdb_readoptions_t**)&pCur->readOpt);
   return pCur;
 }
 int32_t streamDefaultIterValid_rocksdb(void* iter) {

source/libs/stream/src/streamExec.c

@@ -163,7 +163,6 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
   int32_t code = 0;
 
   ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
-
   void* exec = pTask->exec.pExecutor;
 
   qSetStreamOpOpen(exec);
@@ -328,7 +327,11 @@ int32_t streamExecForAll(SStreamTask* pTask) {
 
     while (1) {
       if (streamTaskShouldPause(&pTask->status)) {
-        return 0;
+        if (batchSize > 1) {
+          break;
+        } else {
+          return 0;
+        }
       }
 
       SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue);
@@ -404,7 +407,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
 
     {
       // set input
-      void*   pExecutor = pTask->exec.pExecutor;
+      void* pExecutor = pTask->exec.pExecutor;
 
       const SStreamQueueItem* pItem = pInput;
       if (pItem->type == STREAM_INPUT__GET_RES) {

source/libs/stream/src/streamMeta.c

@@ -268,12 +268,14 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
   SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
   if (ppTask) {
     SStreamTask* pTask = *ppTask;
+
     taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t));
     tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn);
 
     atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
-
     int32_t num = taosArrayGetSize(pMeta->pTaskList);
+
+    qDebug("s-task:%s set the drop task flag, remain running s-task:%d", pTask->id.idStr, num - 1);
     for (int32_t i = 0; i < num; ++i) {
       int32_t* pTaskId = taosArrayGet(pMeta->pTaskList, i);
       if (*pTaskId == taskId) {
@@ -283,6 +285,8 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
     }
 
     streamMetaReleaseTask(pMeta, pTask);
+  } else {
+    qDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId);
   }
 
   taosWUnLockLatch(&pMeta->lock);

source/libs/stream/src/streamState.c

@@ -23,7 +23,6 @@
 #include "tcommon.h"
 #include "tcompare.h"
 #include "tref.h"
-#include "ttimer.h"
 
 #define MAX_TABLE_NAME_NUM 200000
 
@@ -91,13 +90,14 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) {
   return winKeyCmprImpl(&pWin1->key, &pWin2->key);
 }
 
-SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages) {
+SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) {
-  qWarn("open stream state, %s", path);
+  qDebug("open stream state, %s", path);
   SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
   if (pState == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
+
   pState->pTdbState = taosMemoryCalloc(1, sizeof(STdbState));
   if (pState->pTdbState == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -105,29 +105,33 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
     return NULL;
   }
 
+  SStreamTask* pStreamTask = pTask;
   char statePath[1024];
   if (!specPath) {
-    sprintf(statePath, "%s/%d", path, pTask->id.taskId);
+    sprintf(statePath, "%s/%d", path, pStreamTask->id.taskId);
   } else {
     memset(statePath, 0, 1024);
     tstrncpy(statePath, path, 1024);
   }
-  pState->taskId = pTask->id.taskId;
+
-  pState->streamId = pTask->id.streamId;
+  pState->taskId = pStreamTask->id.taskId;
+  pState->streamId = pStreamTask->id.streamId;
+
 #ifdef USE_ROCKSDB
-  // qWarn("open stream state1");
+  SStreamMeta* pMeta = pStreamTask->pMeta;
-  taosAcquireRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid);
+  taosAcquireRef(pMeta->streamBackendId, pMeta->streamBackendRid);
-  int code = streamStateOpenBackend(pTask->pMeta->streamBackend, pState);
+  int code = streamStateOpenBackend(pMeta->streamBackend, pState);
   if (code == -1) {
-    taosReleaseRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid);
+    taosReleaseRef(pMeta->streamBackendId, pMeta->streamBackendRid);
     taosMemoryFree(pState);
     pState = NULL;
   }
+
   pState->pTdbState->pOwner = pTask;
   pState->pFileState = NULL;
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT);
-  pState->parNameMap = tSimpleHashInit(1024, hashFn);
 
+  pState->parNameMap = tSimpleHashInit(1024, hashFn);
   return pState;
 
 #else
@@ -449,7 +453,7 @@ int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pV
 #ifdef USE_ROCKSDB
   taosMemoryFree(pVal);
 #else
-  streamFreeVal(pVal);
+  streamStateFreeVal(pVal);
 #endif
   return 0;
 }
@@ -700,7 +704,7 @@ void streamStateFreeCur(SStreamStateCur* pCur) {
   taosMemoryFree(pCur);
 }
 
-void streamFreeVal(void* val) {
+void streamStateFreeVal(void* val) {
 #ifdef USE_ROCKSDB
   taosMemoryFree(val);
 #else

source/libs/stream/src/tstreamFileState.c

@@ -21,6 +21,7 @@
 #include "tcommon.h"
 #include "thash.h"
 #include "tsimplehash.h"
+#include "storageapi.h"
 
 #define FLUSH_RATIO                    0.5
 #define FLUSH_NUM                      4