diff --git a/README-CN.md b/README-CN.md
index 6c49ce0727..81d64941af 100644
--- a/README-CN.md
+++ b/README-CN.md
@@ -52,7 +52,7 @@ TDengine 还提供一组辅助工具软件 taosTools，目前它包含 taosBench
### Ubuntu 18.04 及以上版本 & Debian：
```bash
-sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev
+sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev libgeos-dev
```
#### 为 taos-tools 安装编译需要的软件
@@ -60,7 +60,7 @@ sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 li
为了在 Ubuntu/Debian 系统上编译 [taos-tools](https://github.com/taosdata/taos-tools) 需要安装如下软件：
```bash
-sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libgeos-dev
+sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config
```
### CentOS 7.9
@@ -68,14 +68,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d
```bash
sudo yum install epel-release
sudo yum update
-sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel
+sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel geos geos-devel
sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake
```
-### CentOS 8 & Fedora
+### CentOS 8/Fedora/Rocky Linux
```bash
-sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
+sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel geos geos-devel
```
#### 在 CentOS 上构建 taosTools 安装依赖软件
@@ -85,23 +85,23 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
```
-sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel
+sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel
```
-#### CentOS 8/Rocky Linux
+#### CentOS 8/Fedora/Rocky Linux
```
sudo yum install -y epel-release
sudo yum install -y dnf-plugins-core
sudo yum config-manager --set-enabled powertools
-sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel
+sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel
```
注意：由于 snappy 缺乏 pkg-config 支持（参考 [链接](https://github.com/google/snappy/pull/86)），会导致 cmake 提示无法发现 libsnappy，实际上工作正常。
若 powertools 安装失败，可以尝试改用：
```
-sudo yum config-manager --set-enabled Powertools
+sudo yum config-manager --set-enabled powertools
```
#### CentOS + devtoolset
diff --git a/README.md b/README.md
index 2f1650c3c9..c81e2d309d 100644
--- a/README.md
+++ b/README.md
@@ -60,7 +60,7 @@ To build TDengine, use [CMake](https://cmake.org/) 3.0.2 or higher versions in t
### Ubuntu 18.04 and above or Debian
```bash
-sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev
+sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev libgeos-dev
```
#### Install build dependencies for taosTools
@@ -68,7 +68,7 @@ sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 li
To build the [taosTools](https://github.com/taosdata/taos-tools) on Ubuntu/Debian, the following packages need to be installed.
```bash
-sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libgeos-dev
+sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config
```
### CentOS 7.9
@@ -76,14 +76,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d
```bash
sudo yum install epel-release
sudo yum update
-sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel
+sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel geos geos-devel
sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake
```
-### CentOS 8 & Fedora
+### CentOS 8/Fedora/Rocky Linux
```bash
-sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
+sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel geos geos-devel
```
#### Install build dependencies for taosTools on CentOS
@@ -91,16 +91,16 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
#### CentOS 7.9
```
-sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel
+sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel
```
-#### CentOS 8/Rocky Linux
+#### CentOS 8/Fedora/Rocky Linux
```
sudo yum install -y epel-release
sudo yum install -y dnf-plugins-core
sudo yum config-manager --set-enabled powertools
-sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel geos geos-devel
+sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel
```
Note: Since snappy lacks pkg-config support (refer to [link](https://github.com/google/snappy/pull/86)), it leads a cmake prompt libsnappy not found. But snappy still works well.
diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in
index 391972960a..ef6ed4af1d 100644
--- a/cmake/taosadapter_CMakeLists.txt.in
+++ b/cmake/taosadapter_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
- GIT_TAG 283b50d
+ GIT_TAG 3.0
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md
index f107512e9c..b0124b9c3f 100644
--- a/docs/en/07-develop/09-udf.md
+++ b/docs/en/07-develop/09-udf.md
@@ -377,7 +377,7 @@ The `pybitand` function implements bitwise addition for multiple columns. If the
#### Aggregate Function [pyl2norm](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pyl2norm.py)
-The `pyl2norm` function finds the second-order norm for all data in the input column. This squares the values, takes a cumulative sum, and finds the square root.
+The `pyl2norm` function finds the second-order norm for all data in the input columns. This squares the values, takes a cumulative sum, and finds the square root.
pyl2norm.py
@@ -387,5 +387,16 @@ The `pyl2norm` function finds the second-order norm for all data in the input co
+#### Aggregate Function [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py)
+
+The `pycumsum` function finds the cumulative sum for all data in the input columns.
+
+pycumsum.py
+
+```c
+{{#include tests/script/sh/pycumsum.py}}
+```
+
+
## Manage and Use UDF
You need to add UDF to TDengine before using it in SQL queries. For more information about how to manage UDF and how to invoke UDF, please see [Manage and Use UDF](../12-taos-sql/26-udf.md).
diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx
index 260b38b24f..114026eca0 100644
--- a/docs/en/14-reference/03-connector/04-java.mdx
+++ b/docs/en/14-reference/03-connector/04-java.mdx
@@ -959,6 +959,7 @@ The preceding example uses the SQL statement `select ts, speed from speed_table`
```java
Properties config = new Properties();
+config.setProperty("bootstrap.servers", "localhost:6030");
config.setProperty("enable.auto.commit", "true");
config.setProperty("group.id", "group1");
config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer");
@@ -966,12 +967,14 @@ config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.Res
TaosConsumer consumer = new TaosConsumer<>(config);
```
+- bootstrap.servers: `ip:port` where the TDengine server is located, or `ip:port` where the taosAdapter is located if WebSocket connection is used.
- enable.auto.commit: Specifies whether to commit automatically.
- group.id: consumer: Specifies the group that the consumer is in.
- value.deserializer: To deserialize the results, you can inherit `com.taosdata.jdbc.tmq.ReferenceDeserializer` and specify the result set bean. You can also inherit `com.taosdata.jdbc.tmq.Deserializer` and perform custom deserialization based on the SQL result set.
- td.connect.type: Specifies the type connect with TDengine, `jni` or `WebSocket`. default is `jni`
- httpConnectTimeout: WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type.
- messageWaitTimeout: socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type.
+- httpPoolSize: Maximum number of concurrent requests on the a connection。It only takes effect when using WebSocket type.
- For more information, see [Consumer Parameters](../../../develop/tmq).
#### Subscribe to consume data
@@ -1015,10 +1018,20 @@ public abstract class ConsumerLoop {
public ConsumerLoop() throws SQLException {
Properties config = new Properties();
+ config.setProperty("td.connect.type", "jni");
+ config.setProperty("bootstrap.servers", "localhost:6030");
+ config.setProperty("td.connect.user", "root");
+ config.setProperty("td.connect.pass", "taosdata");
+ config.setProperty("auto.offset.reset", "earliest");
config.setProperty("msg.with.table.name", "true");
config.setProperty("enable.auto.commit", "true");
+ config.setProperty("auto.commit.interval.ms", "1000");
config.setProperty("group.id", "group1");
+ config.setProperty("client.id", "1");
config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+ config.setProperty("value.deserializer.encoding", "UTF-8");
+ config.setProperty("experimental.snapshot.enable", "true");
+
this.consumer = new TaosConsumer<>(config);
this.topics = Collections.singletonList("topic_speed");
@@ -1090,12 +1103,19 @@ public abstract class ConsumerLoop {
public ConsumerLoop() throws SQLException {
Properties config = new Properties();
- config.setProperty("bootstrap.servers", "localhost:6041");
config.setProperty("td.connect.type", "ws");
+ config.setProperty("bootstrap.servers", "localhost:6041");
+ config.setProperty("td.connect.user", "root");
+ config.setProperty("td.connect.pass", "taosdata");
+ config.setProperty("auto.offset.reset", "earliest");
config.setProperty("msg.with.table.name", "true");
config.setProperty("enable.auto.commit", "true");
+ config.setProperty("auto.commit.interval.ms", "1000");
config.setProperty("group.id", "group2");
+ config.setProperty("client.id", "1");
config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+ config.setProperty("value.deserializer.encoding", "UTF-8");
+ config.setProperty("experimental.snapshot.enable", "true");
this.consumer = new TaosConsumer<>(config);
this.topics = Collections.singletonList("topic_speed");
diff --git a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java
index b5cdedc34f..3c5d2867e2 100644
--- a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java
+++ b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java
@@ -53,20 +53,28 @@ public class SubscribeDemo {
// create consumer
Properties properties = new Properties();
+ properties.getProperty(TMQConstants.CONNECT_TYPE, "jni");
properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6030");
+ properties.setProperty(TMQConstants.CONNECT_USER, "root");
+ properties.setProperty(TMQConstants.CONNECT_PASS, "taosdata");
properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true");
properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true");
- properties.setProperty(TMQConstants.GROUP_ID, "test");
+ properties.setProperty(TMQConstants.AUTO_COMMIT_INTERVAL, "1000");
+ properties.setProperty(TMQConstants.GROUP_ID, "test1");
+ properties.setProperty(TMQConstants.CLIENT_ID, "1");
+ properties.setProperty(TMQConstants.AUTO_OFFSET_RESET, "earliest");
properties.setProperty(TMQConstants.VALUE_DESERIALIZER,
"com.taos.example.MetersDeserializer");
+ properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8");
+ properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true");
// poll data
try (TaosConsumer consumer = new TaosConsumer<>(properties)) {
consumer.subscribe(Collections.singletonList(TOPIC));
while (!shutdown.get()) {
ConsumerRecords meters = consumer.poll(Duration.ofMillis(100));
- for (ConsumerRecord recode : meters) {
- Meters meter = recode.value();
+ for (ConsumerRecord r : meters) {
+ Meters meter = r.value();
System.out.println(meter);
}
}
diff --git a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java
index 83cb04f552..03f7e3a11e 100644
--- a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java
+++ b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java
@@ -1,5 +1,6 @@
package com.taos.example;
+import com.taosdata.jdbc.tmq.ConsumerRecord;
import com.taosdata.jdbc.tmq.ConsumerRecords;
import com.taosdata.jdbc.tmq.TMQConstants;
import com.taosdata.jdbc.tmq.TaosConsumer;
@@ -54,18 +55,26 @@ public class WebsocketSubscribeDemo {
Properties properties = new Properties();
properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6041");
properties.setProperty(TMQConstants.CONNECT_TYPE, "ws");
+ properties.setProperty(TMQConstants.CONNECT_USER, "root");
+ properties.setProperty(TMQConstants.CONNECT_PASS, "taosdata");
+ properties.setProperty(TMQConstants.AUTO_OFFSET_RESET, "earliest");
properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true");
properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true");
- properties.setProperty(TMQConstants.GROUP_ID, "test");
+ properties.setProperty(TMQConstants.AUTO_COMMIT_INTERVAL, "1000");
+ properties.setProperty(TMQConstants.GROUP_ID, "test2");
+ properties.setProperty(TMQConstants.CLIENT_ID, "1");
properties.setProperty(TMQConstants.VALUE_DESERIALIZER,
"com.taos.example.MetersDeserializer");
+ properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8");
+ properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true");
// poll data
try (TaosConsumer consumer = new TaosConsumer<>(properties)) {
consumer.subscribe(Collections.singletonList(TOPIC));
while (!shutdown.get()) {
ConsumerRecords meters = consumer.poll(Duration.ofMillis(100));
- for (Meters meter : meters) {
+ for (ConsumerRecord r : meters) {
+ Meters meter = (Meters) r.value();
System.out.println(meter);
}
}
diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md
index 99ecd903b4..92f5d2a857 100644
--- a/docs/zh/07-develop/09-udf.md
+++ b/docs/zh/07-develop/09-udf.md
@@ -335,7 +335,7 @@ def init()
def destroy()
```
-其中 init 完成初始化工作。 destroy 完成清理工作。如果没有初始化工作，无需定义 init 函数。如果没有清理工作，无需定义 destroy 函数。
+其中 init 完成初始化工作。 destroy 完成清理工作。
### Python 和 TDengine之间的数据类型映射
@@ -386,6 +386,17 @@ pyl2norm 实现了输入列的所有数据的二阶范数，即对每个数据
+### 聚合函数示例 [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py)
+
+pycumsum 使用 numpy 计算输入列所有数据的累积和。
+
+pycumsum.py
+
+```c
+{{#include tests/script/sh/pycumsum.py}}
+```
+
+
## 管理和使用 UDF
在使用 UDF 之前需要先将其加入到 TDengine 系统中。关于如何管理和使用 UDF，请参考[管理和使用 UDF](../12-taos-sql/26-udf.md)
diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx
index 35332a9602..e4cf4a83e7 100644
--- a/docs/zh/08-connector/14-java.mdx
+++ b/docs/zh/08-connector/14-java.mdx
@@ -962,6 +962,7 @@ statement.executeUpdate("create topic if not exists topic_speed as select ts, sp
```java
Properties config = new Properties();
+config.setProperty("bootstrap.servers", "localhost:6030");
config.setProperty("enable.auto.commit", "true");
config.setProperty("group.id", "group1");
config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ResultDeserializer");
@@ -969,12 +970,14 @@ config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.Res
TaosConsumer consumer = new TaosConsumer<>(config);
```
+- bootstrap.servers: TDengine 服务端所在的`ip:port`，如果使用 WebSocket 连接，则为 taosAdapter 所在的`ip:port`。
- enable.auto.commit: 是否允许自动提交。
- group.id: consumer: 所在的 group。
- value.deserializer: 结果集反序列化方法，可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`，根据 SQL 的 resultSet 自定义反序列化方式。
- td.connect.type: 连接方式。jni：表示使用动态库连接的方式，ws/WebSocket：表示使用 WebSocket 进行数据通信。默认为 jni 方式。
-- httpConnectTimeout：创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。
-- messageWaitTimeout：数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。
+- httpConnectTimeout: 创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。
+- messageWaitTimeout: 数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。
+- httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。
其他参数请参考：[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group)
#### 订阅消费数据
@@ -1016,10 +1019,19 @@ public abstract class ConsumerLoop {
public ConsumerLoop() throws SQLException {
Properties config = new Properties();
+ config.setProperty("td.connect.type", "jni");
+ config.setProperty("bootstrap.servers", "localhost:6030");
+ config.setProperty("td.connect.user", "root");
+ config.setProperty("td.connect.pass", "taosdata");
+ config.setProperty("auto.offset.reset", "earliest");
config.setProperty("msg.with.table.name", "true");
config.setProperty("enable.auto.commit", "true");
+ config.setProperty("auto.commit.interval.ms", "1000");
config.setProperty("group.id", "group1");
+ config.setProperty("client.id", "1");
config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+ config.setProperty("value.deserializer.encoding", "UTF-8");
+ config.setProperty("experimental.snapshot.enable", "true");
this.consumer = new TaosConsumer<>(config);
this.topics = Collections.singletonList("topic_speed");
@@ -1093,12 +1105,19 @@ public abstract class ConsumerLoop {
public ConsumerLoop() throws SQLException {
Properties config = new Properties();
- config.setProperty("bootstrap.servers", "localhost:6041");
config.setProperty("td.connect.type", "ws");
+ config.setProperty("bootstrap.servers", "localhost:6041");
+ config.setProperty("td.connect.user", "root");
+ config.setProperty("td.connect.pass", "taosdata");
+ config.setProperty("auto.offset.reset", "earliest");
config.setProperty("msg.with.table.name", "true");
config.setProperty("enable.auto.commit", "true");
+ config.setProperty("auto.commit.interval.ms", "1000");
config.setProperty("group.id", "group2");
+ config.setProperty("client.id", "1");
config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
+ config.setProperty("value.deserializer.encoding", "UTF-8");
+ config.setProperty("experimental.snapshot.enable", "true");
this.consumer = new TaosConsumer<>(config);
this.topics = Collections.singletonList("topic_speed");
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 520a8e9c2c..2f93f8c3e3 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -37,6 +37,13 @@ extern "C" {
)
// clang-format on
+typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
+
+typedef struct STableKeyInfo {
+ uint64_t uid;
+ uint64_t groupId;
+} STableKeyInfo;
+
typedef struct SWinKey {
uint64_t groupId;
TSKEY ts;
diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h
index ce7d038d42..0a9037d21c 100644
--- a/include/libs/executor/dataSinkMgt.h
+++ b/include/libs/executor/dataSinkMgt.h
@@ -59,7 +59,7 @@ typedef struct SDataSinkMgtCfg {
uint32_t maxDataBlockNumPerQuery;
} SDataSinkMgtCfg;
-int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg);
+int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI);
typedef struct SInputData {
const struct SSDataBlock* pData;
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index 1fb00e743f..e90bb4688b 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -23,6 +23,7 @@ extern "C" {
#include "query.h"
#include "tcommon.h"
#include "tmsgcb.h"
+#include "storageapi.h"
typedef void* qTaskInfo_t;
typedef void* DataSinkHandle;
@@ -41,7 +42,6 @@ typedef struct {
typedef struct {
void* tqReader;
- void* meta;
void* config;
void* vnode;
void* mnd;
@@ -51,10 +51,10 @@ typedef struct {
bool initTableReader;
bool initTqReader;
int32_t numOfVgroups;
+ void* sContext; // SSnapContext*
- void* sContext; // SSnapContext*
-
- void* pStateBackend;
+ void* pStateBackend;
+ struct SStorageAPI api;
} SReadHandle;
// in queue mode, data streams are seperated by msg
diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h
new file mode 100644
index 0000000000..9ed47c94f8
--- /dev/null
+++ b/include/libs/executor/storageapi.h
@@ -0,0 +1,523 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef TDENGINE_STORAGEAPI_H
+#define TDENGINE_STORAGEAPI_H
+
+#include "tsimplehash.h"
+#include "tscalablebf.h"
+#include "taosdef.h"
+#include "tmsg.h"
+#include "tcommon.h"
+#include "index.h"
+#include "function.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TIMEWINDOW_RANGE_CONTAINED 1
+#define TIMEWINDOW_RANGE_EXTERNAL 2
+
+#define CACHESCAN_RETRIEVE_TYPE_ALL 0x1
+#define CACHESCAN_RETRIEVE_TYPE_SINGLE 0x2
+#define CACHESCAN_RETRIEVE_LAST_ROW 0x4
+#define CACHESCAN_RETRIEVE_LAST 0x8
+
+#define META_READER_NOLOCK 0x1
+
+typedef struct SMeta SMeta;
+typedef TSKEY (*GetTsFun)(void*);
+
+typedef struct SMetaEntry {
+ int64_t version;
+ int8_t type;
+ int8_t flags; // TODO: need refactor?
+ tb_uid_t uid;
+ char * name;
+ union {
+ struct {
+ SSchemaWrapper schemaRow;
+ SSchemaWrapper schemaTag;
+ SRSmaParam rsmaParam;
+ } stbEntry;
+ struct {
+ int64_t ctime;
+ int32_t ttlDays;
+ int32_t commentLen;
+ char * comment;
+ tb_uid_t suid;
+ uint8_t *pTags;
+ } ctbEntry;
+ struct {
+ int64_t ctime;
+ int32_t ttlDays;
+ int32_t commentLen;
+ char * comment;
+ int32_t ncid; // next column id
+ SSchemaWrapper schemaRow;
+ } ntbEntry;
+ struct {
+ STSma *tsma;
+ } smaEntry;
+ };
+
+ uint8_t *pBuf;
+} SMetaEntry;
+
+typedef struct SMetaReader {
+ int32_t flags;
+ void * pMeta;
+ SDecoder coder;
+ SMetaEntry me;
+ void * pBuf;
+ int32_t szBuf;
+ struct SStoreMeta* pAPI;
+} SMetaReader;
+
+typedef struct SMTbCursor {
+ void * pDbc;
+ void * pKey;
+ void * pVal;
+ int32_t kLen;
+ int32_t vLen;
+ SMetaReader mr;
+} SMTbCursor;
+
+typedef struct SRowBuffPos {
+ void* pRowBuff;
+ void* pKey;
+ bool beFlushed;
+ bool beUsed;
+} SRowBuffPos;
+
+// int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables);
+// int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
+// SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
+// int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
+// SArray *pTableUids);
+// void *tsdbCacherowsReaderClose(void *pReader);
+// int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
+
+// int32_t tsdbReaderOpen(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
+// SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables);
+// int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
+// void tsdbReaderSetId(STsdbReader *pReader, const char *idstr);
+// void tsdbReaderClose(STsdbReader *pReader);
+// int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
+// int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
+// void tsdbReleaseDataBlock(STsdbReader *pReader);
+// SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
+// int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
+// int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
+// int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
+// void *tsdbGetIdx(void *pMeta);
+// void *tsdbGetIvtIdx(void *pMeta);
+// uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader);
+// void tsdbReaderSetCloseFlag(STsdbReader *pReader);
+// int64_t tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr);
+
+// int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
+// int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
+// int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list);
+// void *vnodeGetIdx(void *pVnode);
+// void *vnodeGetIvtIdx(void *pVnode);
+
+// void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+// void metaReaderReleaseLock(SMetaReader *pReader);
+// void metaReaderClear(SMetaReader *pReader);
+// int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
+// int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
+// int metaGetTableEntryByName(SMetaReader *pReader, const char *name);
+// int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList);
+// int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList);
+// const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
+// int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
+//
+// int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
+// int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
+// int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
+// bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
+// int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
+// bool *acquired);
+// int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+// int32_t payloadLen, double selectivityRatio);
+// int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
+// tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
+// int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
+// int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
+// int32_t payloadLen);
+
+// tq
+typedef struct SMetaTableInfo {
+ int64_t suid;
+ int64_t uid;
+ SSchemaWrapper *schema;
+ char tbName[TSDB_TABLE_NAME_LEN];
+} SMetaTableInfo;
+
+typedef struct SSnapContext {
+ SMeta * pMeta; // todo remove it
+ int64_t snapVersion;
+ void * pCur;
+ int64_t suid;
+ int8_t subType;
+ SHashObj * idVersion;
+ SHashObj * suidInfo;
+ SArray * idList;
+ int32_t index;
+ bool withMeta;
+ bool queryMeta; // true-get meta, false-get data
+} SSnapContext;
+
+typedef struct {
+ int64_t uid;
+ int64_t ctbNum;
+} SMetaStbStats;
+
+// void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
+// int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
+// int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
+// int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
+// bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid);
+// bool tqCurrentBlockConsumed(const STqReader* pReader);
+// int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id);
+// bool tqNextBlockInWal(STqReader* pReader, const char* idstr);
+// bool tqNextBlockImpl(STqReader *pReader, const char* idstr);
+// int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
+// SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx);
+// int32_t setForSnapShot(SSnapContext *ctx, int64_t uid);
+// int32_t destroySnapContext(SSnapContext *ctx);
+
+/*-------------------------------------------------new api format---------------------------------------------------*/
+
+// typedef int32_t (*__store_reader_(STsdbReader *pReader, const void *pTableList, int32_t num);
+// typedef void (*tsdbReaderSetId(STsdbReader *pReader, const char *idstr);
+// typedef void (*tsdbReaderClose(STsdbReader *pReader);
+// typedef int32_t (*tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
+// typedef int32_t (*tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
+// typedef void (*tsdbReleaseDataBlock(STsdbReader *pReader);
+// typedef SSDataBlock * (*tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
+// typedef int32_t (*tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
+// typedef int32_t (*tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
+// typedef int64_t (*tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
+// typedef void * (*tsdbGetIdx(void *pMeta);
+// typedef void * (*tsdbGetIvtIdx(void *pMeta);
+// typedef uint64_t (*tsdbGetReaderMaxVersion(STsdbReader *pReader);
+// typedef void (*tsdbReaderSetCloseFlag(STsdbReader *pReader);
+// typedef int64_t (*tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr);
+
+typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *pCond, void *pTableList,
+ int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader,
+ const char *idstr, bool countOnly, SHashObj **pIgnoreTables);
+
+typedef struct TsdReader {
+ __store_reader_open_fn_t tsdReaderOpen;
+ void (*tsdReaderClose)();
+ void (*tsdSetReaderTaskId)(void *pReader, const char *pId);
+ int32_t (*tsdSetQueryTableList)();
+ int32_t (*tsdNextDataBlock)();
+
+ int32_t (*tsdReaderRetrieveBlockSMAInfo)();
+ SSDataBlock *(*tsdReaderRetrieveDataBlock)();
+
+ void (*tsdReaderReleaseDataBlock)();
+
+ int32_t (*tsdReaderResetStatus)();
+ int32_t (*tsdReaderGetDataBlockDistInfo)();
+ int64_t (*tsdReaderGetNumOfInMemRows)();
+ void (*tsdReaderNotifyClosing)();
+} TsdReader;
+
+
+/**
+ * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables);
+int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
+ SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
+int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
+ SArray *pTableUids);
+void *tsdbCacherowsReaderClose(void *pReader);
+ */
+typedef struct SStoreCacheReader {
+ int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
+ SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
+ void *(*closeReader)(void *pReader);
+ int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
+ SArray *pTableUidList);
+ int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables);
+} SStoreCacheReader;
+
+/*------------------------------------------------------------------------------------------------------------------*/
+/*
+void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
+int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
+int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
+int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
+bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid);
+bool tqCurrentBlockConsumed(const STqReader* pReader);
+
+int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id);
+bool tqNextBlockInWal(STqReader* pReader, const char* idstr);
+bool tqNextBlockImpl(STqReader *pReader, const char* idstr);
+
+ int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char* idstr);
+STqReader *tqReaderOpen(void *pVnode);
+void tqReaderClose(STqReader *);
+
+int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
+bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
+SWalReader* tqGetWalReader(STqReader* pReader);
+int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
+*/
+// todo rename
+typedef struct SStoreTqReader {
+ struct STqReader* (*tqReaderOpen)();
+ void (*tqReaderClose)();
+
+ int32_t (*tqReaderSeek)();
+ int32_t (*tqRetrieveBlock)();
+ bool (*tqReaderNextBlockInWal)();
+ bool (*tqNextBlockImpl)(); // todo remove it
+ SSDataBlock* (*tqGetResultBlock)();
+
+ void (*tqReaderSetColIdList)();
+ int32_t (*tqReaderSetQueryTableList)();
+
+ int32_t (*tqReaderAddTables)();
+ int32_t (*tqReaderRemoveTables)();
+
+ bool (*tqReaderIsQueriedTable)();
+ bool (*tqReaderCurrentBlockConsumed)();
+
+ struct SWalReader *(*tqReaderGetWalReader)(); // todo remove it
+ int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it
+
+ int32_t (*tqReaderSetSubmitMsg)(); // todo remove it
+ bool (*tqReaderNextBlockFilterOut)();
+} SStoreTqReader;
+
+typedef struct SStoreSnapshotFn {
+ /*
+ int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
+ SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx);
+ int32_t setForSnapShot(SSnapContext *ctx, int64_t uid);
+ int32_t destroySnapContext(SSnapContext *ctx);
+ */
+ int32_t (*createSnapshot)();
+ int32_t (*destroySnapshot)();
+ SMetaTableInfo (*getMetaTableInfoFromSnapshot)();
+ int32_t (*getTableInfoFromSnapshot)();
+} SStoreSnapshotFn;
+
+/**
+void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+void metaReaderReleaseLock(SMetaReader *pReader);
+void metaReaderClear(SMetaReader *pReader);
+int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
+int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
+int metaGetTableEntryByName(SMetaReader *pReader, const char *name);
+int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList);
+const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
+int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
+
+int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
+int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
+int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
+bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
+int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
+ bool *acquired);
+int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+ int32_t payloadLen, double selectivityRatio);
+int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
+tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
+int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
+int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
+ int32_t payloadLen);
+ */
+
+typedef struct SStoreMeta {
+ SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor
+ void (*closeTableMetaCursor)(SMTbCursor *pTbCur); // metaCloseTbCursor
+ int32_t (*cursorNext)(); // metaTbCursorNext
+ int32_t (*cursorPrev)(); // metaTbCursorPrev
+
+ int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList);
+ int32_t (*getTableTagsByUid)(void *pVnode, int64_t suid, SArray *uidList);
+ const void *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it
+
+ int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid);
+ int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType);
+ int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName);
+ bool (*isTableExisted)(void *pVnode, tb_uid_t uid);
+
+ /**
+ * int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+ int32_t payloadLen, double selectivityRatio);
+int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
+tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
+int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
+int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
+ int32_t payloadLen);
+ */
+ void (*getCachedTableList)();
+ void (*putTableListIntoCache)();
+
+ /**
+ *
+ */
+ void *(*storeGetIndexInfo)();
+ void *(*getInvertIndex)(void* pVnode);
+ int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
+ int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList
+ void *storeGetVersionRange;
+ void *storeGetLastTimestamp;
+
+ int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema
+
+ // db name, vgId, numOfTables, numOfSTables
+ int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo);
+ void (*getBasicInfo)(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);// vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta);
+
+ int64_t (*getNumOfRowsInMem)(void* pVnode);
+ /**
+int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
+int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
+int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list);
+ */
+} SStoreMeta;
+
+typedef struct SStoreMetaReader {
+ void (*initReader)(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI);
+ void (*clearReader)(SMetaReader *pReader);
+ void (*readerReleaseLock)(SMetaReader *pReader);
+ int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid);
+ int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name);
+ int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid);
+} SStoreMetaReader;
+
+typedef struct SUpdateInfo {
+ SArray *pTsBuckets;
+ uint64_t numBuckets;
+ SArray *pTsSBFs;
+ uint64_t numSBFs;
+ int64_t interval;
+ int64_t watermark;
+ TSKEY minTS;
+ SScalableBf *pCloseWinSBF;
+ SHashObj *pMap;
+ uint64_t maxDataVersion;
+} SUpdateInfo;
+
+typedef struct {
+ void* iter; // rocksdb_iterator_t* iter;
+ void* snapshot; // rocksdb_snapshot_t* snapshot;
+ void* readOpt; // rocksdb_readoptions_t* readOpt;
+ void* db; // rocksdb_t* db;
+ void* pCur;
+ int64_t number;
+} SStreamStateCur;
+
+typedef struct SStateStore {
+ int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
+ int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal);
+
+ int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+ int32_t (*streamStateReleaseBuf)(SStreamState* pState, const SWinKey* key, void* pVal);
+ void (*streamStateFreeVal)(void* val);
+
+ int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
+ int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+ bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key);
+ int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal);
+ int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key);
+ int32_t (*streamStateClear)(SStreamState* pState);
+ void (*streamStateSetNumber)(SStreamState* pState, int32_t number);
+ int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
+ int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
+
+ int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
+ int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+ int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key);
+
+ int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur);
+ int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur);
+
+ SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key);
+ SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key);
+ SStreamStateCur* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key);
+ SStreamStateCur* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key);
+ void (*streamStateFreeCur)(SStreamStateCur* pCur);
+
+ int32_t (*streamStateGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
+ int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
+
+ int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen);
+ int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen);
+ int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
+ int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key);
+ int32_t (*streamStateSessionClear)(SStreamState* pState);
+ int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
+ int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
+ state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
+ int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
+
+ SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark);
+ TSKEY (*updateInfoFillBlockData)(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol);
+ bool (*updateInfoIsUpdated)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts);
+ bool (*updateInfoIsTableInserted)(SUpdateInfo *pInfo, int64_t tbUid);
+ void (*updateInfoDestroy)(SUpdateInfo *pInfo);
+
+ SUpdateInfo* (*updateInfoInitP)(SInterval *pInterval, int64_t watermark);
+ void (*updateInfoAddCloseWindowSBF)(SUpdateInfo *pInfo);
+ void (*updateInfoDestoryColseWinSBF)(SUpdateInfo *pInfo);
+ int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const SUpdateInfo *pInfo);
+ int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, SUpdateInfo *pInfo);
+
+ SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);
+ SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key);
+ SStreamStateCur* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key);
+
+ struct SStreamFileState* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize,
+ uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark);
+
+ void (*streamFileStateDestroy)(struct SStreamFileState* pFileState);
+ void (*streamFileStateClear)(struct SStreamFileState* pFileState);
+ bool (*needClearDiskBuff)(struct SStreamFileState* pFileState);
+
+ SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
+ void (*streamStateClose)(SStreamState* pState, bool remove);
+ int32_t (*streamStateBegin)(SStreamState* pState);
+ int32_t (*streamStateCommit)(SStreamState* pState);
+ void (*streamStateDestroy)(SStreamState* pState, bool remove);
+ int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
+} SStateStore;
+
+typedef struct SStorageAPI {
+ SStoreMeta metaFn; // todo: refactor
+ TsdReader tsdReader;
+ SStoreMetaReader metaReaderFn;
+ SStoreCacheReader cacheFn;
+ SStoreSnapshotFn snapshotFn;
+ SStoreTqReader tqReaderFn;
+ SStateStore stateStore;
+ SMetaDataFilterAPI metaFilter;
+ SFunctionStateStore functionStore;
+} SStorageAPI;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TDENGINE_STORAGEAPI_H
diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 85f7cf7e2c..de2494ab3c 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -22,6 +22,7 @@ extern "C" {
#include "tcommon.h"
#include "tvariant.h"
+#include "tsimplehash.h"
struct SqlFunctionCtx;
struct SResultRowEntryInfo;
@@ -127,18 +128,59 @@ typedef struct SSerializeDataHandle {
void *pState;
} SSerializeDataHandle;
+// incremental state storage
+typedef struct STdbState {
+ void* rocksdb;
+ void** pHandle;
+ void* writeOpts;
+ void* readOpts;
+ void** cfOpts;
+ void* dbOpt;
+ struct SStreamTask* pOwner;
+ void* param;
+ void* env;
+ SListNode* pComparNode;
+ void* pBackendHandle;
+ char idstr[64];
+ void* compactFactory;
+
+ void* db;
+ void* pStateDb;
+ void* pFuncStateDb;
+ void* pFillStateDb; // todo refactor
+ void* pSessionStateDb;
+ void* pParNameDb;
+ void* pParTagDb;
+ void* txn;
+} STdbState;
+
+typedef struct {
+ STdbState* pTdbState;
+ struct SStreamFileState* pFileState;
+ int32_t number;
+ SSHashObj* parNameMap;
+ int64_t checkPointId;
+ int32_t taskId;
+ int64_t streamId;
+} SStreamState;
+
+typedef struct SFunctionStateStore {
+ int32_t (*streamStateFuncPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
+ int32_t (*streamStateFuncGet)(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
+} SFunctionStateStore;
+
// sql function runtime context
typedef struct SqlFunctionCtx {
SInputColumnInfoData input;
SResultDataInfo resDataInfo;
- uint32_t order; // data block scanner order: asc|desc
- uint8_t isPseudoFunc;// denote current function is pseudo function or not [added for perf reason]
- uint8_t isNotNullFunc;// not return null value.
- uint8_t scanFlag; // record current running step, default: 0
- int16_t functionId; // function id
- char *pOutput; // final result output buffer, point to sdata->data
+ uint32_t order; // data block scanner order: asc|desc
+ uint8_t isPseudoFunc; // denote current function is pseudo function or not [added for perf reason]
+ uint8_t isNotNullFunc; // not return null value.
+ uint8_t scanFlag; // record current running step, default: 0
+ int16_t functionId; // function id
+ char *pOutput; // final result output buffer, point to sdata->data
// input parameter, e.g., top(k, 20), the number of results of top query is kept in param
- SFunctParam *param;
+ SFunctParam *param;
// corresponding output buffer for timestamp of each result, e.g., diff/csum
SColumnInfoData *pTsOutput;
int32_t numOfParams;
@@ -155,6 +197,7 @@ typedef struct SqlFunctionCtx {
SSerializeDataHandle saveHandle;
int32_t exprIdx;
char *udfName;
+ SFunctionStateStore *pStore;
} SqlFunctionCtx;
typedef struct tExprNode {
diff --git a/include/libs/index/index.h b/include/libs/index/index.h
index 0d31ca2f68..cfcc9993cf 100644
--- a/include/libs/index/index.h
+++ b/include/libs/index/index.h
@@ -212,13 +212,38 @@ typedef struct SIndexMetaArg {
void* idx;
void* ivtIdx;
uint64_t suid;
+ int (*metaFilterFunc)(void* metaEx, void* param, SArray* result);
} SIndexMetaArg;
+/**
+ * the underlying storage module must implement this API to employ the index functions.
+ * @param pMeta
+ * @param param
+ * @param results
+ * @return
+ */
+typedef struct SMetaFltParam {
+ uint64_t suid;
+ int16_t cid;
+ int16_t type;
+ void *val;
+ bool reverse;
+ bool equal;
+ int (*filterFunc)(void *a, void *b, int16_t type);
+} SMetaFltParam;
+
+typedef struct SMetaDataFilterAPI {
+ int32_t (*metaFilterTableIds)(void *pVnode, SMetaFltParam *arg, SArray *pUids);
+ int32_t (*metaFilterCreateTime)(void *pVnode, SMetaFltParam *arg, SArray *pUids);
+ int32_t (*metaFilterTableName)(void *pVnode, SMetaFltParam *arg, SArray *pUids);
+ int32_t (*metaFilterTtl)(void *pVnode, SMetaFltParam *arg, SArray *pUids);
+} SMetaDataFilterAPI;
+
typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus;
-SIdxFltStatus idxGetFltStatus(SNode* pFilterNode);
+SIdxFltStatus idxGetFltStatus(SNode* pFilterNode, SMetaDataFilterAPI* pAPI);
-int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status);
+int32_t doFilterTag(SNode* pFilterNode, SIndexMetaArg* metaArg, SArray* result, SIdxFltStatus* status, SMetaDataFilterAPI* pAPI);
/*
* init index env
diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h
index 66e8b7b2a3..7f9d20a9dd 100644
--- a/include/libs/stream/streamState.h
+++ b/include/libs/stream/streamState.h
@@ -27,66 +27,63 @@
extern "C" {
#endif
+#include "storageapi.h"
+
// void* streamBackendInit(const char* path);
// void streamBackendCleanup(void* arg);
// SListNode* streamBackendAddCompare(void* backend, void* arg);
// void streamBackendDelCompare(void* backend, void* arg);
-typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
-typedef struct STdbState {
- rocksdb_t* rocksdb;
- rocksdb_column_family_handle_t** pHandle;
- rocksdb_writeoptions_t* writeOpts;
- rocksdb_readoptions_t* readOpts;
- rocksdb_options_t** cfOpts;
- rocksdb_options_t* dbOpt;
- struct SStreamTask* pOwner;
- void* param;
- void* env;
- SListNode* pComparNode;
- void* pBackend;
- char idstr[64];
- void* compactFactory;
- TdThreadRwlock rwLock;
+// <<<<<<< HEAD
+// typedef struct STdbState {
+// rocksdb_t* rocksdb;
+// rocksdb_column_family_handle_t** pHandle;
+// rocksdb_writeoptions_t* writeOpts;
+// rocksdb_readoptions_t* readOpts;
+// rocksdb_options_t** cfOpts;
+// rocksdb_options_t* dbOpt;
+// struct SStreamTask* pOwner;
+// void* param;
+// void* env;
+// SListNode* pComparNode;
+// void* pBackend;
+// char idstr[64];
+// void* compactFactory;
+// TdThreadRwlock rwLock;
+// =======
+// typedef struct STdbState {
+// rocksdb_t* rocksdb;
+// rocksdb_column_family_handle_t** pHandle;
+// rocksdb_writeoptions_t* writeOpts;
+// rocksdb_readoptions_t* readOpts;
+// rocksdb_options_t** cfOpts;
+// rocksdb_options_t* dbOpt;
+// struct SStreamTask* pOwner;
+// void* param;
+// void* env;
+// SListNode* pComparNode;
+// void* pBackendHandle;
+// char idstr[64];
+// void* compactFactory;
+//
+// TDB* db;
+// TTB* pStateDb;
+// TTB* pFuncStateDb;
+// TTB* pFillStateDb; // todo refactor
+// TTB* pSessionStateDb;
+// TTB* pParNameDb;
+// TTB* pParTagDb;
+// TXN* txn;
+//} STdbState;
+//>>>>>>> enh/dev3.0
- TDB* db;
- TTB* pStateDb;
- TTB* pFuncStateDb;
- TTB* pFillStateDb; // todo refactor
- TTB* pSessionStateDb;
- TTB* pParNameDb;
- TTB* pParTagDb;
- TXN* txn;
-} STdbState;
-
-// incremental state storage
-typedef struct {
- STdbState* pTdbState;
- SStreamFileState* pFileState;
- int32_t number;
- SSHashObj* parNameMap;
- int64_t checkPointId;
- int32_t taskId;
- int64_t streamId;
-} SStreamState;
-
-SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages);
+SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState);
int32_t streamStateCommit(SStreamState* pState);
void streamStateDestroy(SStreamState* pState, bool remove);
int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
-typedef struct {
- rocksdb_iterator_t* iter;
- rocksdb_snapshot_t* snapshot;
- rocksdb_readoptions_t* readOpt;
- rocksdb_t* db;
-
- TBC* pCur;
- int64_t number;
-} SStreamStateCur;
-
int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
@@ -120,7 +117,7 @@ int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal);
-void streamFreeVal(void* val);
+void streamStateFreeVal(void* val);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key);
diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h
index 7124e2d251..0dbacf6c9f 100644
--- a/include/libs/stream/tstreamFileState.h
+++ b/include/libs/stream/tstreamFileState.h
@@ -21,23 +21,16 @@
#include "tarray.h"
#include "tdef.h"
#include "tlist.h"
+#include "storageapi.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SStreamFileState SStreamFileState;
-typedef struct SRowBuffPos {
- void* pRowBuff;
- void* pKey;
- bool beFlushed;
- bool beUsed;
-} SRowBuffPos;
typedef SList SStreamSnapshot;
-typedef TSKEY (*GetTsFun)(void*);
-
SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
GetTsFun fp, void* pFile, TSKEY delMark);
void streamFileStateDestroy(SStreamFileState* pFileState);
diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h
index 4678aa0bd9..bd5a3be8de 100644
--- a/include/libs/stream/tstreamUpdate.h
+++ b/include/libs/stream/tstreamUpdate.h
@@ -19,7 +19,7 @@
#include "tarray.h"
#include "tcommon.h"
#include "tmsg.h"
-#include "tscalablebf.h"
+#include "storageapi.h"
#ifdef __cplusplus
extern "C" {
@@ -30,18 +30,18 @@ typedef struct SUpdateKey {
TSKEY ts;
} SUpdateKey;
-typedef struct SUpdateInfo {
- SArray *pTsBuckets;
- uint64_t numBuckets;
- SArray *pTsSBFs;
- uint64_t numSBFs;
- int64_t interval;
- int64_t watermark;
- TSKEY minTS;
- SScalableBf *pCloseWinSBF;
- SHashObj *pMap;
- uint64_t maxDataVersion;
-} SUpdateInfo;
+//typedef struct SUpdateInfo {
+// SArray *pTsBuckets;
+// uint64_t numBuckets;
+// SArray *pTsSBFs;
+// uint64_t numSBFs;
+// int64_t interval;
+// int64_t watermark;
+// TSKEY minTS;
+// SScalableBf *pCloseWinSBF;
+// SHashObj *pMap;
+// uint64_t maxDataVersion;
+//} SUpdateInfo;
SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 636a5e63a7..9e5229870e 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -409,6 +409,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MNODE_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0413) // internal
#define TSDB_CODE_MNODE_ONLY_TWO_MNODE TAOS_DEF_ERROR_CODE(0, 0x0414) // internal
#define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal
+#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416)
// vnode
// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x
diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h
index b20fc6f57a..c9eb951014 100644
--- a/source/client/inc/clientSml.h
+++ b/source/client/inc/clientSml.h
@@ -232,9 +232,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *
int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
//SArray *smlJsonParseTags(char *start, char *end);
bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
-void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
-int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
-int nodeListSize(NodeList* list);
+//void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
+//int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
+//int nodeListSize(NodeList* list);
bool smlDoubleToInt64OverFlow(double num);
int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
@@ -251,7 +251,7 @@ int32_t smlClearForRerun(SSmlHandle *info);
int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg);
uint8_t smlGetTimestampLen(int64_t num);
void clearColValArray(SArray* pCols);
-void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag);
+void smlDestroyTableInfo(void *para);
void freeSSmlKv(void* data);
int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 82177302c7..5963e419e1 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -1757,6 +1757,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
+ taosMemoryFreeClear(pResultInfo->convertJson);
pResultInfo->convertJson = taosMemoryCalloc(1, dataLen);
if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
char* p1 = pResultInfo->convertJson;
diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 3ea471fbf8..bea237d09e 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -24,72 +24,91 @@ int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL};
int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL};
-void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
- NodeList *tmp = list;
- while (tmp) {
- if (fn == NULL) {
- if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
- return tmp->data.value;
- }
- } else {
- if (tmp->data.used && fn(tmp->data.key, key) == 0) {
- return tmp->data.value;
- }
- }
+//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
+// NodeList *tmp = list;
+// while (tmp) {
+// if (fn == NULL) {
+// if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
+// return tmp->data.value;
+// }
+// } else {
+// if (tmp->data.used && fn(tmp->data.key, key) == 0) {
+// return tmp->data.value;
+// }
+// }
+//
+// tmp = tmp->next;
+// }
+// return NULL;
+//}
+//
+//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
+// NodeList *tmp = *list;
+// while (tmp) {
+// if (!tmp->data.used) break;
+// if (fn == NULL) {
+// if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
+// return -1;
+// }
+// } else {
+// if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
+// return -1;
+// }
+// }
+//
+// tmp = tmp->next;
+// }
+// if (tmp) {
+// tmp->data.key = key;
+// tmp->data.keyLen = len;
+// tmp->data.value = value;
+// tmp->data.used = true;
+// } else {
+// NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
+// if (newNode == NULL) {
+// return -1;
+// }
+// newNode->data.key = key;
+// newNode->data.keyLen = len;
+// newNode->data.value = value;
+// newNode->data.used = true;
+// newNode->next = *list;
+// *list = newNode;
+// }
+// return 0;
+//}
+//
+//int nodeListSize(NodeList *list) {
+// int cnt = 0;
+// while (list) {
+// if (list->data.used)
+// cnt++;
+// else
+// break;
+// list = list->next;
+// }
+// return cnt;
+//}
- tmp = tmp->next;
- }
- return NULL;
-}
-
-int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
- NodeList *tmp = *list;
- while (tmp) {
- if (!tmp->data.used) break;
- if (fn == NULL) {
- if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
- return -1;
- }
- } else {
- if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
- return -1;
- }
- }
-
- tmp = tmp->next;
- }
- if (tmp) {
- tmp->data.key = key;
- tmp->data.keyLen = len;
- tmp->data.value = value;
- tmp->data.used = true;
+static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){
+ SUserAuthInfo pAuth = {0};
+ snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user);
+ if (NULL == pTabName) {
+ tNameSetDbName(&pAuth.tbName, info->taos->acctId, info->pRequest->pDb, strlen(info->pRequest->pDb));
} else {
- NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
- if (newNode == NULL) {
- return -1;
- }
- newNode->data.key = key;
- newNode->data.keyLen = len;
- newNode->data.value = value;
- newNode->data.used = true;
- newNode->next = *list;
- *list = newNode;
+ toName(info->taos->acctId, info->pRequest->pDb, pTabName, &pAuth.tbName);
}
- return 0;
-}
+ pAuth.type = type;
-int nodeListSize(NodeList *list) {
- int cnt = 0;
- while (list) {
- if (list->data.used)
- cnt++;
- else
- break;
- list = list->next;
- }
- return cnt;
-}
+ int32_t code = TSDB_CODE_SUCCESS;
+ SUserAuthRes authRes = {0};
+ code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes);
+
+
+ return (code == TSDB_CODE_SUCCESS) ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
+
+}
inline bool smlDoubleToInt64OverFlow(double num) {
if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
return false;
@@ -211,6 +230,16 @@ void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tin
}
}
+static void smlDestroySTableMeta(void *para) {
+ SSmlSTableMeta *meta = *(SSmlSTableMeta**)para;
+ taosHashCleanup(meta->tagHash);
+ taosHashCleanup(meta->colHash);
+ taosArrayDestroy(meta->tags);
+ taosArrayDestroy(meta->cols);
+ taosMemoryFreeClear(meta->tableMeta);
+ taosMemoryFree(meta);
+}
+
SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) {
SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1);
if (!meta) {
@@ -245,7 +274,7 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) {
return meta;
cleanup:
- taosMemoryFree(meta);
+ smlDestroySTableMeta(meta);
return NULL;
}
@@ -813,6 +842,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) {
+ code = smlCheckAuth(info, &conn, NULL, AUTH_TYPE_WRITE);
+ if(code != TSDB_CODE_SUCCESS){
+ goto end;
+ }
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname);
SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField));
SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField));
@@ -857,6 +890,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end;
}
if (action != SCHEMA_ACTION_NULL) {
+ code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
+ if(code != TSDB_CODE_SUCCESS){
+ goto end;
+ }
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname,
action);
SArray *pColumns =
@@ -927,6 +964,10 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end;
}
if (action != SCHEMA_ACTION_NULL) {
+ code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
+ if(code != TSDB_CODE_SUCCESS){
+ goto end;
+ }
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname,
action);
SArray *pColumns =
@@ -1004,6 +1045,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
}
}
+ taosMemoryFreeClear(sTableData->tableMeta);
sTableData->tableMeta = pTableMeta;
uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid,
pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
@@ -1062,15 +1104,6 @@ static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) {
}
}
-static void smlDestroySTableMeta(SSmlSTableMeta *meta) {
- taosHashCleanup(meta->tagHash);
- taosHashCleanup(meta->colHash);
- taosArrayDestroy(meta->tags);
- taosArrayDestroy(meta->cols);
- taosMemoryFree(meta->tableMeta);
- taosMemoryFree(meta);
-}
-
static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) {
for (int i = 0; i < taosArrayGetSize(cols); ++i) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
@@ -1110,7 +1143,8 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols
return TSDB_CODE_SUCCESS;
}
-void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
+void smlDestroyTableInfo(void *para) {
+ SSmlTableInfo *tag = *(SSmlTableInfo**)para;
for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) {
SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i);
taosHashCleanup(kvHash);
@@ -1147,18 +1181,18 @@ void smlDestroyInfo(SSmlHandle *info) {
qDestroyQuery(info->pQuery);
// destroy info->childTables
- SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
- while (oneTable) {
- smlDestroyTableInfo(info, *oneTable);
- oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
- }
+// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
+// while (oneTable) {
+// smlDestroyTableInfo(oneTable);
+// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
+// }
// destroy info->superTables
- SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
- while (oneSTable) {
- smlDestroySTableMeta(*oneSTable);
- oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
- }
+// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
+// while (oneSTable) {
+// smlDestroySTableMeta(*oneSTable);
+// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
+// }
// destroy info->pVgHash
taosHashCleanup(info->pVgHash);
@@ -1217,6 +1251,8 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+ taosHashSetFreeFp(info->superTables, smlDestroySTableMeta);
+ taosHashSetFreeFp(info->childTables, smlDestroyTableInfo);
info->id = smlGenId();
info->pQuery = smlInitHandle();
@@ -1323,6 +1359,9 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
info->lineNum);
SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
+ if(meta == NULL){
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
terrno = 0;
smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
@@ -1367,6 +1406,11 @@ static int32_t smlInsertData(SSmlHandle *info) {
conn.requestObjRefId = info->pRequest->self;
conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
+ code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
+ if(code != TSDB_CODE_SUCCESS){
+ return code;
+ }
+
SVgroupInfo vg;
code = catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg);
if (code != TSDB_CODE_SUCCESS) {
@@ -1437,18 +1481,18 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
int32_t smlClearForRerun(SSmlHandle *info) {
info->reRun = false;
// clear info->childTables
- SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
- while (oneTable) {
- smlDestroyTableInfo(info, *oneTable);
- oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
- }
+// SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
+// while (oneTable) {
+// smlDestroyTableInfo(info, *oneTable);
+// oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
+// }
// clear info->superTables
- SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
- while (oneSTable) {
- smlDestroySTableMeta(*oneSTable);
- oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
- }
+// SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
+// while (oneSTable) {
+// smlDestroySTableMeta(*oneSTable);
+// oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
+// }
taosHashClear(info->childTables);
taosHashClear(info->superTables);
@@ -1586,9 +1630,7 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
do {
code = smlModifyDBSchemas(info);
- if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA || code == TSDB_CODE_PAR_TOO_MANY_COLUMNS
- || code == TSDB_CODE_PAR_INVALID_TAGS_NUM || code == TSDB_CODE_PAR_INVALID_TAGS_LENGTH
- || code == TSDB_CODE_PAR_INVALID_ROW_LENGTH || code == TSDB_CODE_MND_FIELD_VALUE_OVERFLOW) {
+ if (code != TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER && code != TSDB_CODE_SDB_OBJ_CREATING && code != TSDB_CODE_MND_TRANS_CONFLICT) {
break;
}
taosMsleep(100);
diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c
index 7ccf930964..0f59505f8c 100644
--- a/source/client/src/clientSmlJson.c
+++ b/source/client/src/clientSmlJson.c
@@ -695,6 +695,10 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
return TSDB_CODE_SUCCESS;
}
sMeta = smlBuildSTableMeta(info->dataFormat);
+ if(sMeta == NULL){
+ taosMemoryFreeClear(pTableMeta);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
sMeta->tableMeta = pTableMeta;
taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
@@ -784,7 +788,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
if (tinfo->tableDataCtx == NULL) {
smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
- smlDestroyTableInfo(info, tinfo);
+ smlDestroyTableInfo(&tinfo);
return TSDB_CODE_SML_INVALID_DATA;
}
}
@@ -1048,12 +1052,18 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
return TSDB_CODE_TSC_INVALID_JSON;
}
- info->lineNum = payloadNum;
- info->dataFormat = true;
+
if (unlikely(info->lines != NULL)) {
+ for (int i = 0; i < info->lineNum; i++) {
+ taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
+ if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
+ }
taosMemoryFree(info->lines);
info->lines = NULL;
}
+ info->lineNum = payloadNum;
+ info->dataFormat = true;
+
ret = smlClearForRerun(info);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c
index e79093398e..40e014458e 100644
--- a/source/client/src/clientSmlLine.c
+++ b/source/client/src/clientSmlLine.c
@@ -168,6 +168,10 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return TSDB_CODE_SUCCESS;
}
sMeta = smlBuildSTableMeta(info->dataFormat);
+ if(sMeta == NULL){
+ taosMemoryFreeClear(pTableMeta);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
sMeta->tableMeta = pTableMeta;
taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
for (int i = pTableMeta->tableInfo.numOfColumns;
@@ -326,7 +330,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
info->currSTableMeta->uid = tinfo->uid;
tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
if (tinfo->tableDataCtx == NULL) {
- smlDestroyTableInfo(info, tinfo);
+ smlDestroyTableInfo(&tinfo);
smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
return TSDB_CODE_SML_INVALID_DATA;
}
@@ -372,6 +376,10 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
return TSDB_CODE_SUCCESS;
}
*tmp = smlBuildSTableMeta(info->dataFormat);
+ if(*tmp == NULL){
+ taosMemoryFreeClear(pTableMeta);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
(*tmp)->tableMeta = pTableMeta;
taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES);
diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c
index 42b8001e59..b3f45a3107 100644
--- a/source/client/src/clientSmlTelnet.c
+++ b/source/client/src/clientSmlTelnet.c
@@ -91,6 +91,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
return TSDB_CODE_SUCCESS;
}
sMeta = smlBuildSTableMeta(info->dataFormat);
+ if(sMeta == NULL){
+ taosMemoryFreeClear(pTableMeta);
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
sMeta->tableMeta = pTableMeta;
taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
@@ -212,7 +216,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
if (tinfo->tableDataCtx == NULL) {
smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
- smlDestroyTableInfo(info, tinfo);
+ smlDestroyTableInfo(&tinfo);
return TSDB_CODE_SML_INVALID_DATA;
}
}
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index aff5846092..59d411cde8 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -27,6 +27,8 @@
#define EMPTY_BLOCK_POLL_IDLE_DURATION 10
#define DEFAULT_AUTO_COMMIT_INTERVAL 5000
+#define OFFSET_IS_RESET_OFFSET(_of) ((_of) < 0)
+
typedef void (*__tmq_askep_fn_t)(tmq_t* pTmq, int32_t code, SDataBuf* pBuf, void* pParam);
struct SMqMgmt {
@@ -2626,12 +2628,12 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo;
int32_t type = pOffsetInfo->currentOffset.type;
- if (type != TMQ_OFFSET__LOG) {
+ if (type != TMQ_OFFSET__LOG && !OFFSET_IS_RESET_OFFSET(type)) {
tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type);
return TSDB_CODE_INVALID_PARA;
}
- if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) {
+ if (type == TMQ_OFFSET__LOG && (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd)) {
tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]",
tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd);
return TSDB_CODE_INVALID_PARA;
diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp
index b04727bfc0..ccc17289b0 100644
--- a/source/client/test/clientTests.cpp
+++ b/source/client/test/clientTests.cpp
@@ -1100,7 +1100,7 @@ TEST(clientCase, sub_tb_test) {
// 创建订阅 topics 列表
tmq_list_t* topicList = tmq_list_new();
- tmq_list_append(topicList, "topic_t1");
+ tmq_list_append(topicList, "t1");
// 启动订阅
tmq_subscribe(tmq, topicList);
@@ -1118,7 +1118,7 @@ TEST(clientCase, sub_tb_test) {
tmq_topic_assignment* pAssign = NULL;
int32_t numOfAssign = 0;
- int32_t code = tmq_get_topic_assignment(tmq, "topic_t1", &pAssign, &numOfAssign);
+ int32_t code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign);
if (code != 0) {
printf("error occurs:%s\n", tmq_err2str(code));
tmq_consumer_close(tmq);
@@ -1127,7 +1127,16 @@ TEST(clientCase, sub_tb_test) {
return;
}
- tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgId, 0);
+ tmq_offset_seek(tmq, "t1", pAssign[0].vgId, 4);
+
+ code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign);
+ if (code != 0) {
+ printf("error occurs:%s\n", tmq_err2str(code));
+ tmq_consumer_close(tmq);
+ taos_close(pConn);
+ fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
+ return;
+ }
while (1) {
TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout);
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index cee3b3c61d..99694ae4b8 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -986,8 +986,15 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
}
int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id);
- if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !force) {
- if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
+ if (numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) {
+ bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs());
+ if (isonline && force) {
+ terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE;
+ mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(),
+ numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
+ goto _OVER;
+ }
+ if (!isonline && !force) {
terrno = TSDB_CODE_DNODE_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(),
numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c
index 19c3d59167..d0b10a5768 100644
--- a/source/dnode/mnode/impl/src/mndMnode.c
+++ b/source/dnode/mnode/impl/src/mndMnode.c
@@ -695,7 +695,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
if (totalMnodes == 2) {
if (force) {
mError("cant't force drop dnode, since a mnode on it and replica is 2");
- terrno = TSDB_CODE_DNODE_OFFLINE;
+ terrno = TSDB_CODE_MNODE_ONLY_TWO_MNODE;
return -1;
}
mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c
index 5278fc7761..8e95fa3d6d 100644
--- a/source/dnode/mnode/impl/src/mndQuery.c
+++ b/source/dnode/mnode/impl/src/mndQuery.c
@@ -33,6 +33,7 @@ void mndPostProcessQueryMsg(SRpcMsg *pMsg) {
int32_t mndProcessQueryMsg(SRpcMsg *pMsg) {
int32_t code = -1;
SMnode *pMnode = pMsg->info.node;
+
SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb};
mTrace("msg:%p, in query queue is processing", pMsg);
diff --git a/source/dnode/qnode/CMakeLists.txt b/source/dnode/qnode/CMakeLists.txt
index 5426cd55d3..10bbbc1b26 100644
--- a/source/dnode/qnode/CMakeLists.txt
+++ b/source/dnode/qnode/CMakeLists.txt
@@ -14,4 +14,7 @@ target_link_libraries(
PRIVATE qworker
PRIVATE qcom
PRIVATE executor
+ PRIVATE tdb
+ PRIVATE wal
+ PRIVATE index
)
\ No newline at end of file
diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c
index 5efc714e95..3482355512 100644
--- a/source/dnode/qnode/src/qnode.c
+++ b/source/dnode/qnode/src/qnode.c
@@ -14,7 +14,6 @@
*/
#include "executor.h"
-#include "libs/function/function.h"
#include "qndInt.h"
#include "query.h"
#include "qworker.h"
diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt
index 6f1e7f9593..ebfe80ecab 100644
--- a/source/dnode/snode/CMakeLists.txt
+++ b/source/dnode/snode/CMakeLists.txt
@@ -14,4 +14,6 @@ target_link_libraries(
PRIVATE util
PRIVATE qcom
PRIVATE stream
+ PRIVATE wal
+ PRIVATE index
)
diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h
index 3fcee862a1..68f7f756d5 100644
--- a/source/dnode/snode/inc/sndInt.h
+++ b/source/dnode/snode/inc/sndInt.h
@@ -53,6 +53,8 @@ int32_t sndStopTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
int32_t sndResumeTaskOfStream(SStreamMeta* pMeta, int64_t streamId);
#endif
+void initStreamStateAPI(SStorageAPI* pAPI);
+
#ifdef __cplusplus
}
#endif
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index e9225f3d6e..678dd34e4a 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -62,8 +62,7 @@ FAIL:
}
int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
- ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
- ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0);
+ ASSERT(pTask->taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->childEpInfo) != 0);
pTask->refCnt = 1;
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
@@ -87,9 +86,10 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) {
}
int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo);
- SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState };
+ SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState };
+ initStreamStateAPI(&handle.api);
- pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0);
+ pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0);
ASSERT(pTask->exec.pExecutor);
streamSetupTrigger(pTask);
diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c
new file mode 100644
index 0000000000..f5e9245252
--- /dev/null
+++ b/source/dnode/snode/src/snodeInitApi.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "storageapi.h"
+#include "tstreamUpdate.h"
+#include "streamState.h"
+
+static void initStateStoreAPI(SStateStore* pStore);
+static void initFunctionStateStore(SFunctionStateStore* pStore);
+
+void initStreamStateAPI(SStorageAPI* pAPI) {
+ initStateStoreAPI(&pAPI->stateStore);
+ initFunctionStateStore(&pAPI->functionStore);
+}
+
+void initStateStoreAPI(SStateStore* pStore) {
+ pStore->streamFileStateInit = streamFileStateInit;
+ pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF;
+
+ pStore->streamStateGetByPos = streamStateGetByPos;
+
+ pStore->streamStatePutParName = streamStatePutParName;
+ pStore->streamStateGetParName = streamStateGetParName;
+
+ pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
+ pStore->streamStateReleaseBuf = streamStateReleaseBuf;
+ pStore->streamStateFreeVal = streamStateFreeVal;
+
+ pStore->streamStatePut = streamStatePut;
+ pStore->streamStateGet = streamStateGet;
+ pStore->streamStateCheck = streamStateCheck;
+ pStore->streamStateGetByPos = streamStateGetByPos;
+ pStore->streamStateDel = streamStateDel;
+ pStore->streamStateClear = streamStateClear;
+ pStore->streamStateSaveInfo = streamStateSaveInfo;
+ pStore->streamStateGetInfo = streamStateGetInfo;
+ pStore->streamStateSetNumber = streamStateSetNumber;
+
+ pStore->streamStateFillPut = streamStateFillPut;
+ pStore->streamStateFillGet = streamStateFillGet;
+ pStore->streamStateFillDel = streamStateFillDel;
+
+ pStore->streamStateCurNext = streamStateCurNext;
+ pStore->streamStateCurPrev = streamStateCurPrev;
+
+ pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur;
+ pStore->streamStateSeekKeyNext = streamStateSeekKeyNext;
+ pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext;
+ pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev;
+ pStore->streamStateFreeCur = streamStateFreeCur;
+
+ pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur;
+ pStore->streamStateGetKVByCur = streamStateGetKVByCur;
+
+ pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
+ pStore->streamStateSessionPut = streamStateSessionPut;
+ pStore->streamStateSessionGet = streamStateSessionGet;
+ pStore->streamStateSessionDel = streamStateSessionDel;
+ pStore->streamStateSessionClear = streamStateSessionClear;
+ pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur;
+ pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist;
+ pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
+
+ pStore->updateInfoInit = updateInfoInit;
+ pStore->updateInfoFillBlockData = updateInfoFillBlockData;
+ pStore->updateInfoIsUpdated = updateInfoIsUpdated;
+ pStore->updateInfoIsTableInserted = updateInfoIsTableInserted;
+ pStore->updateInfoDestroy = updateInfoDestroy;
+
+ pStore->updateInfoInitP = updateInfoInitP;
+ pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF;
+ pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF;
+ pStore->updateInfoSerialize = updateInfoSerialize;
+ pStore->updateInfoDeserialize = updateInfoDeserialize;
+
+ pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
+ pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
+ pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext;
+
+ pStore->streamFileStateInit = streamFileStateInit;
+
+ pStore->streamFileStateDestroy = streamFileStateDestroy;
+ pStore->streamFileStateClear = streamFileStateClear;
+ pStore->needClearDiskBuff = needClearDiskBuff;
+
+ pStore->streamStateOpen = streamStateOpen;
+ pStore->streamStateClose = streamStateClose;
+ pStore->streamStateBegin = streamStateBegin;
+ pStore->streamStateCommit = streamStateCommit;
+ pStore->streamStateDestroy= streamStateDestroy;
+ pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
+}
+
+void initFunctionStateStore(SFunctionStateStore* pStore) {
+ pStore->streamStateFuncPut = streamStateFuncPut;
+ pStore->streamStateFuncGet = streamStateFuncGet;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index e8660cd6ad..04fad2d78e 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -15,6 +15,7 @@ target_sources(
"src/vnd/vnodeSync.c"
"src/vnd/vnodeSnapshot.c"
"src/vnd/vnodeRetention.c"
+ "src/vnd/vnodeInitApi.c"
# meta
"src/meta/metaOpen.c"
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 18fa893fa4..72c8dce52f 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -26,6 +26,7 @@
#include "tfs.h"
#include "wal.h"
+#include "filter.h"
#include "tcommon.h"
#include "tfs.h"
#include "tgrant.h"
@@ -33,6 +34,7 @@
#include "trow.h"
#include "tdb.h"
+#include "storageapi.h"
#ifdef __cplusplus
extern "C" {
@@ -65,17 +67,18 @@ int32_t vnodeStart(SVnode *pVnode);
void vnodeStop(SVnode *pVnode);
int64_t vnodeGetSyncHandle(SVnode *pVnode);
void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
-void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
+void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);
int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
+int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList);
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
int32_t vnodeIsCatchUp(SVnode *pVnode);
ESyncRole vnodeGetRole(SVnode *pVnode);
-int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list);
+int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list);
-void *vnodeGetIdx(SVnode *pVnode);
-void *vnodeGetIvtIdx(SVnode *pVnode);
+void *vnodeGetIdx(void *pVnode);
+void *vnodeGetIvtIdx(void *pVnode);
int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num);
int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num);
@@ -103,57 +106,40 @@ typedef struct SMetaEntry SMetaEntry;
#define META_READER_NOLOCK 0x1
-void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI);
void metaReaderReleaseLock(SMetaReader *pReader);
void metaReaderClear(SMetaReader *pReader);
-int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
-int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
+int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
+int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
int metaGetTableEntryByName(SMetaReader *pReader, const char *name);
-int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList);
-int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList);
+int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *uidList);
+int32_t metaGetTableTagsByUids(void* pVnode, int64_t suid, SArray *uidList);
int32_t metaReadNext(SMetaReader *pReader);
-const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
+const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal);
int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
-int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
+int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid);
int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
-bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
+bool metaIsTableExist(void* pVnode, tb_uid_t uid);
int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
bool *acquired);
int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen, double selectivityRatio);
int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
-int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid);
-int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
-int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
- int32_t payloadLen);
+int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid);
+int32_t metaGetCachedTbGroup(SMeta *pMeta, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
+int32_t metaPutTbGroupToCache(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+ int32_t payloadLen);
int64_t metaGetTbNum(SMeta *pMeta);
int64_t metaGetNtbNum(SMeta *pMeta);
-typedef struct {
- int64_t uid;
- int64_t ctbNum;
-} SMetaStbStats;
-int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo);
-
-typedef struct SMetaFltParam {
- tb_uid_t suid;
- int16_t cid;
- int16_t type;
- void *val;
- bool reverse;
- bool equal;
- int (*filterFunc)(void *a, void *b, int16_t type);
-
-} SMetaFltParam;
-
-// TODO, refactor later
-int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results);
-int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids);
-int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids);
-int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids);
+//typedef struct {
+// int64_t uid;
+// int64_t ctbNum;
+//} SMetaStbStats;
+int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables);
#if 1 // refact APIs below (TODO)
typedef SVCreateTbReq STbCfg;
@@ -161,7 +147,7 @@ typedef SVCreateTSmaReq SSmaCfg;
typedef struct SMTbCursor SMTbCursor;
-SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
+SMTbCursor *metaOpenTbCursor(void *pVnode);
void metaCloseTbCursor(SMTbCursor *pTbCur);
int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType);
int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType);
@@ -183,7 +169,8 @@ typedef struct STsdbReader STsdbReader;
#define CACHESCAN_RETRIEVE_LAST 0x8
int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
- SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables);
+ SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly,
+ SHashObj **pIgnoreTables);
int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
void tsdbReaderSetId(STsdbReader *pReader, const char *idstr);
void tsdbReaderClose(STsdbReader *pReader);
@@ -198,47 +185,47 @@ void *tsdbGetIdx(SMeta *pMeta);
void *tsdbGetIvtIdx(SMeta *pMeta);
uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader);
void tsdbReaderSetCloseFlag(STsdbReader *pReader);
-int64_t tsdbGetLastTimestamp(SVnode* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr);
+int64_t tsdbGetLastTimestamp(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
-int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables);
+int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUids);
void *tsdbCacherowsReaderClose(void *pReader);
-int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
+int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode);
size_t tsdbCacheGetUsage(SVnode *pVnode);
int32_t tsdbCacheGetElems(SVnode *pVnode);
-// tq
-typedef struct SMetaTableInfo {
- int64_t suid;
- int64_t uid;
- SSchemaWrapper *schema;
- char tbName[TSDB_TABLE_NAME_LEN];
-} SMetaTableInfo;
+//// tq
+//typedef struct SMetaTableInfo {
+// int64_t suid;
+// int64_t uid;
+// SSchemaWrapper *schema;
+// char tbName[TSDB_TABLE_NAME_LEN];
+//} SMetaTableInfo;
typedef struct SIdInfo {
int64_t version;
int32_t index;
} SIdInfo;
-typedef struct SSnapContext {
- SMeta *pMeta;
- int64_t snapVersion;
- TBC *pCur;
- int64_t suid;
- int8_t subType;
- SHashObj *idVersion;
- SHashObj *suidInfo;
- SArray *idList;
- int32_t index;
- bool withMeta;
- bool queryMeta; // true-get meta, false-get data
-} SSnapContext;
+//typedef struct SSnapContext {
+// SMeta *pMeta;
+// int64_t snapVersion;
+// TBC *pCur;
+// int64_t suid;
+// int8_t subType;
+// SHashObj *idVersion;
+// SHashObj *suidInfo;
+// SArray *idList;
+// int32_t index;
+// bool withMeta;
+// bool queryMeta; // true-get meta, false-get data
+//} SSnapContext;
typedef struct STqReader {
SPackedData msg;
@@ -257,21 +244,26 @@ typedef struct STqReader {
} STqReader;
STqReader *tqReaderOpen(SVnode *pVnode);
-void tqCloseReader(STqReader *);
+void tqReaderClose(STqReader *);
void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
-int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id);
-bool tqNextBlockInWal(STqReader* pReader, const char* idstr);
-bool tqNextBlockImpl(STqReader *pReader, const char* idstr);
+bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid);
+bool tqCurrentBlockConsumed(const STqReader* pReader);
-int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id);
+int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id);
+bool tqNextBlockInWal(STqReader *pReader, const char *idstr);
+bool tqNextBlockImpl(STqReader *pReader, const char *idstr);
+SWalReader* tqGetWalReader(STqReader* pReader);
+SSDataBlock* tqGetResultBlock (STqReader* pReader);
+
+int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id);
int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
-int32_t tqRetrieveDataBlock(STqReader *pReader, const char* idstr);
+int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock** pRes, const char* idstr);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg);
@@ -288,10 +280,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot);
int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
-int32_t buildSnapContext(SMeta *pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta,
+int32_t buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta,
SSnapContext **ctxRet);
-int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
-SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx);
+int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
+SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx);
int32_t setForSnapShot(SSnapContext *ctx, int64_t uid);
int32_t destroySnapContext(SSnapContext *ctx);
@@ -350,67 +342,62 @@ struct SVnodeCfg {
int32_t tsdbPageSize;
};
-typedef struct {
- uint64_t uid;
- uint64_t groupId;
-} STableKeyInfo;
-
#define TABLE_ROLLUP_ON ((int8_t)0x1)
#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0)
#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON)
-struct SMetaEntry {
- int64_t version;
- int8_t type;
- int8_t flags; // TODO: need refactor?
- tb_uid_t uid;
- char *name;
- union {
- struct {
- SSchemaWrapper schemaRow;
- SSchemaWrapper schemaTag;
- SRSmaParam rsmaParam;
- } stbEntry;
- struct {
- int64_t ctime;
- int32_t ttlDays;
- int32_t commentLen;
- char *comment;
- tb_uid_t suid;
- uint8_t *pTags;
- } ctbEntry;
- struct {
- int64_t ctime;
- int32_t ttlDays;
- int32_t commentLen;
- char *comment;
- int32_t ncid; // next column id
- SSchemaWrapper schemaRow;
- } ntbEntry;
- struct {
- STSma *tsma;
- } smaEntry;
- };
+//struct SMetaEntry {
+// int64_t version;
+// int8_t type;
+// int8_t flags; // TODO: need refactor?
+// tb_uid_t uid;
+// char *name;
+// union {
+// struct {
+// SSchemaWrapper schemaRow;
+// SSchemaWrapper schemaTag;
+// SRSmaParam rsmaParam;
+// } stbEntry;
+// struct {
+// int64_t ctime;
+// int32_t ttlDays;
+// int32_t commentLen;
+// char *comment;
+// tb_uid_t suid;
+// uint8_t *pTags;
+// } ctbEntry;
+// struct {
+// int64_t ctime;
+// int32_t ttlDays;
+// int32_t commentLen;
+// char *comment;
+// int32_t ncid; // next column id
+// SSchemaWrapper schemaRow;
+// } ntbEntry;
+// struct {
+// STSma *tsma;
+// } smaEntry;
+// };
+//
+// uint8_t *pBuf;
+//};
- uint8_t *pBuf;
-};
+//struct SMetaReader {
+// int32_t flags;
+// SMeta *pMeta;
+// SDecoder coder;
+// SMetaEntry me;
+// void *pBuf;
+// int32_t szBuf;
+//};
-struct SMetaReader {
- int32_t flags;
- SMeta *pMeta;
- SDecoder coder;
- SMetaEntry me;
- void *pBuf;
- int32_t szBuf;
-};
-
-struct SMTbCursor {
- TBC *pDbc;
- void *pKey;
- void *pVal;
- int32_t kLen;
- int32_t vLen;
- SMetaReader mr;
-};
+//struct SMTbCursor {
+// TBC *pDbc;
+// void *pKey;
+// void *pVal;
+// int32_t kLen;
+// int32_t vLen;
+// SMetaReader mr;
+//};
#ifdef __cplusplus
}
diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h
index 3999aa0b7f..b39008147b 100644
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@@ -162,6 +162,12 @@ typedef struct {
int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid,
STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey);
+// TODO, refactor later
+int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *param, SArray *results);
+int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *parm, SArray *pUids);
+int32_t metaFilterTableName(void *pVnode, SMetaFltParam *param, SArray *pUids);
+int32_t metaFilterTtl(void *pVnode, SMetaFltParam *param, SArray *pUids);
+
#ifndef META_REFACT
// SMetaDB
int metaOpenDB(SMeta* pMeta);
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 8fddb97909..faf550ab75 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -847,9 +847,6 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
-void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
-size_t tsdbCacheGetCapacity(SVnode *pVnode);
-
// int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
// ========== inline functions ==========
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index f056311388..3f79fe5fd1 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -161,6 +161,8 @@ void* metaGetIdx(SMeta* pMeta);
void* metaGetIvtIdx(SMeta* pMeta);
int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList);
+void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+
int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
@@ -473,6 +475,8 @@ struct SCompactInfo {
STimeWindow tw;
};
+void initStorageAPI(SStorageAPI* pAPI);
+
#ifdef __cplusplus
}
#endif
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index d464f64de3..26182baf7b 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -17,10 +17,16 @@
#include "osMemory.h"
#include "tencode.h"
+void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI) {
+ SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
+ metaReaderInit(pReader, pMeta, flags);
+ pReader->pAPI = pAPI;
+}
+
void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) {
memset(pReader, 0, sizeof(*pReader));
- pReader->flags = flags;
pReader->pMeta = pMeta;
+ pReader->flags = flags;
if (pReader->pMeta && !(flags & META_READER_NOLOCK)) {
metaRLock(pMeta);
}
@@ -64,96 +70,20 @@ _err:
return -1;
}
-// int metaGetTableEntryByUidTest(void* meta, SArray *uidList) {
-//
-// SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader));
-// SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey));
-// SMeta *pMeta = meta;
-// int64_t version;
-// SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
-//
-// int64_t stt1 = taosGetTimestampUs();
-// for(int i = 0; i < taosArrayGetSize(uidList); i++) {
-// void* ppVal = NULL;
-// int vlen = 0;
-// uint64_t * uid = taosArrayGet(uidList, i);
-// // query uid.idx
-// if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) {
-// continue;
-// }
-// version = *(int64_t *)ppVal;
-//
-// STbDbKey tbDbKey = {.version = version, .uid = *uid};
-// taosArrayPush(uidVersion, &tbDbKey);
-// taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t));
-// }
-// int64_t stt2 = taosGetTimestampUs();
-// qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1);
-//
-// TBC *pCur = NULL;
-// tdbTbcOpen(pMeta->pTbDb, &pCur, NULL);
-// tdbTbcMoveToFirst(pCur);
-// void *pKey = NULL;
-// int kLen = 0;
-//
-// while(1){
-// SMetaReader pReader = {0};
-// int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf);
-// if (ret < 0) break;
-// STbDbKey *tmp = (STbDbKey*)pKey;
-// int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t));
-// if(ver == NULL || *ver != tmp->version) continue;
-// taosArrayPush(readerList, &pReader);
-// }
-// tdbTbcClose(pCur);
-//
-// taosArrayClear(readerList);
-// int64_t stt3 = taosGetTimestampUs();
-// qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2);
-// for(int i = 0; i < taosArrayGetSize(uidVersion); i++) {
-// SMetaReader pReader = {0};
-//
-// STbDbKey *tbDbKey = taosArrayGet(uidVersion, i);
-// // query table.db
-// if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) {
-// continue;
-// }
-// taosArrayPush(readerList, &pReader);
-// }
-// int64_t stt4 = taosGetTimestampUs();
-// qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3);
-//
-// for(int i = 0; i < taosArrayGetSize(readerList); i++){
-// SMetaReader* pReader = taosArrayGet(readerList, i);
-// metaReaderInit(pReader, meta, 0);
-// // decode the entry
-// tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf);
-//
-// if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) {
-// }
-// metaReaderClear(pReader);
-// }
-// int64_t stt5 = taosGetTimestampUs();
-// qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4);
-// return 0;
-// }
-
-bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) {
- // query uid.idx
- metaRLock(pMeta);
-
- if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) {
- metaULock(pMeta);
+bool metaIsTableExist(void *pVnode, tb_uid_t uid) {
+ SVnode* pVnodeObj = pVnode;
+ metaRLock(pVnodeObj->pMeta); // query uid.idx
+ if (tdbTbGet(pVnodeObj->pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) {
+ metaULock(pVnodeObj->pMeta);
return false;
}
- metaULock(pMeta);
-
+ metaULock(pVnodeObj->pMeta);
return true;
}
-int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
+int metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
SMeta *pMeta = pReader->pMeta;
int64_t version1;
@@ -167,7 +97,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
return metaGetTableEntryByVersion(pReader, version1, uid);
}
-int metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) {
+int metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid) {
SMeta *pMeta = pReader->pMeta;
SMetaInfo info;
@@ -190,7 +120,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
}
uid = *(tb_uid_t *)pReader->pBuf;
- return metaGetTableEntryByUid(pReader, uid);
+ return metaReaderGetTableEntryByUid(pReader, uid);
}
tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
@@ -210,11 +140,11 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
return uid;
}
-int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) {
+int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) {
int code = 0;
SMetaReader mr = {0};
- metaReaderInit(&mr, (SMeta *)meta, 0);
- code = metaGetTableEntryByUid(&mr, uid);
+ metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0);
+ code = metaReaderGetTableEntryByUid(&mr, uid);
if (code < 0) {
metaReaderClear(&mr);
return -1;
@@ -230,7 +160,7 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) {
int code = 0;
SMetaReader mr = {0};
metaReaderInit(&mr, (SMeta *)meta, 0);
- code = metaGetTableEntryByUid(&mr, uid);
+ code = metaReaderGetTableEntryByUid(&mr, uid);
if (code < 0) {
metaReaderClear(&mr);
return -1;
@@ -241,15 +171,15 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) {
return 0;
}
-int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) {
+int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) {
int code = 0;
SMetaReader mr = {0};
- metaReaderInit(&mr, (SMeta *)meta, 0);
+ metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0);
SMetaReader *pReader = &mr;
// query name.idx
- if (tdbTbGet(pReader->pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
+ if (tdbTbGet(((SMeta*)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
metaReaderClear(&mr);
return -1;
@@ -262,10 +192,10 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) {
return 0;
}
-int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType) {
+int metaGetTableTypeByName(void *pVnode, char *tbName, ETableType *tbType) {
int code = 0;
SMetaReader mr = {0};
- metaReaderInit(&mr, (SMeta *)meta, 0);
+ metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0);
code = metaGetTableEntryByName(&mr, tbName);
if (code == 0) *tbType = mr.me.type;
@@ -283,7 +213,7 @@ int metaReadNext(SMetaReader *pReader) {
}
#if 1 // ===================================================
-SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
+SMTbCursor *metaOpenTbCursor(void *pVnode) {
SMTbCursor *pTbCur = NULL;
pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur));
@@ -291,12 +221,12 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
return NULL;
}
- metaReaderInit(&pTbCur->mr, pMeta, 0);
+ SVnode* pVnodeObj = pVnode;
+ metaReaderInit(&pTbCur->mr, pVnodeObj->pMeta, 0);
- tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL);
-
- tdbTbcMoveToFirst(pTbCur->pDbc);
+ tdbTbcOpen(pVnodeObj->pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL);
+ tdbTbcMoveToFirst((TBC *)pTbCur->pDbc);
return pTbCur;
}
@@ -306,7 +236,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) {
tdbFree(pTbCur->pVal);
metaReaderClear(&pTbCur->mr);
if (pTbCur->pDbc) {
- tdbTbcClose(pTbCur->pDbc);
+ tdbTbcClose((TBC *)pTbCur->pDbc);
}
taosMemoryFree(pTbCur);
}
@@ -318,7 +248,7 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) {
STbCfg tbCfg;
for (;;) {
- ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
+ ret = tdbTbcNext((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
if (ret < 0) {
return -1;
}
@@ -342,7 +272,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) {
STbCfg tbCfg;
for (;;) {
- ret = tdbTbcPrev(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
+ ret = tdbTbcPrev((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
if (ret < 0) {
return -1;
}
@@ -876,7 +806,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
STSma *pTSma = NULL;
for (int i = 0; i < pSW->number; ++i) {
smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i);
- if (metaGetTableEntryByUid(&mr, smaId) < 0) {
+ if (metaReaderGetTableEntryByUid(&mr, smaId) < 0) {
tDecoderClear(&mr.coder);
metaWarn("vgId:%d, no entry for tbId:%" PRIi64 ", smaId:%" PRIi64, TD_VID(pMeta->pVnode), uid, smaId);
continue;
@@ -926,7 +856,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) {
STSma *pTSma = NULL;
SMetaReader mr = {0};
metaReaderInit(&mr, pMeta, 0);
- if (metaGetTableEntryByUid(&mr, indexUid) < 0) {
+ if (metaReaderGetTableEntryByUid(&mr, indexUid) < 0) {
metaWarn("vgId:%d, failed to get table entry for smaId:%" PRIi64, TD_VID(pMeta->pVnode), indexUid);
metaReaderClear(&mr);
return NULL;
@@ -1027,7 +957,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) {
#endif
-const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) {
+const void *metaGetTableTagVal(const void *pTag, int16_t type, STagVal *val) {
STag *tag = (STag *)pTag;
if (type == TSDB_DATA_TYPE_JSON) {
return tag;
@@ -1083,8 +1013,10 @@ typedef struct {
int32_t vLen;
} SIdxCursor;
-int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
- int32_t ret = 0;
+int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
+ SMeta *pMeta = ((SVnode*)pVnode)->pMeta;
+ SMetaFltParam *param = arg;
+ int32_t ret = 0;
SIdxCursor *pCursor = NULL;
pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor));
@@ -1141,9 +1073,11 @@ END:
return ret;
}
-int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
- int32_t ret = 0;
- char *buf = NULL;
+int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
+ SMeta *pMeta = ((SVnode*)pVnode)->pMeta;
+ SMetaFltParam *param = arg;
+ int32_t ret = 0;
+ char *buf = NULL;
STagIdxKey *pKey = NULL;
int32_t nKey = 0;
@@ -1206,9 +1140,11 @@ END:
return ret;
}
-int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
- int32_t ret = 0;
- char *buf = NULL;
+int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
+ SMeta *pMeta = ((SVnode*)pVnode)->pMeta;
+ SMetaFltParam *param = arg;
+ int32_t ret = 0;
+ char *buf = NULL;
STtlIdxKey *pKey = NULL;
int32_t nKey = 0;
@@ -1235,7 +1171,10 @@ END:
// impl later
return 0;
}
-int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) {
+int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
+ SMeta *pMeta = ((SVnode*)pVnode)->pMeta;
+ SMetaFltParam *param = arg;
+
SMetaEntry oStbEntry = {0};
int32_t ret = -1;
char *buf = NULL;
@@ -1418,7 +1357,8 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi
return ret;
}
-int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) {
+int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) {
+ SMeta* pMeta = ((SVnode*) pVnode)->pMeta;
const int32_t LIMIT = 128;
int32_t isLock = false;
@@ -1450,8 +1390,8 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) {
return 0;
}
-int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *pUidTagInfo) {
- SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid, 1);
+int32_t metaGetTableTags(void* pVnode, uint64_t suid, SArray *pUidTagInfo) {
+ SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode*)pVnode)->pMeta, suid, 1);
// If len > 0 means there already have uids, and we only want the
// tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept
@@ -1556,30 +1496,35 @@ _exit:
return code;
}
-int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo) {
+int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t* numOfTables) {
int32_t code = 0;
+ *numOfTables = 0;
- metaRLock(pMeta);
+ SVnode* pVnodeObj = pVnode;
+ metaRLock(pVnodeObj->pMeta);
// fast path: search cache
- if (metaStatsCacheGet(pMeta, uid, pInfo) == TSDB_CODE_SUCCESS) {
- metaULock(pMeta);
+ SMetaStbStats state = {0};
+ if (metaStatsCacheGet(pVnodeObj->pMeta, uid, &state) == TSDB_CODE_SUCCESS) {
+ metaULock(pVnodeObj->pMeta);
+ *numOfTables = state.ctbNum;
goto _exit;
}
// slow path: search TDB
int64_t ctbNum = 0;
- vnodeGetCtbNum(pMeta->pVnode, uid, &ctbNum);
+ vnodeGetCtbNum(pVnode, uid, &ctbNum);
- metaULock(pMeta);
+ metaULock(pVnodeObj->pMeta);
+ *numOfTables = ctbNum;
- pInfo->uid = uid;
- pInfo->ctbNum = ctbNum;
+ state.uid = uid;
+ state.ctbNum = ctbNum;
// upsert the cache
- metaWLock(pMeta);
- metaStatsCacheUpsert(pMeta, pInfo);
- metaULock(pMeta);
+ metaWLock(pVnodeObj->pMeta);
+ metaStatsCacheUpsert(pVnodeObj->pMeta, &state);
+ metaULock(pVnodeObj->pMeta);
_exit:
return code;
diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c
index 8d5821e28b..3fb491848a 100644
--- a/source/dnode/vnode/src/meta/metaSma.c
+++ b/source/dnode/vnode/src/meta/metaSma.c
@@ -36,7 +36,7 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
// validate req
// save smaIndex
metaReaderInit(&mr, pMeta, 0);
- if (metaGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) {
+ if (metaReaderGetTableEntryByUidCache(&mr, pCfg->indexUid) == 0) {
#if 1
terrno = TSDB_CODE_TSMA_ALREADY_EXIST;
metaReaderClear(&mr);
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 707dd66e30..f4e930e509 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -222,29 +222,29 @@ static void destroySTableInfoForChildTable(void* data) {
}
static void MoveToSnapShotVersion(SSnapContext* ctx) {
- tdbTbcClose(ctx->pCur);
- tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL);
+ tdbTbcClose((TBC*)ctx->pCur);
+ tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL);
STbDbKey key = {.version = ctx->snapVersion, .uid = INT64_MAX};
int c = 0;
- tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c);
+ tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c);
if (c < 0) {
- tdbTbcMoveToPrev(ctx->pCur);
+ tdbTbcMoveToPrev((TBC*)ctx->pCur);
}
}
static int32_t MoveToPosition(SSnapContext* ctx, int64_t ver, int64_t uid) {
- tdbTbcClose(ctx->pCur);
- tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL);
+ tdbTbcClose((TBC*)ctx->pCur);
+ tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL);
STbDbKey key = {.version = ver, .uid = uid};
int c = 0;
- tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c);
+ tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c);
return c;
}
static void MoveToFirst(SSnapContext* ctx) {
- tdbTbcClose(ctx->pCur);
- tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL);
- tdbTbcMoveToFirst(ctx->pCur);
+ tdbTbcClose((TBC*)ctx->pCur);
+ tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL);
+ tdbTbcMoveToFirst((TBC*)ctx->pCur);
}
static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) {
@@ -260,12 +260,12 @@ static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo)
taosHashPut(suidInfo, &me->uid, sizeof(tb_uid_t), &dataTmp, sizeof(STableInfoForChildTable));
}
-int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta,
+int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta,
SSnapContext** ctxRet) {
SSnapContext* ctx = taosMemoryCalloc(1, sizeof(SSnapContext));
if (ctx == NULL) return -1;
*ctxRet = ctx;
- ctx->pMeta = pMeta;
+ ctx->pMeta = pVnode->pMeta;
ctx->snapVersion = snapVersion;
ctx->suid = suid;
ctx->subType = subType;
@@ -291,7 +291,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
metaDebug("tmqsnap init snapVersion:%" PRIi64, ctx->snapVersion);
MoveToFirst(ctx);
while (1) {
- int32_t ret = tdbTbcNext(ctx->pCur, &pKey, &kLen, &pVal, &vLen);
+ int32_t ret = tdbTbcNext((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break;
STbDbKey* tmp = (STbDbKey*)pKey;
if (tmp->version > ctx->snapVersion) break;
@@ -301,7 +301,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
continue;
}
- if (tdbTbGet(pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) <
+ if (tdbTbGet(ctx->pMeta->pUidIdx, &tmp->uid, sizeof(tb_uid_t), NULL, NULL) <
0) { // check if table exist for now, need optimize later
continue;
}
@@ -329,7 +329,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
MoveToSnapShotVersion(ctx);
while (1) {
- int32_t ret = tdbTbcPrev(ctx->pCur, &pKey, &kLen, &pVal, &vLen);
+ int32_t ret = tdbTbcPrev((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break;
STbDbKey* tmp = (STbDbKey*)pKey;
@@ -378,7 +378,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
}
int32_t destroySnapContext(SSnapContext* ctx) {
- tdbTbcClose(ctx->pCur);
+ tdbTbcClose((TBC*)ctx->pCur);
taosArrayDestroy(ctx->idList);
taosHashCleanup(ctx->idVersion);
taosHashCleanup(ctx->suidInfo);
@@ -466,7 +466,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
return c;
}
-int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) {
+int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) {
int32_t ret = 0;
void* pKey = NULL;
void* pVal = NULL;
@@ -496,7 +496,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in
metaDebug("tmqsnap get meta not exist uid:%" PRIi64 " version:%" PRIi64, *uid, idInfo->version);
}
- tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
+ tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
SDecoder dc = {0};
SMetaEntry me = {0};
tDecoderInit(&dc, pVal, vLen);
@@ -598,7 +598,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in
return ret;
}
-SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) {
+SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext* ctx) {
SMetaTableInfo result = {0};
void* pKey = NULL;
void* pVal = NULL;
@@ -619,10 +619,10 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) {
int32_t ret = MoveToPosition(ctx, idInfo->version, *uidTmp);
if (ret != 0) {
- metaDebug("tmqsnap getUidfromSnapShot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version);
+ metaDebug("tmqsnap getMetaTableInfoFromSnapshot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version);
continue;
}
- tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
+ tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
SDecoder dc = {0};
SMetaEntry me = {0};
tDecoderInit(&dc, pVal, vLen);
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 0164a82c69..df16304e28 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -690,7 +690,7 @@ _err:
return -1;
}
-int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) {
+int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) {
SMetaEntry me = {0};
SMetaReader mr = {0};
@@ -729,7 +729,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe
metaReaderClear(&mr);
// build SMetaEntry
- me.version = version;
+ me.version = ver;
me.type = pReq->type;
me.uid = pReq->uid;
me.name = pReq->name;
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index ccc00ce25e..39aa5c3043 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -276,12 +276,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
return TSDB_CODE_FAILED;
}
- SReadHandle handle = {
- .meta = pVnode->pMeta,
- .vnode = pVnode,
- .initTqReader = 1,
- .pStateBackend = pStreamState,
- };
+ SReadHandle handle = { .vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState };
+ initStorageAPI(&handle.api);
+
pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode));
if (!pRSmaInfo->taskInfo[idx]) {
terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
@@ -853,11 +850,8 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t
code = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len);
TSDB_CHECK_CODE(code, lino, _exit);
- SReadHandle handle = {
- .meta = pVnode->pMeta,
- .vnode = pVnode,
- .initTqReader = 1,
- };
+ SReadHandle handle = { .vnode = pVnode, .initTqReader = 1 };
+ initStorageAPI(&handle.api);
if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) {
code = TSDB_CODE_APP_ERROR;
@@ -904,7 +898,7 @@ static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) {
metaReaderInit(&mr, SMA_META(pSma), 0);
smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid);
- if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) {
+ if (metaReaderGetTableEntryByUidCache(&mr, pInfo->suid) < 0) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
}
@@ -1131,7 +1125,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
for (int64_t i = 0; i < arrSize; ++i) {
suid = *(tb_uid_t *)taosArrayGet(suidList, i);
smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid);
- if (metaGetTableEntryByUidCache(&mr, suid) < 0) {
+ if (metaReaderGetTableEntryByUidCache(&mr, suid) < 0) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
}
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 0a3173b3cb..bde53dca2d 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -69,12 +69,12 @@ static void destroyTqHandle(void* data) {
if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
taosMemoryFreeClear(pData->execHandle.execCol.qmsg);
} else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) {
- tqCloseReader(pData->execHandle.pTqReader);
+ tqReaderClose(pData->execHandle.pTqReader);
walCloseReader(pData->pWalReader);
taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid);
} else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
walCloseReader(pData->pWalReader);
- tqCloseReader(pData->execHandle.pTqReader);
+ tqReaderClose(pData->execHandle.pTqReader);
}
if(pData->msg != NULL) {
rpcFreeCont(pData->msg->pCont);
@@ -510,8 +510,6 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
int64_t sver = 0, ever = 0;
walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever);
- int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader);
-
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, &req);
@@ -537,7 +535,12 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
dataRsp.rspOffset.type = TMQ_OFFSET__LOG;
if (reqOffset.type == TMQ_OFFSET__LOG) {
- dataRsp.rspOffset.version = currentVer; // return current consume offset value
+ int64_t currentVer = walReaderGetCurrentVer(pHandle->execHandle.pTqReader->pWalReader);
+ if (currentVer == -1) { // not start to read data from wal yet, return req offset directly
+ dataRsp.rspOffset.version = reqOffset.version;
+ } else {
+ dataRsp.rspOffset.version = currentVer; // return current consume offset value
+ }
} else if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
@@ -670,8 +673,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
int64_t ver = pRef->refVer;
pHandle->pRef = pRef;
- SReadHandle handle = {
- .meta = pVnode->pMeta, .vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver};
+ SReadHandle handle = {.vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver};
+ initStorageAPI(&handle.api);
+
pHandle->snapshotVer = ver;
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
@@ -689,7 +693,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
pHandle->execHandle.execDb.pFilterOutTbUid =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
- buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
+ buildSnapContext(handle.vnode, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta,
(SSnapContext**)(&handle.sContext));
pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId);
@@ -708,7 +712,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList);
taosArrayDestroy(tbUidList);
- buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta,
+ buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta,
(SSnapContext**)(&handle.sContext));
pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId);
}
@@ -787,8 +791,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
return -1;
}
- SReadHandle handle = {
- .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState};
+ SReadHandle handle = {.vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState};
+ initStorageAPI(&handle.api);
pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId);
if (pTask->exec.pExecutor == NULL) {
@@ -803,9 +807,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
}
int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo);
- SReadHandle mgHandle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState};
+ SReadHandle handle = {.vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState};
+ initStorageAPI(&handle.api);
- pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId);
+ pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId);
if (pTask->exec.pExecutor == NULL) {
return -1;
}
@@ -1083,6 +1088,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t
qDebug("s-task:%s set the start wal offset to be:%"PRId64, pTask->id.idStr, sversion);
walReaderSeekVer(pTask->exec.pWalReader, sversion);
+ pTask->chkInfo.currentVer = sversion;
if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@@ -1278,6 +1284,8 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
int32_t tqProcessTaskDropReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
+ tqDebug("vgId:%d receive msg to drop stream task:0x%x", TD_VID(pTq->pVnode), pReq->taskId);
+
streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId);
return 0;
}
diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c
index 5654147b6d..be5b5706e2 100644
--- a/source/dnode/vnode/src/tq/tqMeta.c
+++ b/source/dnode/vnode/src/tq/tqMeta.c
@@ -298,13 +298,14 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
walSetRefVer(handle.pRef, handle.snapshotVer);
SReadHandle reader = {
- .meta = pTq->pVnode->pMeta,
.vnode = pTq->pVnode,
.initTableReader = true,
.initTqReader = true,
- .version = handle.snapshotVer,
+ .version = handle.snapshotVer
};
+ initStorageAPI(&reader.api);
+
if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
handle.execHandle.task =
qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0);
@@ -330,7 +331,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode);
- buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta,
+ buildSnapContext(reader.vnode, reader.version, 0, handle.execHandle.subType, handle.fetchMeta,
(SSnapContext**)(&reader.sContext));
handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0);
} else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
@@ -347,7 +348,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList);
taosArrayDestroy(tbUidList);
- buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType,
+ buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType,
handle.fetchMeta, (SSnapContext**)(&reader.sContext));
handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0);
}
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index b756f99f32..dbb7d564f8 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -273,7 +273,7 @@ STqReader* tqReaderOpen(SVnode* pVnode) {
return pReader;
}
-void tqCloseReader(STqReader* pReader) {
+void tqReaderClose(STqReader* pReader) {
// close wal reader
if (pReader->pWalReader) {
walCloseReader(pReader->pWalReader);
@@ -294,7 +294,7 @@ void tqCloseReader(STqReader* pReader) {
taosMemoryFree(pReader);
}
-int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) {
+int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) {
if (walReaderSeekVer(pReader->pWalReader, ver) < 0) {
return -1;
}
@@ -394,8 +394,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) {
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if (pReader->tbIdHash == NULL) {
- int32_t code = tqRetrieveDataBlock(pReader, NULL);
- if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) {
+ SSDataBlock* pRes = NULL;
+ int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL);
+ if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
return true;
}
}
@@ -404,8 +405,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) {
if (ret != NULL) {
tqDebug("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver);
- int32_t code = tqRetrieveDataBlock(pReader, NULL);
- if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) {
+ SSDataBlock* pRes = NULL;
+ int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL);
+ if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
return true;
}
} else {
@@ -440,6 +442,14 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i
return 0;
}
+SWalReader* tqGetWalReader(STqReader* pReader) {
+ return pReader->pWalReader;
+}
+
+SSDataBlock* tqGetResultBlock (STqReader* pReader) {
+ return pReader->pResBlock;
+}
+
bool tqNextBlockImpl(STqReader* pReader, const char* idstr) {
if (pReader->msg.msgStr == NULL) {
return false;
@@ -592,12 +602,14 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol
return code;
}
-int32_t tqRetrieveDataBlock(STqReader* pReader, const char* id) {
+int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) {
tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++);
SSDataBlock* pBlock = pReader->pResBlock;
+ *pRes = pBlock;
+
blockDataCleanup(pBlock);
int32_t sversion = pSubmitTbData->sver;
@@ -1024,6 +1036,14 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) {
return 0;
}
+bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid) {
+ return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t));
+}
+
+bool tqCurrentBlockConsumed(const STqReader* pReader) {
+ return pReader->msg.msgStr == NULL;
+}
+
int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
@@ -1069,7 +1089,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) {
uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i);
- int32_t code = metaGetTableEntryByUidCache(&mr, *id);
+ int32_t code = metaReaderGetTableEntryByUidCache(&mr, *id);
if (code != TSDB_CODE_SUCCESS) {
tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
continue;
diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c
index ff9f95d5fa..6e3036a66b 100644
--- a/source/dnode/vnode/src/tq/tqRestore.c
+++ b/source/dnode/vnode/src/tq/tqRestore.c
@@ -61,9 +61,10 @@ static int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) {
// seek the stored version and extract data from WAL
int64_t firstVer = walReaderGetValidFirstVer(pTask->exec.pWalReader);
if (pTask->chkInfo.currentVer < firstVer) {
+ tqWarn("vgId:%d s-task:%s ver:%"PRId64" earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId,
+ pTask->id.idStr, pTask->chkInfo.currentVer, firstVer, firstVer);
+
pTask->chkInfo.currentVer = firstVer;
- tqWarn("vgId:%d s-task:%s ver earlier than the first ver of wal range %" PRId64 ", forward to %" PRId64, vgId,
- pTask->id.idStr, firstVer, pTask->chkInfo.currentVer);
// todo need retry if failed
int32_t code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer);
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index e268199e16..98304f9c18 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -51,7 +51,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in
metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
// TODO add reference to gurantee success
- if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
+ if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) {
metaReaderClear(&mr);
return -1;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index d33f04c145..ef77c9a7c4 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -703,6 +703,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
*pTmpLastCol = *pLastCol;
pLastCol = pTmpLastCol;
+ reallocVarData(&pLastCol->colVal);
size_t charge = sizeof(*pLastCol);
if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) {
charge += pLastCol->colVal.value.nData;
@@ -789,7 +790,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA
code = -1;
}
- taosArraySet(pLastArray, idxKey->idx, pLastCol);
+ SLastCol lastCol = *pLastCol;
+ reallocVarData(&lastCol.colVal);
+ taosArraySet(pLastArray, idxKey->idx, &lastCol);
taosArrayRemove(remainCols, j);
taosMemoryFree(values_list[i]);
@@ -825,7 +828,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h);
SLastCol lastCol = *pLastCol;
- // reallocVarData(&lastCol.colVal);
+ reallocVarData(&lastCol.colVal);
taosArrayPush(pLastArray, &lastCol);
if (h) {
@@ -853,8 +856,8 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
SLastCol lastCol = *pLastCol;
reallocVarData(&lastCol.colVal);
-
taosArraySet(pLastArray, idxKey->idx, &lastCol);
+
if (h) {
taosLRUCacheRelease(pCache, h, false);
}
@@ -937,14 +940,14 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
}
// build keys & multi get from rocks
- int num_keys = pTSchema->numOfCols;
- char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *));
- size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t));
+ int num_keys = pTSchema->numOfCols;
+ char **keys_list = taosMemoryCalloc(num_keys * 2, sizeof(char *));
+ size_t *keys_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t));
+ const size_t klen = ROCKS_KEY_LEN;
for (int i = 0; i < num_keys; ++i) {
int16_t cid = pTSchema->columns[i].colId;
- size_t klen = ROCKS_KEY_LEN;
- char *keys = taosMemoryCalloc(2, sizeof(SLastKey));
+ char *keys = taosMemoryCalloc(2, sizeof(SLastKey));
((SLastKey *)keys)[0] = (SLastKey){.ltype = 1, .uid = uid, .cid = cid};
((SLastKey *)keys)[1] = (SLastKey){.ltype = 0, .uid = uid, .cid = cid};
@@ -960,39 +963,35 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
rocksMayWrite(pTsdb, true, false, false);
rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list,
keys_list_sizes, values_list, values_list_sizes, errs);
- for (int i = 0; i < num_keys; ++i) {
- taosMemoryFree(keys_list[i]);
- }
for (int i = 0; i < num_keys * 2; ++i) {
- rocksdb_free(errs[i]);
+ if (errs[i]) {
+ rocksdb_free(errs[i]);
+ }
}
- taosMemoryFree(keys_list);
- taosMemoryFree(keys_list_sizes);
taosMemoryFree(errs);
rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
for (int i = 0; i < num_keys; ++i) {
SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]);
if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) {
- SLastKey *key = &(SLastKey){.ltype = 1, .uid = uid, .cid = pLastCol->colVal.cid};
- size_t klen = ROCKS_KEY_LEN;
-
- rocksdb_writebatch_delete(wb, (char *)key, klen);
- taosLRUCacheErase(pTsdb->lruCache, key, klen);
+ rocksdb_writebatch_delete(wb, keys_list[i], klen);
}
+ taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen);
pLastCol = tsdbCacheDeserialize(values_list[i + num_keys]);
if (NULL != pLastCol && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) {
- SLastKey *key = &(SLastKey){.ltype = 0, .uid = uid, .cid = pLastCol->colVal.cid};
- size_t klen = ROCKS_KEY_LEN;
-
- rocksdb_writebatch_delete(wb, (char *)key, klen);
- taosLRUCacheErase(pTsdb->lruCache, key, klen);
+ rocksdb_writebatch_delete(wb, keys_list[num_keys + i], klen);
}
+ taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen);
rocksdb_free(values_list[i]);
rocksdb_free(values_list[i + num_keys]);
}
+ for (int i = 0; i < num_keys; ++i) {
+ taosMemoryFree(keys_list[i]);
+ }
+ taosMemoryFree(keys_list);
+ taosMemoryFree(keys_list_sizes);
taosMemoryFree(values_list);
taosMemoryFree(values_list_sizes);
@@ -1432,7 +1431,7 @@ static tb_uid_t getTableSuidByUid(tb_uid_t uid, STsdb *pTsdb) {
SMetaReader mr = {0};
metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
- if (metaGetTableEntryByUidCache(&mr, uid) < 0) {
+ if (metaReaderGetTableEntryByUidCache(&mr, uid) < 0) {
metaReaderClear(&mr); // table not esist
return 0;
}
@@ -1871,10 +1870,14 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
- } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+ } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
- }
+ }*/
+ }
+
+ if (!isLast) {
+ skipBlock = false;
}
if (skipBlock) {
@@ -1908,6 +1911,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
if (checkRemainingRow) {
bool skipBlock = true;
int inputColIndex = 0;
+ if (aCols[0] == PRIMARYKEY_TIMESTAMP_COL_ID) {
+ ++inputColIndex;
+ }
for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) {
SColData *pColData = &state->pBlockData->aColData[colIndex];
int16_t cid = pColData->cid;
@@ -1916,15 +1922,19 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
if (isLast && (pColData->flag & HAS_VALUE)) {
skipBlock = false;
break;
- } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+ } /*else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
skipBlock = false;
break;
- }
+ }*/
++inputColIndex;
}
}
+ if (!isLast) {
+ skipBlock = false;
+ }
+
if (skipBlock) {
if (--state->iBlock < 0) {
tsdbDataFReaderClose(state->pDataFReader);
@@ -2145,9 +2155,14 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) {
return false;
} else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) {
if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) {
+ // if (key->version <= pItemFront->version || key->version <= pItemBack->version) {
return true;
} else {
- return false;
+ if (*iSkyline > 1) {
+ --*iSkyline;
+ } else {
+ return false;
+ }
}
} else {
if (*iSkyline > 1) {
@@ -2959,7 +2974,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray,
do {
TSDBROW *pRow = NULL;
- nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray));
+ nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray));
if (!pRow) {
break;
diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
index 0e0b7a2ffa..5df1ea0672 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
@@ -315,14 +315,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
tsdbCacheGetBatch(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype);
// tsdbCacheGet(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype);
if (TARRAY_SIZE(pRow) <= 0) {
- // taosArrayClearEx(pRow, freeItem);
- taosArrayClear(pRow);
+ taosArrayClearEx(pRow, freeItem);
+ // taosArrayClear(pRow);
continue;
}
SLastCol* pColVal = taosArrayGet(pRow, 0);
if (COL_VAL_IS_NONE(&pColVal->colVal)) {
- // taosArrayClearEx(pRow, freeItem);
- taosArrayClear(pRow);
+ taosArrayClearEx(pRow, freeItem);
+ // taosArrayClear(pRow);
continue;
}
@@ -381,8 +381,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
}
}
- // taosArrayClearEx(pRow, freeItem);
- taosArrayClear(pRow);
+ taosArrayClearEx(pRow, freeItem);
+ // taosArrayClear(pRow);
}
if (hasRes) {
@@ -394,20 +394,20 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
if (TARRAY_SIZE(pRow) <= 0) {
- // taosArrayClearEx(pRow, freeItem);
- taosArrayClear(pRow);
+ taosArrayClearEx(pRow, freeItem);
+ // taosArrayClear(pRow);
continue;
}
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0);
if (COL_VAL_IS_NONE(&pColVal->colVal)) {
- // taosArrayClearEx(pRow, freeItem);
- taosArrayClear(pRow);
+ taosArrayClearEx(pRow, freeItem);
+ // taosArrayClear(pRow);
continue;
}
saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
- // taosArrayClearEx(pRow, freeItem);
- taosArrayClear(pRow);
+ taosArrayClearEx(pRow, freeItem);
+ // taosArrayClear(pRow);
taosArrayPush(pTableUidList, &uid);
diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
index 80967a906f..6d223e00c5 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@@ -190,9 +190,9 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
tsdbCacheDeleteLast(pTsdb->lruCache, pTbData->uid, eKey);
}
*/
- if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) {
- tsdbCacheDel(pTsdb, suid, uid, sKey, eKey);
- }
+ // if (eKey >= pTbData->maxKey && sKey <= pTbData->maxKey) {
+ tsdbCacheDel(pTsdb, suid, uid, sKey, eKey);
+ //}
tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
" at version %" PRId64,
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index b4b090249e..268d17bacb 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -1848,7 +1848,7 @@ static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBloc
SDataBlockToLoadInfo info = {0};
getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader);
bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf ||
- info.overlapWithDelInfo || info.overlapWithLastBlock);
+ info.overlapWithDelInfo || info.overlapWithLastBlock || info.partiallyRequired);
return isCleanFileBlock;
}
@@ -3440,6 +3440,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) {
if (!hasNexTable) {
return TSDB_CODE_SUCCESS;
}
+ pBlockScanInfo = pStatus->pTableIter;
}
initMemDataIterator(*pBlockScanInfo, pReader);
@@ -5380,10 +5381,10 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) {
return rows;
}
-int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) {
+int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) {
SMetaReader mr = {0};
- metaReaderInit(&mr, pVnode->pMeta, 0);
- int32_t code = metaGetTableEntryByUidCache(&mr, uid);
+ metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0);
+ int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
metaReaderClear(&mr);
@@ -5396,7 +5397,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
if (mr.me.type == TSDB_CHILD_TABLE) {
tDecoderClear(&mr.coder);
*suid = mr.me.ctbEntry.suid;
- code = metaGetTableEntryByUidCache(&mr, *suid);
+ code = metaReaderGetTableEntryByUidCache(&mr, *suid);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
metaReaderClear(&mr);
@@ -5412,7 +5413,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
metaReaderClear(&mr);
// get the newest table schema version
- code = metaGetTbTSchemaEx(pVnode->pMeta, *suid, uid, -1, pSchema);
+ code = metaGetTbTSchemaEx(((SVnode*)pVnode)->pMeta, *suid, uid, -1, pSchema);
return code;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c
new file mode 100644
index 0000000000..1cc985f7df
--- /dev/null
+++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+#include "storageapi.h"
+#include "vnodeInt.h"
+#include "tstreamUpdate.h"
+#include "meta.h"
+
+static void initTsdbReaderAPI(TsdReader* pReader);
+static void initMetadataAPI(SStoreMeta* pMeta);
+static void initTqAPI(SStoreTqReader* pTq);
+static void initStateStoreAPI(SStateStore* pStore);
+static void initMetaReaderAPI(SStoreMetaReader* pMetaReader);
+static void initMetaFilterAPI(SMetaDataFilterAPI* pFilter);
+static void initFunctionStateStore(SFunctionStateStore* pStore);
+static void initCacheFn(SStoreCacheReader* pCache);
+static void initSnapshotFn(SStoreSnapshotFn* pSnapshot);
+
+void initStorageAPI(SStorageAPI* pAPI) {
+ initTsdbReaderAPI(&pAPI->tsdReader);
+ initMetadataAPI(&pAPI->metaFn);
+ initTqAPI(&pAPI->tqReaderFn);
+ initStateStoreAPI(&pAPI->stateStore);
+ initMetaReaderAPI(&pAPI->metaReaderFn);
+ initMetaFilterAPI(&pAPI->metaFilter);
+ initFunctionStateStore(&pAPI->functionStore);
+ initCacheFn(&pAPI->cacheFn);
+ initSnapshotFn(&pAPI->snapshotFn);
+}
+
+void initTsdbReaderAPI(TsdReader* pReader) {
+ pReader->tsdReaderOpen = (__store_reader_open_fn_t)tsdbReaderOpen;
+ pReader->tsdReaderClose = tsdbReaderClose;
+
+ pReader->tsdNextDataBlock = tsdbNextDataBlock;
+
+ pReader->tsdReaderRetrieveDataBlock = tsdbRetrieveDataBlock;
+ pReader->tsdReaderReleaseDataBlock = tsdbReleaseDataBlock;
+
+ pReader->tsdReaderRetrieveBlockSMAInfo = tsdbRetrieveDatablockSMA;
+
+ pReader->tsdReaderNotifyClosing = tsdbReaderSetCloseFlag;
+ pReader->tsdReaderResetStatus = tsdbReaderReset;
+
+ pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo;
+ pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away
+
+ pReader->tsdSetQueryTableList = tsdbSetTableList;
+ pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId;
+}
+
+void initMetadataAPI(SStoreMeta* pMeta) {
+ pMeta->isTableExisted = metaIsTableExist;
+
+ pMeta->openTableMetaCursor = metaOpenTbCursor;
+ pMeta->closeTableMetaCursor = metaCloseTbCursor;
+ pMeta->cursorNext = metaTbCursorNext;
+ pMeta->cursorPrev = metaTbCursorPrev;
+
+ pMeta->getBasicInfo = vnodeGetInfo;
+ pMeta->getNumOfChildTables = metaGetStbStats;
+
+ pMeta->getChildTableList = vnodeGetCtbIdList;
+
+ pMeta->storeGetIndexInfo = vnodeGetIdx;
+ pMeta->getInvertIndex = vnodeGetIvtIdx;
+
+ pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal;
+ pMeta->getTableTags = metaGetTableTags;
+ pMeta->getTableTagsByUid = metaGetTableTagsByUids;
+
+ pMeta->getTableUidByName = metaGetTableUidByName;
+ pMeta->getTableTypeByName = metaGetTableTypeByName;
+ pMeta->getTableNameByUid = metaGetTableNameByUid;
+
+ pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor
+ pMeta->storeGetTableList = vnodeGetTableList;
+}
+
+void initTqAPI(SStoreTqReader* pTq) {
+ pTq->tqReaderOpen = tqReaderOpen;
+ pTq->tqReaderSetColIdList = tqReaderSetColIdList;
+
+ pTq->tqReaderClose = tqReaderClose;
+ pTq->tqReaderSeek = tqReaderSeek;
+ pTq->tqRetrieveBlock = tqRetrieveDataBlock;
+
+ pTq->tqReaderNextBlockInWal = tqNextBlockInWal;
+
+ pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it
+
+ pTq->tqReaderAddTables = tqReaderAddTbUidList;
+ pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList;
+
+ pTq->tqReaderRemoveTables = tqReaderRemoveTbUidList;
+
+ pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable;
+ pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed;
+
+ pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it
+ pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it
+
+ pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it
+ pTq->tqGetResultBlock = tqGetResultBlock;
+
+ pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
+}
+
+void initStateStoreAPI(SStateStore* pStore) {
+ pStore->streamFileStateInit = streamFileStateInit;
+ pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF;
+
+ pStore->streamStateGetByPos = streamStateGetByPos;
+
+ pStore->streamStatePutParName = streamStatePutParName;
+ pStore->streamStateGetParName = streamStateGetParName;
+
+ pStore->streamStateAddIfNotExist = streamStateAddIfNotExist;
+ pStore->streamStateReleaseBuf = streamStateReleaseBuf;
+ pStore->streamStateFreeVal = streamStateFreeVal;
+
+ pStore->streamStatePut = streamStatePut;
+ pStore->streamStateGet = streamStateGet;
+ pStore->streamStateCheck = streamStateCheck;
+ pStore->streamStateGetByPos = streamStateGetByPos;
+ pStore->streamStateDel = streamStateDel;
+ pStore->streamStateClear = streamStateClear;
+ pStore->streamStateSaveInfo = streamStateSaveInfo;
+ pStore->streamStateGetInfo = streamStateGetInfo;
+ pStore->streamStateSetNumber = streamStateSetNumber;
+
+ pStore->streamStateFillPut = streamStateFillPut;
+ pStore->streamStateFillGet = streamStateFillGet;
+ pStore->streamStateFillDel = streamStateFillDel;
+
+ pStore->streamStateCurNext = streamStateCurNext;
+ pStore->streamStateCurPrev = streamStateCurPrev;
+
+ pStore->streamStateGetAndCheckCur = streamStateGetAndCheckCur;
+ pStore->streamStateSeekKeyNext = streamStateSeekKeyNext;
+ pStore->streamStateFillSeekKeyNext = streamStateFillSeekKeyNext;
+ pStore->streamStateFillSeekKeyPrev = streamStateFillSeekKeyPrev;
+ pStore->streamStateFreeCur = streamStateFreeCur;
+
+ pStore->streamStateGetGroupKVByCur = streamStateGetGroupKVByCur;
+ pStore->streamStateGetKVByCur = streamStateGetKVByCur;
+
+ pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
+ pStore->streamStateSessionPut = streamStateSessionPut;
+ pStore->streamStateSessionGet = streamStateSessionGet;
+ pStore->streamStateSessionDel = streamStateSessionDel;
+ pStore->streamStateSessionClear = streamStateSessionClear;
+ pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur;
+ pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist;
+ pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange;
+
+ pStore->updateInfoInit = updateInfoInit;
+ pStore->updateInfoFillBlockData = updateInfoFillBlockData;
+ pStore->updateInfoIsUpdated = updateInfoIsUpdated;
+ pStore->updateInfoIsTableInserted = updateInfoIsTableInserted;
+ pStore->updateInfoDestroy = updateInfoDestroy;
+
+ pStore->updateInfoInitP = updateInfoInitP;
+ pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF;
+ pStore->updateInfoDestoryColseWinSBF = updateInfoDestoryColseWinSBF;
+ pStore->updateInfoSerialize = updateInfoSerialize;
+ pStore->updateInfoDeserialize = updateInfoDeserialize;
+
+ pStore->streamStateSessionSeekKeyNext = streamStateSessionSeekKeyNext;
+ pStore->streamStateSessionSeekKeyCurrentPrev = streamStateSessionSeekKeyCurrentPrev;
+ pStore->streamStateSessionSeekKeyCurrentNext = streamStateSessionSeekKeyCurrentNext;
+
+ pStore->streamFileStateInit = streamFileStateInit;
+
+ pStore->streamFileStateDestroy = streamFileStateDestroy;
+ pStore->streamFileStateClear = streamFileStateClear;
+ pStore->needClearDiskBuff = needClearDiskBuff;
+
+ pStore->streamStateOpen = streamStateOpen;
+ pStore->streamStateClose = streamStateClose;
+ pStore->streamStateBegin = streamStateBegin;
+ pStore->streamStateCommit = streamStateCommit;
+ pStore->streamStateDestroy= streamStateDestroy;
+ pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
+}
+
+void initMetaReaderAPI(SStoreMetaReader* pMetaReader) {
+ pMetaReader->initReader = _metaReaderInit;
+ pMetaReader->clearReader = metaReaderClear;
+
+ pMetaReader->getTableEntryByUid = metaReaderGetTableEntryByUid;
+
+ pMetaReader->getEntryGetUidCache = metaReaderGetTableEntryByUidCache;
+ pMetaReader->getTableEntryByName = metaGetTableEntryByName;
+
+ pMetaReader->readerReleaseLock = metaReaderReleaseLock;
+}
+
+void initMetaFilterAPI(SMetaDataFilterAPI* pFilter) {
+ pFilter->metaFilterCreateTime = metaFilterCreateTime;
+ pFilter->metaFilterTableIds = metaFilterTableIds;
+ pFilter->metaFilterTableName = metaFilterTableName;
+ pFilter->metaFilterTtl = metaFilterTtl;
+}
+
+void initFunctionStateStore(SFunctionStateStore* pStore) {
+ pStore->streamStateFuncPut = streamStateFuncPut;
+ pStore->streamStateFuncGet = streamStateFuncGet;
+}
+
+void initCacheFn(SStoreCacheReader* pCache) {
+ pCache->openReader = tsdbCacherowsReaderOpen;
+ pCache->closeReader = tsdbCacherowsReaderClose;
+ pCache->retrieveRows = tsdbRetrieveCacheRows;
+ pCache->reuseReader = tsdbReuseCacherowsReader;
+}
+
+void initSnapshotFn(SStoreSnapshotFn* pSnapshot) {
+ pSnapshot->createSnapshot = setForSnapShot;
+ pSnapshot->destroySnapshot = destroySnapContext;
+ pSnapshot->getMetaTableInfoFromSnapshot = getMetaTableInfoFromSnapshot;
+ pSnapshot->getTableInfoFromSnapshot = getTableInfoFromSnapshot;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index 303d2a9ca4..022fc4c951 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -80,7 +80,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaRsp.suid = mer1.me.uid;
} else if (mer1.me.type == TSDB_CHILD_TABLE) {
metaReaderInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
- if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
+ if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
strcpy(metaRsp.stbName, mer2.me.name);
metaRsp.suid = mer2.me.uid;
@@ -189,7 +189,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
goto _exit;
} else if (mer1.me.type == TSDB_CHILD_TABLE) {
metaReaderInit(&mer2, pVnode->pMeta, 0);
- if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
+ if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
strcpy(cfgRsp.stbName, mer2.me.name);
schema = mer2.me.stbEntry.schemaRow;
@@ -410,13 +410,32 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
"nBatchInsertSuccess");
}
-void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
+void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables) {
+ SVnode* pVnodeObj = pVnode;
+ SVnodeCfg* pConf = &pVnodeObj->config;
+
if (dbname) {
- *dbname = pVnode->config.dbname;
+ *dbname = pConf->dbname;
}
if (vgId) {
- *vgId = TD_VID(pVnode);
+ *vgId = TD_VID(pVnodeObj);
+ }
+
+ if (numOfTables) {
+ *numOfTables = pConf->vndStats.numOfNTables + pConf->vndStats.numOfCTables;
+ }
+
+ if (numOfNormalTables) {
+ *numOfNormalTables = pConf->vndStats.numOfNTables;
+ }
+}
+
+int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) {
+ if (type == TSDB_SUPER_TABLE) {
+ return vnodeGetStbIdList(pVnode, 0, pList);
+ } else {
+ return TSDB_CODE_INVALID_PARA;
}
}
@@ -440,8 +459,10 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) {
return 0;
}
-int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
- SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 1);
+
+int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
+ SVnode *pVnodeObj = pVnode;
+ SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj->pMeta, suid, 1);
while (1) {
tb_uid_t id = metaCtbCursorNext(pCur);
@@ -529,10 +550,8 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
for (int64_t i = 0; i < arrSize; ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
- SMetaStbStats stats = {0};
- metaGetStbStats(pVnode->pMeta, suid, &stats);
- int64_t ctbNum = stats.ctbNum;
- // vnodeGetCtbNum(pVnode, id, &ctbNum);
+ int64_t ctbNum = 0;
+ metaGetStbStats(pVnode, suid, &ctbNum);
int numOfCols = 0;
vnodeGetStbColumnNum(pVnode, suid, &numOfCols);
@@ -567,16 +586,17 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) {
return TSDB_CODE_SUCCESS;
}
-void *vnodeGetIdx(SVnode *pVnode) {
+void *vnodeGetIdx(void *pVnode) {
if (pVnode == NULL) {
return NULL;
}
- return metaGetIdx(pVnode->pMeta);
+
+ return metaGetIdx(((SVnode*)pVnode)->pMeta);
}
-void *vnodeGetIvtIdx(SVnode *pVnode) {
+void *vnodeGetIvtIdx(void *pVnode) {
if (pVnode == NULL) {
return NULL;
}
- return metaGetIvtIdx(pVnode->pMeta);
+ return metaGetIvtIdx(((SVnode*)pVnode)->pMeta);
}
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index fe1ccf90c8..b950437f23 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -19,23 +19,23 @@
#include "vnode.h"
#include "vnodeInt.h"
-static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime, int64_t *pUid) {
int32_t code = 0;
@@ -250,7 +250,9 @@ static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
uint8_t *pCont;
SEncoder *pCoder = &(SEncoder){0};
SDeleteRes res = {0};
- SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
+
+ SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
+ initStorageAPI(&handle.api);
code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res);
if (code) goto _exit;
@@ -301,32 +303,32 @@ _exit:
return code;
}
-int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) {
+int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg *pRsp) {
void *ptr = NULL;
void *pReq;
int32_t len;
int32_t ret;
- if (version <= pVnode->state.applied) {
- vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version,
+ if (ver <= pVnode->state.applied) {
+ vError("vgId:%d, duplicate write request. ver: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), ver,
pVnode->state.applied);
terrno = TSDB_CODE_VND_DUP_REQUEST;
return -1;
}
vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
- version);
+ ver);
ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm);
- ASSERT(pVnode->state.applied + 1 == version);
+ ASSERT(pVnode->state.applied + 1 == ver);
- atomic_store_64(&pVnode->state.applied, version);
+ atomic_store_64(&pVnode->state.applied, ver);
atomic_store_64(&pVnode->state.applyTerm, pMsg->info.conn.applyTerm);
if (!syncUtilUserCommit(pMsg->msgType)) goto _exit;
if (pMsg->msgType == TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE || pMsg->msgType == TDMT_STREAM_TASK_CHECK_RSP) {
- if (tqCheckLogInWal(pVnode->pTq, version)) return 0;
+ if (tqCheckLogInWal(pVnode->pTq, ver)) return 0;
}
// skip header
@@ -337,123 +339,123 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
switch (pMsg->msgType) {
/* META */
case TDMT_VND_CREATE_STB:
- if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessCreateStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_ALTER_STB:
- if (vnodeProcessAlterStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessAlterStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_DROP_STB:
- if (vnodeProcessDropStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessDropStbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_CREATE_TABLE:
- if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessCreateTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_ALTER_TABLE:
- if (vnodeProcessAlterTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessAlterTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_DROP_TABLE:
- if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessDropTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_DROP_TTL_TABLE:
- if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessDropTtlTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_TRIM:
- if (vnodeProcessTrimReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_CREATE_SMA:
- if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessCreateTSmaReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
/* TSDB */
case TDMT_VND_SUBMIT:
- if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err;
+ if (vnodeProcessSubmitReq(pVnode, ver, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err;
break;
case TDMT_VND_DELETE:
- if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_BATCH_DEL:
- if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+ if (vnodeProcessBatchDeleteReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
/* TQ */
case TDMT_VND_TMQ_SUBSCRIBE:
- if (tqProcessSubscribeReq(pVnode->pTq, version, pReq, len) < 0) {
+ if (tqProcessSubscribeReq(pVnode->pTq, ver, pReq, len) < 0) {
goto _err;
}
break;
case TDMT_VND_TMQ_DELETE_SUB:
- if (tqProcessDeleteSubReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+ if (tqProcessDeleteSubReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
goto _err;
}
break;
case TDMT_VND_TMQ_COMMIT_OFFSET:
- if (tqProcessOffsetCommitReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
+ if (tqProcessOffsetCommitReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
goto _err;
}
break;
case TDMT_VND_TMQ_SEEK_TO_OFFSET:
- if (tqProcessSeekReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
+ if (tqProcessSeekReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
goto _err;
}
break;
case TDMT_VND_TMQ_ADD_CHECKINFO:
- if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) {
+ if (tqProcessAddCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) {
goto _err;
}
break;
case TDMT_VND_TMQ_DEL_CHECKINFO:
- if (tqProcessDelCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) {
+ if (tqProcessDelCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) {
goto _err;
}
break;
case TDMT_STREAM_TASK_DEPLOY: {
- if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) {
+ if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, ver, pReq, len) < 0) {
goto _err;
}
} break;
case TDMT_STREAM_TASK_DROP: {
- if (tqProcessTaskDropReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+ if (tqProcessTaskDropReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
goto _err;
}
} break;
case TDMT_STREAM_TASK_PAUSE: {
- if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+ if (pVnode->restored && tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
goto _err;
}
} break;
case TDMT_STREAM_TASK_RESUME: {
- if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+ if (pVnode->restored && tqProcessTaskResumeReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
goto _err;
}
} break;
case TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE: {
- if (tqProcessTaskRecover2Req(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) {
+ if (tqProcessTaskRecover2Req(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) {
goto _err;
}
} break;
case TDMT_STREAM_TASK_CHECK_RSP: {
- if (tqProcessStreamTaskCheckRsp(pVnode->pTq, version, pReq, len) < 0) {
+ if (tqProcessStreamTaskCheckRsp(pVnode->pTq, ver, pReq, len) < 0) {
goto _err;
}
} break;
case TDMT_VND_ALTER_CONFIRM:
needCommit = pVnode->config.hashChange;
- if (vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp) < 0) {
+ if (vnodeProcessAlterConfirmReq(pVnode, ver, pReq, len, pRsp) < 0) {
goto _err;
}
break;
case TDMT_VND_ALTER_CONFIG:
- vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp);
+ vnodeProcessAlterConfigReq(pVnode, ver, pReq, len, pRsp);
break;
case TDMT_VND_COMMIT:
needCommit = true;
break;
case TDMT_VND_CREATE_INDEX:
- vnodeProcessCreateIndexReq(pVnode, version, pReq, len, pRsp);
+ vnodeProcessCreateIndexReq(pVnode, ver, pReq, len, pRsp);
break;
case TDMT_VND_DROP_INDEX:
- vnodeProcessDropIndexReq(pVnode, version, pReq, len, pRsp);
+ vnodeProcessDropIndexReq(pVnode, ver, pReq, len, pRsp);
break;
case TDMT_VND_COMPACT:
- vnodeProcessCompactVnodeReq(pVnode, version, pReq, len, pRsp);
+ vnodeProcessCompactVnodeReq(pVnode, ver, pReq, len, pRsp);
goto _exit;
default:
vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType);
@@ -461,18 +463,18 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
}
vTrace("vgId:%d, process %s request, code:0x%x index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), pRsp->code,
- version);
+ ver);
- walApplyVer(pVnode->pWal, version);
+ walApplyVer(pVnode->pWal, ver);
- if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
+ if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) {
vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
// commit if need
if (needCommit) {
- vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version);
+ vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), ver);
if (vnodeAsyncCommit(pVnode) < 0) {
vError("vgId:%d, failed to vnode async commit since %s.", TD_VID(pVnode), tstrerror(terrno));
goto _err;
@@ -489,8 +491,8 @@ _exit:
return 0;
_err:
- vError("vgId:%d, process %s request failed since %s, version:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
- tstrerror(terrno), version);
+ vError("vgId:%d, process %s request failed since %s, ver:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
+ tstrerror(terrno), ver);
return -1;
}
@@ -514,7 +516,9 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
return 0;
}
- SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
+ SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb};
+ initStorageAPI(&handle.api);
+
switch (pMsg->msgType) {
case TDMT_SCH_QUERY:
case TDMT_SCH_MERGE_QUERY:
@@ -603,7 +607,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) {
}
extern int32_t vnodeAsyncRentention(SVnode *pVnode, int64_t now);
-static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0;
SVTrimDbReq trimReq = {0};
@@ -624,7 +628,7 @@ _exit:
return code;
}
-static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SArray *tbUids = taosArrayInit(8, sizeof(int64_t));
if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY;
@@ -650,7 +654,7 @@ end:
return ret;
}
-static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVCreateStbReq req = {0};
SDecoder coder;
@@ -667,7 +671,7 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p
goto _err;
}
- if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) {
+ if (metaCreateSTable(pVnode->pMeta, ver, &req) < 0) {
pRsp->code = terrno;
goto _err;
}
@@ -685,7 +689,7 @@ _err:
return -1;
}
-static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SDecoder decoder = {0};
SEncoder encoder = {0};
int32_t rcode = 0;
@@ -742,7 +746,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR
}
// do create table
- if (metaCreateTable(pVnode->pMeta, version, pCreateReq, &cRsp.pMeta) < 0) {
+ if (metaCreateTable(pVnode->pMeta, ver, pCreateReq, &cRsp.pMeta) < 0) {
if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
cRsp.code = TSDB_CODE_SUCCESS;
} else {
@@ -790,7 +794,7 @@ _exit:
return rcode;
}
-static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVCreateStbReq req = {0};
SDecoder dc = {0};
@@ -808,7 +812,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR
return -1;
}
- if (metaAlterSTable(pVnode->pMeta, version, &req) < 0) {
+ if (metaAlterSTable(pVnode->pMeta, ver, &req) < 0) {
pRsp->code = terrno;
tDecoderClear(&dc);
return -1;
@@ -819,7 +823,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pR
return 0;
}
-static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVDropStbReq req = {0};
int32_t rcode = TSDB_CODE_SUCCESS;
SDecoder decoder = {0};
@@ -839,7 +843,7 @@ static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pRe
// process request
tbUidList = taosArrayInit(8, sizeof(int64_t));
if (tbUidList == NULL) goto _exit;
- if (metaDropSTable(pVnode->pMeta, version, &req, tbUidList) < 0) {
+ if (metaDropSTable(pVnode->pMeta, ver, &req, tbUidList) < 0) {
rcode = terrno;
goto _exit;
}
@@ -862,7 +866,7 @@ _exit:
return 0;
}
-static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVAlterTbReq vAlterTbReq = {0};
SVAlterTbRsp vAlterTbRsp = {0};
SDecoder dc = {0};
@@ -887,7 +891,7 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pRe
}
// process
- if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq, &vMetaRsp) < 0) {
+ if (metaAlterTable(pVnode->pMeta, ver, &vAlterTbReq, &vMetaRsp) < 0) {
vAlterTbRsp.code = terrno;
tDecoderClear(&dc);
rcode = -1;
@@ -912,7 +916,7 @@ _exit:
return 0;
}
-static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVDropTbBatchReq req = {0};
SVDropTbBatchRsp rsp = {0};
SDecoder decoder = {0};
@@ -946,7 +950,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq
tb_uid_t tbUid = 0;
/* code */
- ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, &tbUid);
+ ret = metaDropTable(pVnode->pMeta, ver, pDropTbReq, tbUids, &tbUid);
if (ret < 0) {
if (pDropTbReq->igNotExists && terrno == TSDB_CODE_TDB_TABLE_NOT_EXIST) {
dropTbRsp.code = TSDB_CODE_SUCCESS;
@@ -1189,7 +1193,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) {
return code;
}
-static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0;
terrno = 0;
@@ -1251,7 +1255,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
for (int32_t iRow = 0; iRow < pColData->nVal; iRow++) {
if (aKey[iRow] < minKey || aKey[iRow] > maxKey || (iRow > 0 && aKey[iRow] <= aKey[iRow - 1])) {
code = TSDB_CODE_INVALID_MSG;
- vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version);
+ vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver);
goto _exit;
}
}
@@ -1263,7 +1267,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
for (int32_t iRow = 0; iRow < nRow; ++iRow) {
if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) {
code = TSDB_CODE_INVALID_MSG;
- vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), version);
+ vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(terrno), ver);
goto _exit;
}
}
@@ -1350,7 +1354,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1);
// create table
- if (metaCreateTable(pVnode->pMeta, version, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) {
+ if (metaCreateTable(pVnode->pMeta, ver, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) {
// create table success
if (newTbUids == NULL &&
@@ -1376,7 +1380,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
// insert data
int32_t affectedRows;
- code = tsdbInsertTableData(pVnode->pTsdb, version, pSubmitTbData, &affectedRows);
+ code = tsdbInsertTableData(pVnode->pTsdb, ver, pSubmitTbData, &affectedRows);
if (code) goto _exit;
pSubmitRsp->affectedRows += affectedRows;
@@ -1404,7 +1408,7 @@ _exit:
atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1);
if (code == 0) {
atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1);
- tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT);
+ tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT);
}
// clear
@@ -1419,7 +1423,7 @@ _exit:
return code;
}
-static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVCreateTSmaReq req = {0};
SDecoder coder = {0};
@@ -1439,20 +1443,20 @@ static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *
goto _err;
}
- if (tdProcessTSmaCreate(pVnode->pSma, version, (const char *)&req) < 0) {
+ if (tdProcessTSmaCreate(pVnode->pSma, ver, (const char *)&req) < 0) {
if (pRsp) pRsp->code = terrno;
goto _err;
}
tDecoderClear(&coder);
vDebug("vgId:%d, success to create tsma %s:%" PRIi64 " version %" PRIi64 " for table %" PRIi64, TD_VID(pVnode),
- req.indexName, req.indexUid, version, req.tableUid);
+ req.indexName, req.indexUid, ver, req.tableUid);
return 0;
_err:
tDecoderClear(&coder);
vError("vgId:%d, failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s",
- TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr());
+ TD_VID(pVnode), req.indexName, req.indexUid, ver, req.tableUid, terrstr());
return -1;
}
@@ -1468,28 +1472,28 @@ int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) {
return vnodeProcessCreateTSmaReq(pVnode, 1, pCont, contLen, NULL);
}
-static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t version) {
+static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) {
int32_t code = TSDB_CODE_SUCCESS;
vInfo("vgId:%d, trim meta of tables per hash range [%" PRIu32 ", %" PRIu32 "]. apply-index:%" PRId64, TD_VID(pVnode),
- pVnode->config.hashBegin, pVnode->config.hashEnd, version);
+ pVnode->config.hashBegin, pVnode->config.hashEnd, ver);
// TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd]
return code;
}
-static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
vInfo("vgId:%d, vnode handle msgType:alter-confirm, alter confim msg is processed", TD_VID(pVnode));
int32_t code = TSDB_CODE_SUCCESS;
if (!pVnode->config.hashChange) {
goto _exit;
}
- code = vnodeConsolidateAlterHashRange(pVnode, version);
+ code = vnodeConsolidateAlterHashRange(pVnode, ver);
if (code < 0) {
vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(),
- version);
+ ver);
goto _exit;
}
pVnode->config.hashChange = false;
@@ -1503,7 +1507,7 @@ _exit:
return code;
}
-static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
bool walChanged = false;
bool tsdbChanged = false;
@@ -1606,7 +1610,7 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void
return 0;
}
-static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SBatchDeleteReq deleteReq;
SDecoder decoder;
tDecoderInit(&decoder, pReq, len);
@@ -1626,7 +1630,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void
int64_t uid = mr.me.uid;
- int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs);
+ int32_t code = tsdbDeleteTableData(pVnode->pTsdb, ver, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs);
if (code < 0) {
terrno = code;
vError("vgId:%d, delete error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 ", end ts:%" PRId64,
@@ -1640,7 +1644,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void
return 0;
}
-static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0;
SDecoder *pCoder = &(SDecoder){0};
SDeleteRes *pRes = &(SDeleteRes){0};
@@ -1661,7 +1665,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq
ASSERT(taosArrayGetSize(pRes->uidList) == 0 || (pRes->skey != 0 && pRes->ekey != 0));
for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) {
- code = tsdbDeleteTableData(pVnode->pTsdb, version, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid),
+ code = tsdbDeleteTableData(pVnode->pTsdb, ver, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid),
pRes->skey, pRes->ekey);
if (code) goto _err;
}
@@ -1682,7 +1686,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq
_err:
return code;
}
-static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVCreateStbReq req = {0};
SDecoder dc = {0};
@@ -1698,7 +1702,7 @@ static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void
tDecoderClear(&dc);
return -1;
}
- if (metaAddIndexToSTable(pVnode->pMeta, version, &req) < 0) {
+ if (metaAddIndexToSTable(pVnode->pMeta, ver, &req) < 0) {
pRsp->code = terrno;
goto _err;
}
@@ -1708,7 +1712,7 @@ _err:
tDecoderClear(&dc);
return -1;
}
-static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SDropIndexReq req = {0};
pRsp->msgType = TDMT_VND_DROP_INDEX_RSP;
pRsp->code = TSDB_CODE_SUCCESS;
@@ -1720,21 +1724,21 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *p
return -1;
}
- if (metaDropIndexFromSTable(pVnode->pMeta, version, &req) < 0) {
+ if (metaDropIndexFromSTable(pVnode->pMeta, ver, &req) < 0) {
pRsp->code = terrno;
return -1;
}
return TSDB_CODE_SUCCESS;
}
-extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
+extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
-static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
- return vnodeProcessCompactVnodeReqImpl(pVnode, version, pReq, len, pRsp);
+static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
+ return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp);
}
#ifndef TD_ENTERPRISE
-int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
+int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
return 0;
}
#endif
diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt
index 8b3d04e32c..d2c39aba74 100644
--- a/source/libs/executor/CMakeLists.txt
+++ b/source/libs/executor/CMakeLists.txt
@@ -1,10 +1,9 @@
aux_source_directory(src EXECUTOR_SRC)
-#add_library(executor ${EXECUTOR_SRC})
add_library(executor STATIC ${EXECUTOR_SRC})
target_link_libraries(executor
- PRIVATE os util common function parser planner qcom vnode scalar nodes index stream
+ PRIVATE os util common function parser planner qcom scalar nodes index wal tdb
)
target_include_directories(
diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h
index 57a771b275..9893b4eb76 100644
--- a/source/libs/executor/inc/dataSinkInt.h
+++ b/source/libs/executor/inc/dataSinkInt.h
@@ -22,6 +22,7 @@ extern "C" {
#include "dataSinkMgt.h"
#include "plannodes.h"
+#include "storageapi.h"
#include "tcommon.h"
struct SDataSink;
@@ -29,6 +30,7 @@ struct SDataSinkHandle;
typedef struct SDataSinkManager {
SDataSinkMgtCfg cfg;
+ SStorageAPI* pAPI;
} SDataSinkManager;
typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue);
diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h
index 896858ac7f..966c5fa2e4 100644
--- a/source/libs/executor/inc/executil.h
+++ b/source/libs/executor/inc/executil.h
@@ -12,17 +12,17 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
-#ifndef TDENGINE_QUERYUTIL_H
-#define TDENGINE_QUERYUTIL_H
+#ifndef TDENGINE_EXECUTIL_H
+#define TDENGINE_EXECUTIL_H
#include "executor.h"
#include "function.h"
#include "nodes.h"
#include "plannodes.h"
+#include "storageapi.h"
#include "tcommon.h"
#include "tpagedbuf.h"
#include "tsimplehash.h"
-#include "vnode.h"
#define T_LONG_JMP(_obj, _c) \
do { \
@@ -154,7 +154,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode);
EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
-int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId);
+int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId, SStorageAPI* pAPI);
size_t getTableTagsBufLen(const SNodeList* pGroups);
SArray* createSortInfo(SNodeList* pNodeList);
@@ -166,7 +166,7 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId);
void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode);
SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs);
-SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset);
+SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore);
void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn);
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
@@ -178,8 +178,8 @@ void cleanupQueryTableDataCond(SQueryTableDataCond* pCond);
int32_t convertFillType(int32_t mode);
int32_t resultrowComparAsc(const void* p1, const void* p2);
-int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified);
+int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI);
void printDataBlock(SSDataBlock* pBlock, const char* flag);
-#endif // TDENGINE_QUERYUTIL_H
+#endif // TDENGINE_EXECUTIL_H
diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index 2daeb70260..79391bc1c5 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -38,13 +38,15 @@ extern "C" {
#include "tlockfree.h"
#include "tmsg.h"
#include "tpagedbuf.h"
-#include "tstream.h"
-#include "tstreamUpdate.h"
-
-#include "vnode.h"
+//#include "tstream.h"
+//#include "tstreamUpdate.h"
+#include "tlrucache.h"
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
+typedef struct STsdbReader STsdbReader;
+typedef struct STqReader STqReader;
+
#define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0)
#define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN)
#define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0)
@@ -206,6 +208,7 @@ typedef struct STableScanBase {
SLimitInfo limitInfo;
// there are more than one table list exists in one task, if only one vnode exists.
STableListInfo* pTableListInfo;
+ TsdReader readerAPI;
} STableScanBase;
typedef struct STableScanInfo {
@@ -221,6 +224,7 @@ typedef struct STableScanInfo {
int8_t assignBlockUid;
bool hasGroupByTag;
bool countOnly;
+// TsdReader readerAPI;
} STableScanInfo;
typedef struct STableMergeScanInfo {
@@ -280,6 +284,7 @@ typedef struct SStreamAggSupporter {
int32_t stateKeySize;
int16_t stateKeyType;
SDiskbasedBuf* pResultBuf;
+ SStateStore stateStore;
} SStreamAggSupporter;
typedef struct SWindowSupporter {
@@ -335,7 +340,7 @@ typedef struct SStreamScanInfo {
STqReader* tqReader;
uint64_t groupId;
- SUpdateInfo* pUpdateInfo;
+ struct SUpdateInfo* pUpdateInfo;
EStreamScanMode scanMode;
struct SOperatorInfo* pStreamScanOp;
@@ -366,15 +371,18 @@ typedef struct SStreamScanInfo {
SSDataBlock* pCreateTbRes;
int8_t igCheckUpdate;
int8_t igExpired;
- SStreamState* pState;
+ void* pState; //void
+ SStoreTqReader readerFn;
+ SStateStore stateStore;
} SStreamScanInfo;
typedef struct {
- SVnode* vnode;
- SSDataBlock pRes; // result SSDataBlock
- STsdbReader* dataReader;
- SSnapContext* sContext;
- STableListInfo* pTableListInfo;
+ struct SVnode* vnode; // todo remove this
+ SSDataBlock pRes; // result SSDataBlock
+ STsdbReader* dataReader;
+ struct SSnapContext* sContext;
+ SStorageAPI* pAPI;
+ STableListInfo* pTableListInfo;
} SStreamRawScanInfo;
typedef struct STableCountScanSupp {
@@ -441,12 +449,13 @@ typedef struct SStreamIntervalOperatorInfo {
bool isFinal;
SArray* pChildren;
int32_t numOfChild;
- SStreamState* pState;
+ SStreamState* pState; // void
SWinKey delKey;
uint64_t numOfDatapack;
SArray* pUpdated;
SSHashObj* pUpdatedMap;
int64_t dataVersion;
+ SStateStore statestore;
} SStreamIntervalOperatorInfo;
typedef struct SDataGroupInfo {
@@ -543,6 +552,7 @@ typedef struct SStreamFillSupporter {
int32_t rowSize;
SSHashObj* pResMap;
bool hasDelete;
+ SStorageAPI* pAPI;
} SStreamFillSupporter;
typedef struct SStreamFillOperatorInfo {
@@ -569,12 +579,11 @@ void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo);
void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
void cleanupBasicInfo(SOptrBasicInfo* pInfo);
-int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr);
+int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore);
void cleanupExprSupp(SExprSupp* pSup);
-
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
- const char* pkey, void* pState);
+ const char* pkey, void* pState, SFunctionStateStore* pStore);
void cleanupAggSup(SAggSupporter* pAggSup);
void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
@@ -636,7 +645,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap);
bool functionNeedToExecute(SqlFunctionCtx* pCtx);
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
-bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup);
+bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore);
void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
uint64_t* pGp, void* pTbName);
uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
@@ -645,20 +654,17 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos
SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
bool groupbyTbname(SNodeList* pGroupList);
-int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
+int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo);
-int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size);
-int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock,
+int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI);
+int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
SExprSupp* pSup, SGroupResInfo* pGroupResInfo);
-int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId,
- SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup);
-int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult);
-int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
+int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI);
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order);
int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order,
int64_t* pData);
-void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
- SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock);
+void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
+ SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI);
SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag);
SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs);
diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h
index 632b817a07..1d2685b8c6 100644
--- a/source/libs/executor/inc/operator.h
+++ b/source/libs/executor/inc/operator.h
@@ -156,7 +156,7 @@ void destroyOperator(SOperatorInfo* pOperator);
SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id);
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder);
-int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr);
+int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI);
int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList);
#ifdef __cplusplus
diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h
index 37c93fef5c..6497bd90b4 100644
--- a/source/libs/executor/inc/querytask.h
+++ b/source/libs/executor/inc/querytask.h
@@ -56,20 +56,19 @@ typedef struct STaskStopInfo {
} STaskStopInfo;
typedef struct {
- STqOffsetVal currentOffset; // for tmq
- SMqMetaRsp metaRsp; // for tmq fetching meta
- int64_t snapshotVer;
-// SPackedData submit; // todo remove it
- SSchemaWrapper* schema;
- char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
- int8_t recoverStep;
- int8_t recoverScanFinished;
- SQueryTableDataCond tableCond;
- int64_t fillHistoryVer1;
- int64_t fillHistoryVer2;
- SStreamState* pState;
- int64_t dataVersion;
- int64_t checkPointId;
+ STqOffsetVal currentOffset; // for tmq
+ SMqMetaRsp metaRsp; // for tmq fetching meta
+ int64_t snapshotVer;
+ SSchemaWrapper* schema;
+ char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
+ int8_t recoverStep;
+ int8_t recoverScanFinished;
+ SQueryTableDataCond tableCond;
+ int64_t fillHistoryVer1;
+ int64_t fillHistoryVer2;
+ SStreamState* pState;
+ int64_t dataVersion;
+ int64_t checkPointId;
} SStreamTaskInfo;
struct SExecTaskInfo {
@@ -92,10 +91,11 @@ struct SExecTaskInfo {
SArray* pResultBlockList; // result block list
STaskStopInfo stopInfo;
SRWLatch lock; // secure the access of STableListInfo
+ SStorageAPI storageAPI;
};
void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst);
-SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model);
+SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI);
void doDestroyTask(SExecTaskInfo* pTaskInfo);
bool isTaskKilled(SExecTaskInfo* pTaskInfo);
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode);
diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c
index 5cd95d3311..b34c6f4b82 100644
--- a/source/libs/executor/src/aggregateoperator.c
+++ b/source/libs/executor/src/aggregateoperator.c
@@ -21,7 +21,6 @@
#include "tname.h"
#include "executorInt.h"
-#include "index.h"
#include "operator.h"
#include "query.h"
#include "querytask.h"
@@ -30,6 +29,7 @@
#include "tglobal.h"
#include "thash.h"
#include "ttypes.h"
+#include "index.h"
typedef struct {
bool hasAgg;
@@ -84,7 +84,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
+ pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -95,7 +95,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr);
}
- code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr);
+ code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -485,8 +485,8 @@ void cleanupAggSup(SAggSupporter* pAggSup) {
}
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
- const char* pkey, void* pState) {
- int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
+ const char* pkey, void* pState, SFunctionStateStore* pStore) {
+ int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c
index 96dac643a5..48003b277b 100644
--- a/source/libs/executor/src/cachescanoperator.c
+++ b/source/libs/executor/src/cachescanoperator.c
@@ -13,8 +13,8 @@
* along with this program. If not, see .
*/
-#include "function.h"
#include "os.h"
+#include "function.h"
#include "tname.h"
#include "tdatablock.h"
@@ -27,6 +27,8 @@
#include "thash.h"
#include "ttypes.h"
+#include "storageapi.h"
+
typedef struct SCacheRowsScanInfo {
SSDataBlock* pRes;
SReadHandle readHandle;
@@ -102,9 +104,9 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
uint64_t suid = tableListGetSuid(pTableListInfo);
- code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
- taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds, suid,
- &pInfo->pLastrowReader, pTaskInfo->id.str);
+ code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
+ taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds,
+ suid, &pInfo->pLastrowReader, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -124,7 +126,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
if (pScanNode->scan.pScanPseudoCols != NULL) {
SExprSupp* p = &pInfo->pseudoExprSup;
p->pExprInfo = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &p->numOfExprs);
- p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset);
+ p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore);
}
setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED,
@@ -172,7 +174,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
blockDataCleanup(pInfo->pBufferredRes);
taosArrayClear(pInfo->pUidList);
- int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds,
+ int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds,
pInfo->pDstSlotIds, pInfo->pUidList);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
@@ -239,7 +241,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
}
if (NULL == pInfo->pLastrowReader) {
- code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
+ code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader,
pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
@@ -248,12 +250,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
continue;
}
} else {
- tsdbReuseCacherowsReader(pInfo->pLastrowReader, pList, num);
+ pInfo->readHandle.api.cacheFn.reuseReader(pInfo->pLastrowReader, pList, num);
}
taosArrayClear(pInfo->pUidList);
- code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds,
+ code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds,
pInfo->pUidList);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
@@ -287,7 +289,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
}
}
- pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
+ pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader);
setOperatorCompleted(pOperator);
return NULL;
}
@@ -305,7 +307,7 @@ void destroyCacheScanOperator(void* param) {
tableListDestroy(pInfo->pTableList);
if (pInfo->pLastrowReader != NULL) {
- pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
+ pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader);
}
cleanupExprSupp(&pInfo->pseudoExprSup);
diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c
index d31ac0bc51..646964ebf4 100644
--- a/source/libs/executor/src/dataInserter.c
+++ b/source/libs/executor/src/dataInserter.c
@@ -17,6 +17,7 @@
#include "dataSinkMgt.h"
#include "executorInt.h"
#include "planner.h"
+#include "storageapi.h"
#include "tcompression.h"
#include "tdatablock.h"
#include "tglobal.h"
@@ -428,8 +429,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
inserter->explain = pInserterNode->explain;
int64_t suid = 0;
- int32_t code =
- tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid);
+ int32_t code = pManager->pAPI->metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid);
if (code) {
destroyDataSinker((SDataSinkHandle*)inserter);
taosMemoryFree(inserter);
diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c
index b3cb57325b..3a972c1c20 100644
--- a/source/libs/executor/src/dataSinkMgt.c
+++ b/source/libs/executor/src/dataSinkMgt.c
@@ -21,8 +21,9 @@
static SDataSinkManager gDataSinkManager = {0};
SDataSinkStat gDataSinkStat = {0};
-int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg) {
+int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI) {
gDataSinkManager.cfg = *cfg;
+ gDataSinkManager.pAPI = pAPI;
return 0; // to avoid compiler eror
}
diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c
index 956d5b714d..b5dea6d94d 100644
--- a/source/libs/executor/src/eventwindowoperator.c
+++ b/source/libs/executor/src/eventwindowoperator.c
@@ -92,7 +92,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
if (pEventWindowNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pEventWindowNode->window.pExprs, NULL, &numOfScalarExpr);
- code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
+ code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -110,7 +110,7 @@ SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNo
initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
+ pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index fa46715c22..562a6daf8a 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -27,6 +27,7 @@
#include "executorInt.h"
#include "querytask.h"
#include "tcompression.h"
+#include "storageapi.h"
typedef struct tagFilterAssist {
SHashObj* colHash;
@@ -41,13 +42,13 @@ typedef enum {
} FilterCondType;
static FilterCondType checkTagCond(SNode* cond);
-static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond);
-static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond);
+static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SStorageAPI* pAPI);
+static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond, SStorageAPI* pStoreAPI);
-static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond,
- SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr);
+static int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond,
+ SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI);
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList,
- void* metaHandle);
+ void* pVnode, SStorageAPI* pStorageAPI);
static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; }
static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; }
@@ -262,7 +263,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
STagVal tagVal = {0};
tagVal.cid = pSColumnNode->colId;
- const char* p = metaGetTableTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal);
+ const char* p = mr->pAPI->extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal);
if (p == NULL) {
res->node.resType.type = TSDB_DATA_TYPE_NULL;
} else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) {
@@ -301,14 +302,14 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
-int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) {
+int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI) {
int32_t code = TSDB_CODE_SUCCESS;
SMetaReader mr = {0};
- metaReaderInit(&mr, metaHandle, 0);
- code = metaGetTableEntryByUidCache(&mr, info->uid);
+ pAPI->metaReaderFn.initReader(&mr, metaHandle, 0, &pAPI->metaFn);
+ code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, info->uid);
if (TSDB_CODE_SUCCESS != code) {
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
*pQualified = false;
return TSDB_CODE_SUCCESS;
@@ -317,7 +318,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle,
SNode* pTagCondTmp = nodesCloneNode(pTagCond);
nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr);
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
SNode* pNew = NULL;
code = scalarCalculateConstants(pTagCondTmp, &pNew);
@@ -435,7 +436,6 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) {
taosMemoryFree(payload);
}
-
static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_CTX* pContext) {
char* payload = NULL;
int32_t len = 0;
@@ -453,8 +453,8 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C
taosMemoryFree(payload);
}
-
-int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, uint8_t *digest) {
+int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInfo* pTableListInfo, uint8_t* digest,
+ SStorageAPI* pAPI) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* pBlockList = NULL;
SSDataBlock* pResBlock = NULL;
@@ -465,7 +465,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
int32_t rows = taosArrayGetSize(pTableListInfo->pTableList);
if (rows == 0) {
- return TDB_CODE_SUCCESS;
+ return TSDB_CODE_SUCCESS;
}
tagFilterAssist ctx = {0};
@@ -492,18 +492,19 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
if (tsTagFilterCache) {
SNodeListNode* listNode = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST);
listNode->pNodeList = group;
- genTbGroupDigest((SNode *)listNode, digest, &context);
+ genTbGroupDigest((SNode*)listNode, digest, &context);
nodesFree(listNode);
-
- metaGetCachedTbGroup(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList);
+
+ pAPI->metaFn.getCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList);
if (tableList) {
taosArrayDestroy(pTableListInfo->pTableList);
pTableListInfo->pTableList = tableList;
- qDebug("retrieve tb group list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pTableListInfo->pTableList));
+ qDebug("retrieve tb group list from cache, numOfTables:%d",
+ (int32_t)taosArrayGetSize(pTableListInfo->pTableList));
goto end;
}
}
-
+
pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo));
for (int32_t i = 0; i < rows; ++i) {
STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i);
@@ -511,14 +512,13 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
taosArrayPush(pUidTagList, &info);
}
- // int64_t stt = taosGetTimestampUs();
- code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList);
+ code = pAPI->metaFn.getTableTags(pVnode, pTableListInfo->idInfo.suid, pUidTagList);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
int32_t numOfTables = taosArrayGetSize(pUidTagList);
- pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle);
+ pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI);
if (pResBlock == NULL) {
code = terrno;
goto end;
@@ -632,9 +632,9 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
if (tsTagFilterCache) {
tableList = taosArrayDup(pTableListInfo->pTableList, NULL);
- metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo));
+ pAPI->metaFn.putTableListIntoCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo));
}
-
+
// int64_t st2 = taosGetTimestampUs();
// qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1);
@@ -734,12 +734,12 @@ static FilterCondType checkTagCond(SNode* cond) {
return FILTER_OTHER;
}
-static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond) {
+static int32_t optimizeTbnameInCond(void* pVnode, int64_t suid, SArray* list, SNode* cond, SStorageAPI* pAPI) {
int32_t ret = -1;
int32_t ntype = nodeType(cond);
if (ntype == QUERY_NODE_OPERATOR) {
- ret = optimizeTbnameInCondImpl(metaHandle, list, cond);
+ ret = optimizeTbnameInCondImpl(pVnode, list, cond, pAPI);
}
if (ntype != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) {
@@ -758,7 +758,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
SListCell* cell = pList->pHead;
for (int i = 0; i < len; i++) {
if (cell == NULL) break;
- if (optimizeTbnameInCondImpl(metaHandle, list, cell->pNode) == 0) {
+ if (optimizeTbnameInCondImpl(pVnode, list, cell->pNode, pAPI) == 0) {
hasTbnameCond = true;
break;
}
@@ -769,14 +769,14 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
taosArrayRemoveDuplicate(list, filterTableInfoCompare, NULL);
if (hasTbnameCond) {
- ret = metaGetTableTagsByUids(metaHandle, suid, list);
+ ret = pAPI->metaFn.getTableTagsByUid(pVnode, suid, list);
}
return ret;
}
// only return uid that does not contained in pExistedUidList
-static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) {
+static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, SNode* pTagCond, SStorageAPI* pStoreAPI) {
if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) {
return -1;
}
@@ -813,9 +813,9 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis
char* name = taosArrayGetP(pTbList, i);
uint64_t uid = 0;
- if (metaGetTableUidByName(metaHandle, name, &uid) == 0) {
+ if (pStoreAPI->metaFn.getTableUidByName(pVnode, name, &uid) == 0) {
ETableType tbType = TSDB_TABLE_MAX;
- if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
+ if (pStoreAPI->metaFn.getTableTypeByName(pVnode, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) {
STUidTagInfo s = {.uid = uid, .name = name, .pTagVal = NULL};
taosArrayPush(pExistedUidList, &s);
@@ -839,9 +839,8 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis
return -1;
}
-
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList,
- void* metaHandle) {
+ void* pVnode, SStorageAPI* pStorageAPI) {
SSDataBlock* pResBlock = createDataBlock();
if (pResBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -876,7 +875,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
if (p1->name != NULL) {
STR_TO_VARSTR(str, p1->name);
} else { // name is not retrieved during filter
- metaGetTableNameByUid(metaHandle, p1->uid, str);
+ pStorageAPI->metaFn.getTableNameByUid(pVnode, p1->uid, str);
}
colDataSetVal(pColInfo, i, str, false);
@@ -889,7 +888,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
if (p1->pTagVal == NULL) {
colDataSetNULL(pColInfo, i);
} else {
- const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal);
+ const char* p = pStorageAPI->metaFn.extractTagVal(p1->pTagVal, pColInfo->info.type, &tagVal);
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
colDataSetNULL(pColInfo, i);
@@ -949,13 +948,13 @@ static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) {
}
}
-static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle,
- SIdxFltStatus status) {
+static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* pVnode,
+ SIdxFltStatus status, SStorageAPI* pAPI) {
if (pTagCond == NULL) {
return TSDB_CODE_SUCCESS;
}
- terrno = TDB_CODE_SUCCESS;
+ terrno = TSDB_CODE_SUCCESS;
int32_t code = TSDB_CODE_SUCCESS;
SArray* pBlockList = NULL;
@@ -985,7 +984,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
FilterCondType condType = checkTagCond(pTagCond);
- int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond);
+ int32_t filter = optimizeTbnameInCond(pVnode, pListInfo->idInfo.suid, pUidTagList, pTagCond, pAPI);
if (filter == 0) { // tbname in filter is activated, do nothing and return
taosArrayClear(pUidList);
@@ -998,9 +997,9 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
terrno = 0;
} else {
if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) {
- code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList);
+ code = pAPI->metaFn.getTableTagsByUid(pVnode, pListInfo->idInfo.suid, pUidTagList);
} else {
- code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList);
+ code = pAPI->metaFn.getTableTags(pVnode, pListInfo->idInfo.suid, pUidTagList);
}
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid);
@@ -1014,7 +1013,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
goto end;
}
- pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle);
+ pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI);
if (pResBlock == NULL) {
code = terrno;
goto end;
@@ -1052,8 +1051,8 @@ end:
return code;
}
-int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
- STableListInfo* pListInfo, uint8_t* digest, const char* idstr) {
+int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
+ STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI) {
int32_t code = TSDB_CODE_SUCCESS;
size_t numOfTables = 0;
@@ -1065,10 +1064,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
SIdxFltStatus status = SFLT_NOT_INDEX;
if (pScanNode->tableType != TSDB_SUPER_TABLE) {
pListInfo->idInfo.uid = pScanNode->uid;
- if (metaIsTableExist(metaHandle, pScanNode->uid)) {
+ if (pStorageAPI->metaFn.isTableExisted(pVnode, pScanNode->uid)) {
taosArrayPush(pUidList, &pScanNode->uid);
}
- code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status);
+ code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
@@ -1080,7 +1079,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
genTagFilterDigest(pTagCond, &context);
bool acquired = false;
- metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList,
+ pStorageAPI->metaFn.getCachedTableList(pVnode, pScanNode->suid, context.digest, tListLen(context.digest), pUidList,
&acquired);
if (acquired) {
digest[0] = 1;
@@ -1091,26 +1090,27 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
}
if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table
- vnodeGetCtbIdList(pVnode, pScanNode->suid, pUidList);
+ pStorageAPI->metaFn.getChildTableList(pVnode, pScanNode->suid, pUidList);
} else {
// failed to find the result in the cache, let try to calculate the results
if (pTagIndexCond) {
- void* pIndex = tsdbGetIvtIdx(metaHandle);
+ void* pIndex = pStorageAPI->metaFn.getInvertIndex(pVnode);
+
SIndexMetaArg metaArg = {
- .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid};
+ .metaEx = pVnode, .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), .ivtIdx = pIndex, .suid = pScanNode->uid};
status = SFLT_NOT_INDEX;
- code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status);
+ code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status, &pStorageAPI->metaFilter);
if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake
qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid);
- code = TDB_CODE_SUCCESS;
+ code = TSDB_CODE_SUCCESS;
} else {
qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList));
}
}
}
- code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status);
+ code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
@@ -1127,7 +1127,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
memcpy(pPayload + sizeof(int32_t), taosArrayGet(pUidList, 0), numOfTables * sizeof(uint64_t));
}
- metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1);
+// metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1);
digest[0] = 1;
memcpy(digest + 1, context.digest, tListLen(context.digest));
}
@@ -1164,11 +1164,13 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) {
return keyLen;
}
-int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) {
+int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId,
+ SStorageAPI* pAPI) {
SMetaReader mr = {0};
- metaReaderInit(&mr, pMeta, 0);
- if (metaGetTableEntryByUidCache(&mr, uid) != 0) { // table not exist
- metaReaderClear(&mr);
+
+ pAPI->metaReaderFn.initReader(&mr, pVnode, 0, &pAPI->metaFn);
+ if (pAPI->metaReaderFn.getEntryGetUidCache(&mr, uid) != 0) { // table not exist
+ pAPI->metaReaderFn.clearReader(&mr);
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
}
@@ -1187,7 +1189,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode,
REPLACE_NODE(pNew);
} else {
nodesDestroyList(groupNew);
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
return code;
}
@@ -1204,7 +1206,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode,
if (tTagIsJson(data)) {
terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
nodesDestroyList(groupNew);
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
return terrno;
}
int32_t len = getJsonValueLen(data);
@@ -1224,7 +1226,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode,
*pGroupId = calcGroupId(keyBuf, len);
nodesDestroyList(groupNew);
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
return TSDB_CODE_SUCCESS;
}
@@ -1514,7 +1516,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu
return TSDB_CODE_OUT_OF_MEMORY;
}
- SHashObj *pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+ SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
for (int32_t i = 0; i < numOfOutput; ++i) {
const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) {
@@ -1542,7 +1544,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu
return TSDB_CODE_SUCCESS;
}
-SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset) {
+SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore) {
SqlFunctionCtx* pFuncCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));
if (pFuncCtx == NULL) {
return NULL;
@@ -1605,6 +1607,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
pCtx->numOfParams = pExpr->base.numOfParams;
pCtx->param = pFunct->pParam;
pCtx->saveHandle.currentPage = -1;
+ pCtx->pStore = pStore;
}
for (int32_t i = 1; i < numOfOutput; ++i) {
@@ -1841,9 +1844,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) {
return taosArrayGetSize(pTableList->pTableList);
}
-uint64_t tableListGetSuid(const STableListInfo* pTableList) {
- return pTableList->idInfo.suid;
-}
+uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->idInfo.suid; }
STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) {
if (taosArrayGetSize(pTableList->pTableList) == 0) {
@@ -2030,11 +2031,11 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups);
taosArrayDestroy(pList);
- return TDB_CODE_SUCCESS;
+ return TSDB_CODE_SUCCESS;
}
int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group,
- bool groupSort, uint8_t *digest) {
+ bool groupSort, uint8_t *digest, SStorageAPI* pAPI) {
int32_t code = TSDB_CODE_SUCCESS;
bool groupByTbname = groupbyTbname(group);
@@ -2054,7 +2055,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
pTableListInfo->numOfOuputGroups = 1;
}
} else {
- code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest);
+ code = getColInfoResultForGroupby(pHandle->vnode, group, pTableListInfo, digest, pAPI);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@@ -2086,7 +2087,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
}
uint8_t digest[17] = {0};
- int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr);
+ int32_t code = getTableList(pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr, &pTaskInfo->storageAPI);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to getTableList, code: %s", tstrerror(code));
return code;
@@ -2104,7 +2105,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
return TSDB_CODE_SUCCESS;
}
- code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest);
+ code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest, &pTaskInfo->storageAPI);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index b6d4615997..fb35b211c9 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -14,6 +14,8 @@
*/
#include "executor.h"
+#include
+#include
#include "executorInt.h"
#include "operator.h"
#include "planner.h"
@@ -21,7 +23,8 @@
#include "tdatablock.h"
#include "tref.h"
#include "tudf.h"
-#include "vnode.h"
+
+#include "storageapi.h"
static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT;
int32_t exchangeObjRefPool = -1;
@@ -156,18 +159,18 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
}
}
-void doSetTaskId(SOperatorInfo* pOperator) {
+void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamScanInfo* pStreamScanInfo = pOperator->info;
if (pStreamScanInfo->pTableScanOp != NULL) {
STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info;
if (pScanInfo->base.dataReader != NULL) {
- tsdbReaderSetId(pScanInfo->base.dataReader, pTaskInfo->id.str);
+ pAPI->tsdReader.tsdSetReaderTaskId(pScanInfo->base.dataReader, pTaskInfo->id.str);
}
}
} else {
- doSetTaskId(pOperator->pDownstream[0]);
+ doSetTaskId(pOperator->pDownstream[0], pAPI);
}
}
@@ -177,7 +180,7 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) {
buildTaskId(taskId, queryId, pTaskInfo->id.str);
// set the idstr for tsdbReader
- doSetTaskId(pTaskInfo->pRoot);
+ doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI);
}
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
@@ -249,7 +252,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks,
qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols,
uint64_t id) {
if (msg == NULL) { // create raw scan
- SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE);
+ SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, &pReaderHandle->api);
if (NULL == pTaskInfo) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
@@ -261,6 +264,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3
return NULL;
}
+ pTaskInfo->storageAPI = pReaderHandle->api;
qDebug("create raw scan task info completed, vgId:%d, %s", vgId, GET_TASKID(pTaskInfo));
return pTaskInfo;
}
@@ -320,7 +324,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v
return pTaskInfo;
}
-static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) {
+static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr,
+ SStorageAPI* pAPI) {
SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
int32_t numOfUids = taosArrayGetSize(tableIdList);
if (numOfUids == 0) {
@@ -336,11 +341,11 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
// let's discard the tables those are not created according to the queried super table.
SMetaReader mr = {0};
- metaReaderInit(&mr, pScanInfo->readHandle.meta, 0);
+ pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, 0, &pAPI->metaFn);
for (int32_t i = 0; i < numOfUids; ++i) {
uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i);
- int32_t code = metaGetTableEntryByUid(&mr, *id);
+ int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, *id);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr);
continue;
@@ -368,7 +373,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
if (pScanInfo->pTagCond != NULL) {
bool qualified = false;
STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid};
- code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified);
+ code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.vnode, &qualified, pAPI);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr);
continue;
@@ -383,7 +388,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
taosArrayPush(qa, id);
}
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
return qa;
}
@@ -401,10 +406,10 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
SStreamScanInfo* pScanInfo = pInfo->info;
if (isAdd) { // add new table id
- SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id);
+ SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id, &pTaskInfo->storageAPI);
int32_t numOfQualifiedTables = taosArrayGetSize(qa);
qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id);
- code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
+ code = pTaskInfo->storageAPI.tqReaderFn.tqReaderAddTables(pScanInfo->tqReader, qa);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(qa);
return code;
@@ -433,8 +438,8 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
if (assignUid) {
keyInfo.groupId = keyInfo.uid;
} else {
- code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf,
- &keyInfo.groupId);
+ code = getGroupIdFromTagsVal(pScanInfo->readHandle.vnode, keyInfo.uid, pScanInfo->pGroupTags, keyBuf,
+ &keyInfo.groupId, &pTaskInfo->storageAPI);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(keyBuf);
taosArrayDestroy(qa);
@@ -456,7 +461,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
} else { // remove the table id in current list
qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id);
taosWLockLatch(&pTaskInfo->lock);
- code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList);
+ code = pTaskInfo->storageAPI.tqReaderFn.tqReaderRemoveTables(pScanInfo->tqReader, tableIdList);
taosWUnLockLatch(&pTaskInfo->lock);
}
@@ -502,7 +507,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
}
SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50};
- code = dsDataSinkMgtInit(&cfg);
+ code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to dsDataSinkMgtInit, code:%s, %s", tstrerror(code), (*pTask)->id.str);
goto _error;
@@ -1060,6 +1065,8 @@ void qStreamSetOpen(qTaskInfo_t tinfo) {
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SOperatorInfo* pOperator = pTaskInfo->pRoot;
const char* id = GET_TASKID(pTaskInfo);
@@ -1081,12 +1088,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
if (pOffset->type == TMQ_OFFSET__LOG) {
// todo refactor: move away
- tsdbReaderClose(pScanBaseInfo->dataReader);
+ pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pScanBaseInfo->dataReader);
pScanBaseInfo->dataReader = NULL;
- walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset);
- if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) {
- qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
+ SStoreTqReader* pReaderAPI = &pTaskInfo->storageAPI.tqReaderFn;
+ SWalReader* pWalReader = pReaderAPI->tqReaderGetWalReader(pInfo->tqReader);
+ walReaderVerifyOffset(pWalReader, pOffset);
+ if (pReaderAPI->tqReaderSeek(pInfo->tqReader, pOffset->version + 1, id) < 0) {
+ qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
return -1;
}
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
@@ -1141,8 +1150,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
pScanInfo->scanTimes = 0;
if (pScanBaseInfo->dataReader == NULL) {
- int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1,
- pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false, NULL);
+ int32_t code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1,
+ pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL);
if (code != TSDB_CODE_SUCCESS) {
qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id);
terrno = code;
@@ -1152,8 +1161,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s",
uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
} else {
- tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
- tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
+ pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
+ pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s",
uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
}
@@ -1175,14 +1184,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id);
STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo;
- if (setForSnapShot(sContext, pOffset->uid) != 0) {
+ if (pAPI->snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) {
qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1;
}
- SMetaTableInfo mtInfo = getUidfromSnapShot(sContext);
- tsdbReaderClose(pInfo->dataReader);
+ SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.getMetaTableInfoFromSnapshot(sContext);
+ pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader);
pInfo->dataReader = NULL;
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
@@ -1200,7 +1209,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
int32_t size = tableListGetSize(pTableListInfo);
- tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL,
+ pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL,
false, NULL);
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
@@ -1212,7 +1221,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
SStreamRawScanInfo* pInfo = pOperator->info;
SSnapContext* sContext = pInfo->sContext;
- if (setForSnapShot(sContext, pOffset->uid) != 0) {
+ if (pTaskInfo->storageAPI.snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) {
qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1;
@@ -1221,7 +1230,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
id);
} else if (pOffset->type == TMQ_OFFSET__LOG) {
SStreamRawScanInfo* pInfo = pOperator->info;
- tsdbReaderClose(pInfo->dataReader);
+ pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pInfo->dataReader);
pInfo->dataReader = NULL;
qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id);
}
diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c
index ad7258ae1f..521b9284a8 100644
--- a/source/libs/executor/src/executorInt.c
+++ b/source/libs/executor/src/executorInt.c
@@ -33,7 +33,7 @@
#include "tcompare.h"
#include "thash.h"
#include "ttypes.h"
-#include "vnode.h"
+#include "storageapi.h"
#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN)
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
@@ -844,6 +844,8 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
SDiskbasedBuf* pBuf) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SSDataBlock* pBlock = pbInfo->pRes;
// set output datablock version
@@ -860,12 +862,13 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr
doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false);
void* tbname = NULL;
- if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
+ if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
}
- streamFreeVal(tbname);
+
+ pAPI->stateStore.streamStateFreeVal(tbname);
}
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
@@ -976,11 +979,11 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
return NULL;
}
-int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr) {
+int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr, SFunctionStateStore* pStore) {
pSup->pExprInfo = pExprInfo;
pSup->numOfExprs = numOfExpr;
if (pSup->pExprInfo != NULL) {
- pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset);
+ pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset, pStore);
if (pSup->pCtx == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@@ -1066,122 +1069,22 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo*
return TSDB_CODE_SUCCESS;
}
-int32_t resultRowEncode(void* k, int32_t* size, char* buf) {
- // SResultRow* key = k;
- // int len = 0;
- // int struLen = *size;
- // len += taosEncodeFixedI32((void**)&buf, key->pageId);
-
- // uint32_t offset = key->offset;
- // len += taosEncodeFixedU32((void**)&buf, offset);
-
- // len += taosEncodeFixedI8((void**)&buf, key->startInterp);
- // len += taosEncodeFixedI8((void**)&buf, key->endInterp);
- // len += taosEncodeFixedI8((void**)&buf, key->closed);
- // len += taosEncodeFixedU32((void**)&buf, key->numOfRows);
-
- // len += taosEncodeFixedI64((void**)&buf, key->win.skey);
- // len += taosEncodeFixedI64((void**)&buf, key->win.ekey);
-
- // int32_t numOfEntryInfo = (struLen - sizeof(SResultRow)) / sizeof(struct SResultRowEntryInfo);
- // len += taosEncodeFixedI32((void**)&buf, numOfEntryInfo);
- // for (int i = 0; i < numOfEntryInfo; i++) {
- // SResultRowEntryInfo* p = &key->pEntryInfo[i];
-
- // uint8_t value = p->initialized ? 1 : 0;
- // len += taosEncodeFixedU8((void**)&buf, value);
-
- // value = p->complete ? 1 : 0;
- // len += taosEncodeFixedU8((void**)&buf, value);
-
- // value = p->isNullRes;
- // len += taosEncodeFixedU8((void**)&buf, value);
-
- // len += taosEncodeFixedU16((void**)&buf, p->numOfRes);
- // }
- // {
- // char* strBuf = taosMemoryCalloc(1, *size * 100);
- // resultRowToString(key, *size, strBuf);
- // qWarn("encode result row:%s", strBuf);
- // }
-
- // return len;
- return 0;
-}
-
-int32_t resultRowDecode(void** k, size_t size, char* buf) {
- // char* p1 = buf;
- // int32_t numOfEntryInfo = 0;
- // uint32_t entryOffset = sizeof(int32_t) + sizeof(uint32_t) + sizeof(int8_t) + sizeof(int8_t) + sizeof(int8_t) +
- // sizeof(uint32_t) + sizeof(int64_t) + sizeof(int64_t);
- // taosDecodeFixedI32(p1 + entryOffset, &numOfEntryInfo);
-
- // char* p = buf;
- // size = sizeof(SResultRow) + numOfEntryInfo * sizeof(SResultRowEntryInfo);
- // SResultRow* key = taosMemoryCalloc(1, size);
-
- // p = taosDecodeFixedI32(p, (int32_t*)&key->pageId);
- // uint32_t offset = 0;
- // p = taosDecodeFixedU32(p, &offset);
- // key->offset = offset;
-
- // p = taosDecodeFixedI8(p, (int8_t*)(&key->startInterp));
- // p = taosDecodeFixedI8(p, (int8_t*)(&key->endInterp));
- // p = taosDecodeFixedI8(p, (int8_t*)&key->closed);
- // p = taosDecodeFixedU32(p, &key->numOfRows);
-
- // p = taosDecodeFixedI64(p, &key->win.skey);
- // p = taosDecodeFixedI64(p, &key->win.ekey);
- // p = taosDecodeFixedI32(p, &numOfEntryInfo);
- // for (int i = 0; i < numOfEntryInfo; i++) {
- // SResultRowEntryInfo* pInfo = &key->pEntryInfo[i];
- // uint8_t value = 0;
- // p = taosDecodeFixedU8(p, &value);
- // pInfo->initialized = (value == 1) ? true : false;
-
- // p = taosDecodeFixedU8(p, &value);
- // pInfo->complete = (value == 1) ? true : false;
-
- // p = taosDecodeFixedU8(p, &value);
- // pInfo->isNullRes = value;
-
- // p = taosDecodeFixedU16(p, &pInfo->numOfRes);
- // }
- // *k = key;
-
- // {
- // char* strBuf = taosMemoryCalloc(1, size * 100);
- // resultRowToString(key, size, strBuf);
- // qWarn("decode result row:%s", strBuf);
- // }
- // return size;
- return 0;
-}
-
-int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize) {
- // char* buf = taosMemoryCalloc(1, resSize * 10);
- // int len = resultRowEncode((void*)pResult, &resSize, buf);
- // char* buf = taosMemoryCalloc(1, resSize);
- // memcpy(buf, pResult, resSize);
- streamStatePut(pState, pKey, (char*)pResult, resSize);
- // taosMemoryFree(buf);
+int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI) {
+ pAPI->streamStateReleaseBuf(pState, pKey, pResult);
return TSDB_CODE_SUCCESS;
}
-int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult) {
- streamStateReleaseBuf(pState, pKey, pResult);
+int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI) {
+ pAPI->streamStateSessionPut(pState, key, (const void*)buf, size);
+ releaseOutputBuf(pState, NULL, (SResultRow*)buf, pAPI);
return TSDB_CODE_SUCCESS;
}
-int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size) {
- streamStateSessionPut(pState, key, (const void*)buf, size);
- releaseOutputBuf(pState, NULL, (SResultRow*)buf);
- return TSDB_CODE_SUCCESS;
-}
-
-int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock,
+int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
SExprSupp* pSup, SGroupResInfo* pGroupResInfo) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SExprInfo* pExprInfo = pSup->pExprInfo;
int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
@@ -1193,7 +1096,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
SSessionKey* pKey = taosArrayGet(pGroupResInfo->pRows, i);
int32_t size = 0;
void* pVal = NULL;
- int32_t code = streamStateSessionGet(pState, pKey, &pVal, &size);
+ int32_t code = pAPI->stateStore.streamStateSessionGet(pState, pKey, &pVal, &size);
ASSERT(code == 0);
if (code == -1) {
// coverity scan
@@ -1205,7 +1108,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
// no results, continue to check the next one
if (pRow->numOfRows == 0) {
pGroupResInfo->index += 1;
- releaseOutputBuf(pState, NULL, pRow);
+ releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
continue;
}
@@ -1213,23 +1116,23 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
pBlock->info.id.groupId = pKey->groupId;
void* tbname = NULL;
- if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
+ if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
}
- streamFreeVal(tbname);
+ pAPI->stateStore.streamStateFreeVal(tbname);
} else {
// current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.id.groupId != pKey->groupId) {
- releaseOutputBuf(pState, NULL, pRow);
+ releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
break;
}
}
if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
ASSERT(pBlock->info.rows > 0);
- releaseOutputBuf(pState, NULL, pRow);
+ releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
break;
}
@@ -1260,7 +1163,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
pBlock->info.dataLoad = 1;
pBlock->info.rows += pRow->numOfRows;
- releaseOutputBuf(pState, NULL, pRow);
+ releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
}
blockDataUpdateTsWindow(pBlock, 0);
return TSDB_CODE_SUCCESS;
@@ -1292,7 +1195,7 @@ void qStreamCloseTsdbReader(void* task) {
qDebug("wait for the reader stopping");
}
- tsdbReaderClose(pTSInfo->base.dataReader);
+ pTaskInfo->storageAPI.tsdReader.tsdReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL;
// restore the status, todo refactor.
diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c
index 5101e36992..90b7f4a77e 100644
--- a/source/libs/executor/src/filloperator.c
+++ b/source/libs/executor/src/filloperator.c
@@ -338,7 +338,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
goto _error;
}
- code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs);
+ code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -355,7 +355,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
- code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr);
+ code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -443,7 +443,7 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols);
tSimpleHashCleanup(pFillSup->pResMap);
pFillSup->pResMap = NULL;
- releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal);
+ releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pFillSup->pAPI->stateStore); //?????
pFillSup->cur.pRowVal = NULL;
cleanupExprSupp(&pFillSup->notFillExprSup);
@@ -490,74 +490,78 @@ static void resetFillWindow(SResultRowData* pRowData) {
pRowData->pRowVal = NULL;
}
-void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) {
+void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) {
resetFillWindow(&pFillSup->prev);
- releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal);
+ releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pAPI->stateStore); //???
resetFillWindow(&pFillSup->cur);
resetFillWindow(&pFillSup->next);
resetFillWindow(&pFillSup->nextNext);
}
void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
- SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState;
- resetPrevAndNextWindow(pFillSup, pState);
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
+ void* pState = pOperator->pTaskInfo->streamInfo.pState;
+ resetPrevAndNextWindow(pFillSup, pState, pAPI);
SWinKey key = {.ts = ts, .groupId = groupId};
int32_t curVLen = 0;
- int32_t code = streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen);
+
+ int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen);
ASSERT(code == TSDB_CODE_SUCCESS);
pFillSup->cur.key = key.ts;
}
void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
- SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState;
- resetPrevAndNextWindow(pFillSup, pState);
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+ void* pState = pOperator->pTaskInfo->streamInfo.pState;
+ resetPrevAndNextWindow(pFillSup, pState, pAPI);
SWinKey key = {.ts = ts, .groupId = groupId};
void* curVal = NULL;
int32_t curVLen = 0;
- int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen);
+ int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen);
ASSERT(code == TSDB_CODE_SUCCESS);
pFillSup->cur.key = key.ts;
pFillSup->cur.pRowVal = curVal;
- SStreamStateCur* pCur = streamStateFillSeekKeyPrev(pState, &key);
+ SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key);
SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId};
void* preVal = NULL;
int32_t preVLen = 0;
- code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen);
+ code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen);
if (code == TSDB_CODE_SUCCESS) {
pFillSup->prev.key = preKey.ts;
pFillSup->prev.pRowVal = preVal;
- code = streamStateCurNext(pState, pCur);
+ code = pAPI->stateStore.streamStateCurNext(pState, pCur);
ASSERT(code == TSDB_CODE_SUCCESS);
- code = streamStateCurNext(pState, pCur);
+ code = pAPI->stateStore.streamStateCurNext(pState, pCur);
if (code != TSDB_CODE_SUCCESS) {
- streamStateFreeCur(pCur);
+ pAPI->stateStore.streamStateFreeCur(pCur);
pCur = NULL;
}
} else {
- streamStateFreeCur(pCur);
- pCur = streamStateFillSeekKeyNext(pState, &key);
+ pAPI->stateStore.streamStateFreeCur(pCur);
+ pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key);
}
SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId};
void* nextVal = NULL;
int32_t nextVLen = 0;
- code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen);
+ code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen);
if (code == TSDB_CODE_SUCCESS) {
pFillSup->next.key = nextKey.ts;
pFillSup->next.pRowVal = nextVal;
if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) {
- code = streamStateCurNext(pState, pCur);
+ code = pAPI->stateStore.streamStateCurNext(pState, pCur);
if (code == TSDB_CODE_SUCCESS) {
SWinKey nextNextKey = {.groupId = groupId};
void* nextNextVal = NULL;
int32_t nextNextVLen = 0;
- code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen);
+ code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen);
if (code == TSDB_CODE_SUCCESS) {
pFillSup->nextNext.key = nextNextKey.ts;
pFillSup->nextNext.pRowVal = nextNextVal;
@@ -565,7 +569,7 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId,
}
}
}
- streamStateFreeCur(pCur);
+ pAPI->stateStore.streamStateFreeCur(pCur);
}
static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; }
@@ -922,8 +926,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
}
static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) {
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
SWinKey key = {.groupId = groupId, .ts = pRow->key};
- int32_t code = streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len);
+ int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len);
qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code);
ASSERT(code == TSDB_CODE_SUCCESS);
}
@@ -1021,7 +1027,8 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) {
}
static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) {
- SStreamState* pState = pOp->pTaskInfo->streamInfo.pState;
+ SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;
+ void* pState = pOp->pTaskInfo->streamInfo.pState;
SSDataBlock* pBlock = delRes;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
@@ -1041,14 +1048,14 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_
SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
void* tbname = NULL;
- streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname);
+ pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname);
if (tbname == NULL) {
colDataSetNULL(pTableCol, pBlock->info.rows);
} else {
char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
- streamFreeVal(tbname);
+ pAPI->stateStore.streamStateFreeVal(tbname);
}
pBlock->info.rows++;
@@ -1070,12 +1077,13 @@ static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY end
}
static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) {
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamFillOperatorInfo* pInfo = pOperator->info;
getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup);
setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo);
SWinKey key = {.ts = startTs, .groupId = groupId};
if (!pInfo->pFillInfo->needFill) {
- streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
+ pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
} else {
STimeRange tw = {
@@ -1093,6 +1101,8 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE
}
static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
SStreamFillOperatorInfo* pInfo = pOperator->info;
SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
int32_t size = taosArrayGetSize(pFillInfo->delRanges);
@@ -1109,17 +1119,16 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
pInfo->pRes->info.id.groupId = range->groupId;
}
SWinKey key = {.ts = range->skey, .groupId = range->groupId};
- streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
+ pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
}
}
static void doDeleteFillResult(SOperatorInfo* pOperator) {
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
SStreamFillOperatorInfo* pInfo = pOperator->info;
- SStreamFillSupporter* pFillSup = pInfo->pFillSup;
SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
SSDataBlock* pBlock = pInfo->pSrcDelBlock;
- SSDataBlock* pRes = pInfo->pRes;
- SSDataBlock* pDelRes = pInfo->pDelRes;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* tsStarts = (TSKEY*)pStartCol->pData;
@@ -1130,7 +1139,8 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
TSKEY endTs = ts;
uint64_t groupId = groupIds[pInfo->srcDelRowIndex];
SWinKey key = {.ts = ts, .groupId = groupId};
- SStreamStateCur* pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key);
+ SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key);
+
if (!pCur) {
pInfo->srcDelRowIndex++;
continue;
@@ -1152,15 +1162,15 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
SWinKey delKey = {.groupId = delGroupId, .ts = delTs};
if (delTs == nextKey.ts) {
- code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
+ code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
if (code == TSDB_CODE_SUCCESS) {
- code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen);
+ code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen);
}
// ts will be deleted later
if (delTs != ts) {
- streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey);
- streamStateFreeCur(pCur);
- pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
+ pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey);
+ pAPI->stateStore.streamStateFreeCur(pCur);
+ pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
}
endTs = TMAX(delTs, nextKey.ts - 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -1169,9 +1179,11 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
}
pInfo->srcDelRowIndex++;
}
- streamStateFreeCur(pCur);
+
+ pAPI->stateStore.streamStateFreeCur(pCur);
doDeleteFillResultImpl(pOperator, ts, endTs, groupId);
}
+
pFillInfo->current = pFillInfo->end + 1;
}
@@ -1322,7 +1334,7 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) {
}
static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval,
- SExprInfo* pFillExprInfo, int32_t numOfFillCols) {
+ SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) {
SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter));
if (!pFillSup) {
return NULL;
@@ -1335,6 +1347,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
pFillSup->type = convertFillType(pPhyFillNode->mode);
pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols;
pFillSup->interval = *pInterval;
+ pFillSup->pAPI = pAPI;
int32_t code = initResultBuf(pFillSup);
if (code != TSDB_CODE_SUCCESS) {
@@ -1343,7 +1356,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
}
SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
- code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols);
+ code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore);
if (code != TSDB_CODE_SUCCESS) {
destroyStreamFillSupporter(pFillSup);
return NULL;
@@ -1414,7 +1427,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval;
int32_t numOfFillCols = 0;
SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols);
- pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols);
+ pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI);
if (!pInfo->pFillSup) {
goto _error;
}
@@ -1479,7 +1492,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
goto _error;
}
- code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
+ code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index 936030fa57..7aac639027 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -451,7 +451,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
}
pInfo->pGroupCols = extractColumnInfo(pAggNode->pGroupKeys);
- code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
+ code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -467,7 +467,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
+ pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -849,7 +849,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
if (pPartNode->pExprs != NULL) {
int32_t num = 0;
SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num);
- int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num);
+ int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
pTaskInfo->code = terrno;
@@ -951,6 +951,8 @@ static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo
static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; }
static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
SStreamPartitionOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pDest = pInfo->binfo.pRes;
ASSERT(hasRemainPartion(pInfo));
@@ -973,9 +975,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
pDest->info.parTbName[0] = 0;
if (pInfo->tbnameCalSup.numOfExprs > 0) {
void* tbname = NULL;
- if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) {
+ if (pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) {
memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
- streamFreeVal(tbname);
+ pAPI->stateStore.streamStateFreeVal(tbname);
}
}
taosArrayDestroy(pParInfo->rowIds);
@@ -991,10 +993,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
return pDest;
}
-void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
- SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) {
+void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
+ SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) {
void* pValue = NULL;
- if (streamStateGetParName(pState, groupId, &pValue) != 0) {
+ if (pAPI->streamStateGetParName(pState, groupId, &pValue) != 0) {
SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId);
memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN);
pTmpBlock->info.id.groupId = groupId;
@@ -1011,7 +1013,7 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1);
len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1);
memcpy(tbName, varDataVal(pData), len);
- streamStatePutParName(pState, groupId, tbName);
+ pAPI->streamStatePutParName(pState, groupId, tbName);
}
memcpy(pTmpBlock->info.parTbName, tbName, len);
pDestBlock->info.rows--;
@@ -1035,10 +1037,12 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
} else {
memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN);
}
- streamStateReleaseBuf(pState, NULL, pValue);
+ pAPI->streamStateReleaseBuf(pState, NULL, pValue);
}
static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) {
+ SExecTaskInfo* pTask = pOperator->pTaskInfo;
+
SStreamPartitionOperatorInfo* pInfo = pOperator->info;
if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) ||
taosHashGetSize(pInfo->pPartitions) == 0) {
@@ -1051,8 +1055,8 @@ static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) {
if (pInfo->pTbNameIte != NULL) {
SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte;
int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0);
- appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup,
- pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes);
+ appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup,
+ pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore);
pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte);
}
return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL;
@@ -1165,14 +1169,17 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
}
void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) {
+ SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI;
+
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
return;
}
+
SStreamScanInfo* pScanInfo = downstream->info;
pScanInfo->partitionSup = *pParSup;
pScanInfo->pPartScalarSup = pExpr;
if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) {
- pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0);
+ pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0);
}
}
@@ -1236,7 +1243,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
if (pPartNode->part.pExprs != NULL) {
int32_t num = 0;
SExprInfo* pCalExprInfo = createExprInfo(pPartNode->part.pExprs, NULL, &num);
- code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num);
+ code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -1251,7 +1258,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
}
pInfo->tbnameCalSup.pExprInfo = pSubTableExpr;
createExprFromOneNode(pSubTableExpr, pPartNode->pSubtable, 0);
- code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1);
+ code = initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -1265,7 +1272,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
- if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) {
+ if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c
index 729178dc60..730252c7ee 100644
--- a/source/libs/executor/src/operator.c
+++ b/source/libs/executor/src/operator.c
@@ -25,7 +25,8 @@
#include "operator.h"
#include "query.h"
#include "querytask.h"
-#include "vnode.h"
+
+#include "storageapi.h"
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn,
@@ -233,11 +234,12 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
}
static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) {
+ SStorageAPI* pAPI = pParam->pParam;
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pInfo = pOperator->info;
if (pInfo->base.dataReader != NULL) {
- tsdbReaderSetCloseFlag(pInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderNotifyClosing(pInfo->base.dataReader);
}
return OPTR_FN_RET_ABORT;
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
@@ -246,7 +248,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam,
if (pInfo->pTableScanOp != NULL) {
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) {
- tsdbReaderSetCloseFlag(pTableScanInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderNotifyClosing(pTableScanInfo->base.dataReader);
}
}
@@ -256,8 +258,8 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam,
return OPTR_FN_RET_CONTINUE;
}
-int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr) {
- STraverParam p = {0};
+int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI) {
+ STraverParam p = {.pParam = pAPI};
traverseOperatorTree(pOperator, doStopDataReader, &p, pIdStr);
return p.code;
}
@@ -378,17 +380,18 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
STableListInfo* pTableListInfo = tableListCreate();
if (pBlockNode->tableType == TSDB_SUPER_TABLE) {
- SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo));
- int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pList);
+ SArray* pList = taosArrayInit(4, sizeof(uint64_t));
+ int32_t code = pTaskInfo->storageAPI.metaFn.getChildTableList(pHandle->vnode, pBlockNode->uid, pList);
if (code != TSDB_CODE_SUCCESS) {
- pTaskInfo->code = terrno;
+ pTaskInfo->code = code;
+ taosArrayDestroy(pList);
return NULL;
}
size_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) {
- STableKeyInfo* p = taosArrayGet(pList, i);
- tableListAddTableInfo(pTableListInfo, p->uid, 0);
+ uint64_t* id = taosArrayGet(pList, i);
+ tableListAddTableInfo(pTableListInfo, *id, 0);
}
taosArrayDestroy(pList);
diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c
index 02f504bef0..dde6f7c0e8 100644
--- a/source/libs/executor/src/projectoperator.c
+++ b/source/libs/executor/src/projectoperator.c
@@ -18,6 +18,7 @@
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
+#include "tdatablock.h"
typedef struct SProjectOperatorInfo {
SOptrBasicInfo binfo;
@@ -114,7 +115,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
initResultSizeInfo(&pOperator->resultInfo, numOfRows);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
+ pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -369,7 +370,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
if (pPhyNode->pExprs != NULL) {
int32_t num = 0;
SExprInfo* pSExpr = createExprInfo(pPhyNode->pExprs, NULL, &num);
- int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num);
+ int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -391,7 +392,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
blockDataEnsureCapacity(pResBlock, numOfRows);
int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
+ pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c
index 7716b5976b..22d171e74a 100644
--- a/source/libs/executor/src/querytask.c
+++ b/source/libs/executor/src/querytask.c
@@ -29,13 +29,13 @@
#include "operator.h"
#include "query.h"
#include "querytask.h"
+#include "storageapi.h"
#include "thash.h"
#include "ttypes.h"
-#include "vnode.h"
#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st)))
-SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model) {
+SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, SStorageAPI* pAPI) {
SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
if (pTaskInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -48,6 +48,7 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP
pTaskInfo->execModel = model;
pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo));
pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES);
+ pTaskInfo->storageAPI = *pAPI;
taosInitRWLatch(&pTaskInfo->lock);
@@ -55,7 +56,6 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP
pTaskInfo->id.queryId = queryId;
pTaskInfo->id.str = taosMemoryMalloc(64);
buildTaskId(taskId, queryId, pTaskInfo->id.str);
-
return pTaskInfo;
}
@@ -63,7 +63,7 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); }
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) {
pTaskInfo->code = rspCode;
- stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str);
+ stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI);
}
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
@@ -78,7 +78,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) {
- *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model);
+ *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model, &pHandle->api);
if (*pTaskInfo == NULL) {
taosMemoryFree(sql);
return terrno;
@@ -120,13 +120,15 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
return terrno;
}
- metaReaderInit(&mr, pHandle->meta, 0);
- int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid);
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
+ pAPI->metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pAPI->metaFn);
+ int32_t code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, pScanNode->uid);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid,
GET_TASKID(pTaskInfo));
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
return terrno;
}
@@ -142,9 +144,9 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
tDecoderClear(&mr.coder);
tb_uid_t suid = mr.me.ctbEntry.suid;
- code = metaGetTableEntryByUidCache(&mr, suid);
+ code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, suid);
if (code != TSDB_CODE_SUCCESS) {
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
return terrno;
}
@@ -154,7 +156,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow);
}
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode);
return TSDB_CODE_SUCCESS;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 0537702a75..3984e9a792 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -33,6 +33,9 @@
#include "operator.h"
#include "querytask.h"
+#include "storageapi.h"
+#include "wal.h"
+
int32_t scanDebug = 0;
#define MULTI_READER_MAX_TABLE_NUM 5000
@@ -261,8 +264,10 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA
}
static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
bool allColumnsHaveAgg = true;
- int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg);
+ int32_t code = pAPI->tsdReader.tsdReaderRetrieveBlockSMAInfo(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
@@ -323,6 +328,8 @@ bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
pCost->totalBlocks += 1;
@@ -343,7 +350,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->filterOutBlocks += 1;
pCost->totalRows += pBlock->info.rows;
- tsdbReleaseDataBlock(pTableScanInfo->dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64,
@@ -351,7 +358,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pBlockInfo->id.uid);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
pCost->skipBlocks += 1;
- tsdbReleaseDataBlock(pTableScanInfo->dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) {
pCost->loadBlockStatis += 1;
@@ -361,7 +368,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
- tsdbReleaseDataBlock(pTableScanInfo->dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
return TSDB_CODE_SUCCESS;
} else {
qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo));
@@ -383,7 +390,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost->filterOutBlocks += 1;
(*status) = FUNC_DATA_REQUIRED_FILTEROUT;
- tsdbReleaseDataBlock(pTableScanInfo->dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
return TSDB_CODE_SUCCESS;
}
}
@@ -398,10 +405,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64,
GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->skipBlocks += 1;
- tsdbReleaseDataBlock(pTableScanInfo->dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
- STableScanInfo* pTableScanInfo = pOperator->info;
- if (taosHashGetSize(pTableScanInfo->pIgnoreTables) == taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList)) {
+ STableScanInfo* p1 = pOperator->info;
+ if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) {
*status = FUNC_DATA_REQUIRED_ALL_FILTEROUT;
} else {
*status = FUNC_DATA_REQUIRED_FILTEROUT;
@@ -412,7 +419,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1;
- SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
+ SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
if (p == NULL) {
return terrno;
}
@@ -524,8 +531,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
// 1. check if it is existed in meta cache
if (pCache == NULL) {
- metaReaderInit(&mr, pHandle->meta, 0);
- code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid);
+ pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn);
+ code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid);
if (code != TSDB_CODE_SUCCESS) {
// when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed.
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
@@ -538,11 +545,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno),
idStr);
}
- metaReaderClear(&mr);
+ pHandle->api.metaReaderFn.clearReader(&mr);
return terrno;
}
- metaReaderReleaseLock(&mr);
+ pHandle->api.metaReaderFn.readerReleaseLock(&mr);
val.pName = mr.me.name;
val.pTags = (STag*)mr.me.ctbEntry.pTags;
@@ -553,8 +560,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid));
if (h == NULL) {
- metaReaderInit(&mr, pHandle->meta, 0);
- code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid);
+ pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn);
+ code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid);
if (code != TSDB_CODE_SUCCESS) {
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",
@@ -565,11 +572,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno),
idStr);
}
- metaReaderClear(&mr);
+ pHandle->api.metaReaderFn.clearReader(&mr);
return terrno;
}
- metaReaderReleaseLock(&mr);
+ pHandle->api.metaReaderFn.readerReleaseLock(&mr);
STableCachedVal* pVal = createTableCacheVal(&mr);
@@ -609,7 +616,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
} else { // these are tags
STagVal tagVal = {0};
tagVal.cid = pExpr1->base.pParam[0].pCol->colId;
- const char* p = metaGetTableTagVal(val.pTags, pColInfoData->info.type, &tagVal);
+ const char* p = pHandle->api.metaFn.extractTagVal(val.pTags, pColInfoData->info.type, &tagVal);
char* data = NULL;
if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) {
@@ -628,7 +635,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
}
if (code) {
if (freeReader) {
- metaReaderClear(&mr);
+ pHandle->api.metaReaderFn.clearReader(&mr);
}
return code;
}
@@ -643,7 +650,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
// restore the rows
pBlock->info.rows = backupRows;
if (freeReader) {
- metaReaderClear(&mr);
+ pHandle->api.metaReaderFn.clearReader(&mr);
}
return TSDB_CODE_SUCCESS;
@@ -677,6 +684,8 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData,
static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
STableScanInfo* pTableScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
bool hasNext = false;
int32_t code = TSDB_CODE_SUCCESS;
@@ -684,9 +693,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
int64_t st = taosGetTimestampUs();
while (true) {
- code = tsdbNextDataBlock(pTableScanInfo->base.dataReader, &hasNext);
+ code = pAPI->tsdReader.tsdNextDataBlock(pTableScanInfo->base.dataReader, &hasNext);
if (code) {
- tsdbReleaseDataBlock(pTableScanInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader);
T_LONG_JMP(pTaskInfo->env, code);
}
@@ -695,12 +704,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
}
if (isTaskKilled(pTaskInfo)) {
- tsdbReleaseDataBlock(pTableScanInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader);
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
if (pOperator->status == OP_EXEC_DONE) {
- tsdbReleaseDataBlock(pTableScanInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader);
break;
}
@@ -733,14 +742,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime;
-
- // todo refactor
- /*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/
- /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/
- // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA;
- // pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid;
- // pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey;
-
return pBlock;
}
return NULL;
@@ -749,7 +750,8 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pTableScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
-
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
// The read handle is not initialized yet, since no qualified tables exists
if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) {
return NULL;
@@ -772,7 +774,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo));
// do prepare for the next round table scan operation
- tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
+ pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
}
}
@@ -780,7 +782,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
if (pTableScanInfo->scanTimes < total) {
if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) {
prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0);
- tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
+ pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
}
@@ -798,7 +800,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
pTableScanInfo->base.scanFlag = MAIN_SCAN;
qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo));
- tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
+ pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
}
}
}
@@ -809,6 +811,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
// scan table one by one sequentially
if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
@@ -836,11 +839,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable);
taosRUnLockLatch(&pTaskInfo->lock);
- tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1);
+ pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1);
qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables,
pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo));
- tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond);
+ pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond);
pInfo->scanTimes = 0;
}
} else { // scan table group by group sequentially
@@ -855,8 +858,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
ASSERT(pInfo->base.dataReader == NULL);
- int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock,
- (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables);
+ int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock,
+ (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
@@ -884,8 +887,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STableKeyInfo* pList = NULL;
tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
- tsdbSetTableList(pInfo->base.dataReader, pList, num);
- tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond);
+ pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num);
+ pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond);
pInfo->scanTimes = 0;
result = doGroupedTableScan(pOperator);
@@ -907,10 +910,10 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr
return 0;
}
-static void destroyTableScanBase(STableScanBase* pBase) {
+static void destroyTableScanBase(STableScanBase* pBase, TsdReader* pAPI) {
cleanupQueryTableDataCond(&pBase->cond);
- tsdbReaderClose(pBase->dataReader);
+ pAPI->tsdReaderClose(pBase->dataReader);
pBase->dataReader = NULL;
if (pBase->matchInfo.pList != NULL) {
@@ -926,7 +929,7 @@ static void destroyTableScanOperatorInfo(void* param) {
STableScanInfo* pTableScanInfo = (STableScanInfo*)param;
blockDataDestroy(pTableScanInfo->pResBlock);
taosHashCleanup(pTableScanInfo->pIgnoreTables);
- destroyTableScanBase(&pTableScanInfo->base);
+ destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI);
taosMemoryFreeClear(param);
}
@@ -959,7 +962,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
if (pScanNode->pScanPseudoCols != NULL) {
SExprSupp* pSup = &pInfo->base.pseudoSup;
pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs);
- pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset);
+ pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore);
}
pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]};
@@ -972,6 +975,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pInfo->sample.sampleRatio = pTableScanNode->ratio;
pInfo->sample.seed = taosGetTimestampSec();
+ pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader;
initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
// blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
@@ -1066,13 +1070,12 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou
pInfo->groupId = groupCol[rowIndex];
}
-void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t version) {
+void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t ver) {
pTableScanInfo->base.cond.twindows = *pWin;
- pTableScanInfo->base.cond.endVersion = version;
+ pTableScanInfo->base.cond.endVersion = ver;
pTableScanInfo->scanTimes = 0;
pTableScanInfo->currentGroupId = -1;
- tsdbReaderClose(pTableScanInfo->base.dataReader);
- qDebug("1");
+ pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->base.dataReader = NULL;
}
@@ -1088,11 +1091,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs};
SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
STsdbReader* pReader = NULL;
- int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock,
- (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false, NULL);
+ int32_t code = pAPI->tsdReader.tsdReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock,
+ (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
T_LONG_JMP(pTaskInfo->env, code);
@@ -1100,7 +1104,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
}
bool hasNext = false;
- code = tsdbNextDataBlock(pReader, &hasNext);
+ code = pAPI->tsdReader.tsdNextDataBlock(pReader, &hasNext);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
T_LONG_JMP(pTaskInfo->env, code);
@@ -1108,12 +1112,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
}
if (hasNext) {
- /*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL);
+ /*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL);
doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
}
- tsdbReaderClose(pReader);
+ pAPI->tsdReader.tsdReaderClose(pReader);
qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64
", suid:%" PRIu64,
pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid);
@@ -1178,15 +1182,18 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
win.ekey = TMAX(win.ekey, endData[*pRowIndex]);
continue;
}
+
if (win.skey == endData[*pRowIndex] && groupId == gpData[*pRowIndex]) {
win.skey = TMIN(win.skey, startData[*pRowIndex]);
continue;
}
+
ASSERT(!(win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) ||
!(isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0)));
break;
}
+ STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info;
resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion);
pInfo->pTableScanOp->status = OP_OPENED;
return true;
@@ -1245,8 +1252,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
*pRowIndex = 0;
pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
- tsdbReaderClose(pTableScanInfo->base.dataReader);
- qDebug("2");
+ pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->base.dataReader = NULL;
return NULL;
}
@@ -1291,8 +1297,8 @@ static int32_t getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
pKey->win.ekey = endTs;
pKey->groupId = groupId;
- SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey);
- int32_t code = streamStateSessionGetKVByCur(pCur, pKey, NULL, 0);
+ void* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey);
+ int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pKey, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pKey);
}
@@ -1324,9 +1330,9 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr
SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
- int64_t version = pSrcBlock->info.version - 1;
+ int64_t ver = pSrcBlock->info.version - 1;
for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
- uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version);
+ uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], ver);
// gap must be 0.
SSessionKey startWin = {0};
getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], startData[i], groupId, &startWin);
@@ -1372,13 +1378,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData;
TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData;
- int64_t version = pSrcBlock->info.version - 1;
+ int64_t ver = pSrcBlock->info.version - 1;
if (pInfo->partitionSup.needCalc && srcStartTsCol[0] != srcEndTsCol[0]) {
uint64_t srcUid = srcUidData[0];
TSKEY startTs = srcStartTsCol[0];
TSKEY endTs = srcEndTsCol[0];
- SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, startTs, endTs, version);
+ SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, startTs, endTs, ver);
printDataBlock(pPreRes, "pre res");
blockDataCleanup(pSrcBlock);
int32_t code = blockDataEnsureCapacity(pSrcBlock, pPreRes->info.rows);
@@ -1416,7 +1422,7 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
uint64_t srcUid = srcUidData[i];
uint64_t groupId = srcGp[i];
if (groupId == 0) {
- groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version);
+ groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver);
}
TSKEY calStartTs = srcStartTsCol[i];
colDataSetVal(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
@@ -1453,21 +1459,21 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS
ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData;
TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData;
- int64_t version = pSrcBlock->info.version - 1;
+ int64_t ver = pSrcBlock->info.version - 1;
for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
uint64_t srcUid = srcUidData[i];
uint64_t groupId = srcGp[i];
char* tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0};
if (groupId == 0) {
- groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version);
+ groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver);
}
if (pInfo->tbnameCalSup.pExprInfo) {
void* parTbname = NULL;
- streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname);
+ pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname);
memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN);
varDataSetLen(tbname, strlen(varDataVal(tbname)));
- streamFreeVal(parTbname);
+ pInfo->stateStore.streamStateFreeVal(parTbname);
}
appendOneRowToStreamSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId,
tbname[0] == 0 ? NULL : tbname);
@@ -1498,7 +1504,7 @@ static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
pBlock->info.parTbName[0] = 0;
} else {
appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup,
- pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes);
+ pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes, &pInfo->stateStore);
}
}
@@ -1529,7 +1535,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
TSKEY* tsCol = (TSKEY*)pColDataInfo->pData;
- bool tableInserted = updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid);
+ bool tableInserted = pInfo->stateStore.updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid);
for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) {
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
@@ -1545,10 +1551,9 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
isClosed = isCloseWindow(&win, &pInfo->twAggSup);
}
// must check update info first.
- bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]);
+ bool update = pInfo->stateStore.updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]);
bool closedWin = isClosed && isSignleIntervalWindow(pInfo) &&
- isDeletedStreamWindow(&win, pBlock->info.id.groupId,
- pInfo->pState, &pInfo->twAggSup);
+ isDeletedStreamWindow(&win, pBlock->info.id.groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore);
if ((update || closedWin) && out) {
qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin);
uint64_t gpId = 0;
@@ -1636,58 +1641,30 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
}
static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SStreamScanInfo* pInfo = pOperator->info;
const char* id = GET_TASKID(pTaskInfo);
qDebug("start to exec queue scan, %s", id);
-#if 0
- if (pTaskInfo->streamInfo.submit.msgStr != NULL) {
- if (pInfo->tqReader->msg.msgStr == NULL) {
- SPackedData submit = pTaskInfo->streamInfo.submit;
- if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) {
- qError("submit msg messed up when initing stream submit block %p", submit.msgStr);
- return NULL;
- }
- }
-
- blockDataCleanup(pInfo->pRes);
- SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
-
- while (tqNextBlockImpl(pInfo->tqReader, NULL)) {
- int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL);
- if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) {
- continue;
- }
-
- setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true);
-
- if (pBlockInfo->rows > 0) {
- return pInfo->pRes;
- }
- }
-
- pInfo->tqReader->msg = (SPackedData){0};
- pTaskInfo->streamInfo.submit = (SPackedData){0};
- return NULL;
- }
-#endif
-
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
if (pResult && pResult->info.rows > 0) {
- qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64,
- pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey,
- pInfo->tqReader->pWalReader->curVersion);
+// qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64,
+// pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey,
+// pInfo->tqReader->pWalReader->curVersion);
tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey);
return pResult;
}
+
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
- tsdbReaderClose(pTSInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader);
+
pTSInfo->base.dataReader = NULL;
qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1);
- if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) {
+ if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) {
return NULL;
}
@@ -1697,11 +1674,13 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) {
while (1) {
- bool hasResult = tqNextBlockInWal(pInfo->tqReader, id);
- SSDataBlock* pRes = pInfo->tqReader->pResBlock;
+ bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id);
+
+ SSDataBlock* pRes = pAPI->tqReaderFn.tqGetResultBlock(pInfo->tqReader);
+ struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader);
// curVersion move to next, so currentOffset = curVersion - 1
- tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1);
+ tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion - 1);
if (hasResult) {
qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows,
@@ -1737,9 +1716,10 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S
SColumnInfoData* pDstStartCol = taosArrayGet(pDst->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pDstEndCol = taosArrayGet(pDst->pDataBlock, END_TS_COLUMN_INDEX);
SColumnInfoData* pDstUidCol = taosArrayGet(pDst->pDataBlock, UID_COLUMN_INDEX);
- int32_t j = 0;
+
+ int32_t j = 0;
for (int32_t i = 0; i < rows; i++) {
- if (taosHashGet(pReader->tbIdHash, &uidCol[i], sizeof(uint64_t))) {
+ if (pInfo->readerFn.tqReaderIsQueriedTable(pReader, uidCol[i])) {
colDataSetVal(pDstStartCol, j, (const char*)&startCol[i], false);
colDataSetVal(pDstEndCol, j, (const char*)&endCol[i], false);
colDataSetVal(pDstUidCol, j, (const char*)&uidCol[i], false);
@@ -1750,6 +1730,7 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S
j++;
}
}
+
uint32_t cap = pDst->info.capacity;
pDst->info = pSrc->info;
pDst->info.rows = j;
@@ -1794,21 +1775,21 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl
}
}
-int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) {
- int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo);
- *pBuff = taosMemoryCalloc(1, len);
- updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo);
- return len;
-}
+//int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) {
+// int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo);
+// *pBuff = taosMemoryCalloc(1, len);
+// updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo);
+// return len;
+//}
// other properties are recovered from the execution plan
-void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo) {
+void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) {
if (!pBuff || len == 0) {
return;
}
- SUpdateInfo* pUpInfo = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0);
- int32_t code = updateInfoDeserialize(pBuff, len, pUpInfo);
+ void* pUpInfo = pInfo->stateStore.updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0);
+ int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo);
if (code == TSDB_CODE_SUCCESS) {
pInfo->pUpdateInfo = pUpInfo;
}
@@ -1817,6 +1798,8 @@ void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo)
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SStreamScanInfo* pInfo = pOperator->info;
qDebug("stream scan started, %s", GET_TASKID(pTaskInfo));
@@ -1839,7 +1822,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2;
}
- tsdbReaderClose(pTSInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL;
pInfo->pTableScanOp->status = OP_OPENED;
@@ -1902,7 +1885,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
calBlockTbName(pInfo, pInfo->pRecoverRes);
if (pInfo->pUpdateInfo) {
if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) {
- TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex);
+ TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
} else {
doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes);
@@ -1919,7 +1902,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
}
pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE;
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
- tsdbReaderClose(pTSInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL;
@@ -1943,7 +1926,7 @@ FETCH_NEXT_BLOCK:
SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current);
SSDataBlock* pBlock = pPacked->pDataBlock;
if (pBlock->info.parTbName[0]) {
- streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName);
+ pAPI->stateStore.streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName);
}
// TODO move into scan
@@ -1964,7 +1947,7 @@ FETCH_NEXT_BLOCK:
copyDataBlock(pInfo->pUpdateRes, pBlock);
pInfo->updateResIndex = 0;
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
- updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
+ pAPI->stateStore.updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
} break;
case STREAM_DELETE_DATA: {
printDataBlock(pBlock, "stream scan delete recv");
@@ -2081,9 +2064,9 @@ FETCH_NEXT_BLOCK:
NEXT_SUBMIT_BLK:
while (1) {
- if (pInfo->tqReader->msg.msgStr == NULL) {
+ if (pInfo->readerFn.tqReaderCurrentBlockConsumed(pInfo->tqReader)) {
if (pInfo->validBlockIndex >= totalBlocks) {
- updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
+ pAPI->stateStore.updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
doClearBufferedBlocks(pInfo);
qDebug("stream scan return empty, all %d submit blocks consumed, %s", totalBlocks, id);
@@ -2094,7 +2077,7 @@ FETCH_NEXT_BLOCK:
SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current);
qDebug("set %d/%d as the input submit block, %s", current, totalBlocks, id);
- if (tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) {
+ if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) {
qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current, totalBlocks, id);
continue;
}
@@ -2102,13 +2085,14 @@ FETCH_NEXT_BLOCK:
blockDataCleanup(pBlock);
- while (tqNextBlockImpl(pInfo->tqReader, id)) {
- int32_t code = tqRetrieveDataBlock(pInfo->tqReader, id);
- if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) {
+ while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) {
+ SSDataBlock* pRes = NULL;
+ int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id);
+ if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) {
continue;
}
- setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, false);
+ setBlockIntoRes(pInfo, pRes, false);
if (pInfo->pCreateTbRes->info.rows > 0) {
pInfo->scanMode = STREAM_SCAN_FROM_RES;
@@ -2166,7 +2150,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableListInfo) {
static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SStreamRawScanInfo* pInfo = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta
@@ -2176,20 +2162,20 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
bool hasNext = false;
if (pInfo->dataReader) {
- code = tsdbNextDataBlock(pInfo->dataReader, &hasNext);
+ code = pAPI->tsdReader.tsdNextDataBlock(pInfo->dataReader, &hasNext);
if (code) {
- tsdbReleaseDataBlock(pInfo->dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader);
T_LONG_JMP(pTaskInfo->env, code);
}
}
if (pInfo->dataReader && hasNext) {
if (isTaskKilled(pTaskInfo)) {
- tsdbReleaseDataBlock(pInfo->dataReader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader);
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
- SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL);
+ SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL);
if (pBlock == NULL) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
@@ -2199,7 +2185,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
return pBlock;
}
- SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext);
+ SMetaTableInfo mtInfo = pAPI->snapshotFn.getMetaTableInfoFromSnapshot(pInfo->sContext);
STqOffsetVal offset = {0};
if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal
qDebug("tmqsnap read snapshot done, change to get data from wal");
@@ -2217,8 +2203,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
int32_t dataLen = 0;
int16_t type = 0;
int64_t uid = 0;
- if (getMetafromSnapShot(sContext, &data, &dataLen, &type, &uid) < 0) {
- qError("tmqsnap getMetafromSnapShot error");
+ if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) {
+ qError("tmqsnap getTableInfoFromSnapshot error");
taosMemoryFreeClear(data);
return NULL;
}
@@ -2241,8 +2227,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
static void destroyRawScanOperatorInfo(void* param) {
SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param;
- tsdbReaderClose(pRawScan->dataReader);
- destroySnapContext(pRawScan->sContext);
+ pRawScan->pAPI->tsdReader.tsdReaderClose(pRawScan->dataReader);
+ pRawScan->pAPI->snapshotFn.destroySnapshot(pRawScan->sContext);
tableListDestroy(pRawScan->pTableListInfo);
taosMemoryFree(pRawScan);
}
@@ -2267,6 +2253,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT
pInfo->pTableListInfo = tableListCreate();
pInfo->vnode = pHandle->vnode;
+ pInfo->pAPI = &pTaskInfo->storageAPI;
pInfo->sContext = pHandle->sContext;
setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo,
@@ -2290,7 +2277,7 @@ static void destroyStreamScanOperatorInfo(void* param) {
}
if (pStreamScan->tqReader) {
- tqCloseReader(pStreamScan->tqReader);
+ pStreamScan->readerFn.tqReaderClose(pStreamScan->tqReader);
}
if (pStreamScan->matchInfo.pList) {
taosArrayDestroy(pStreamScan->matchInfo.pList);
@@ -2303,7 +2290,7 @@ static void destroyStreamScanOperatorInfo(void* param) {
cleanupExprSupp(&pStreamScan->tbnameCalSup);
cleanupExprSupp(&pStreamScan->tagCalSup);
- updateInfoDestroy(pStreamScan->pUpdateInfo);
+ pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo);
blockDataDestroy(pStreamScan->pRes);
blockDataDestroy(pStreamScan->pUpdateRes);
blockDataDestroy(pStreamScan->pPullDataRes);
@@ -2319,6 +2306,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
SArray* pColIds = NULL;
SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
if (pInfo == NULL || pOperator == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -2362,7 +2350,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo->tbnameCalSup.pExprInfo = pSubTableExpr;
createExprFromOneNode(pSubTableExpr, pTableScanNode->pSubtable, 0);
- if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1) != 0) {
+ if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore) != 0) {
tableListDestroy(pTableListInfo);
goto _error;
}
@@ -2376,7 +2364,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
tableListDestroy(pTableListInfo);
goto _error;
}
- if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) {
+ if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tableListDestroy(pTableListInfo);
goto _error;
@@ -2408,7 +2396,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
if (pHandle->initTqReader) {
ASSERT(pHandle->tqReader == NULL);
- pInfo->tqReader = tqReaderOpen(pHandle->vnode);
+ pInfo->tqReader = pAPI->tqReaderFn.tqReaderOpen(pHandle->vnode);
ASSERT(pInfo->tqReader);
} else {
ASSERT(pHandle->tqReader);
@@ -2418,7 +2406,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo->pUpdateInfo = NULL;
pInfo->pTableScanOp = pTableScanOp;
if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) {
- streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1);
+ pAPI->stateStore.streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1);
}
pInfo->readHandle = *pHandle;
@@ -2427,9 +2415,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
blockDataEnsureCapacity(pInfo->pCreateTbRes, 8);
// set the extract column id to streamHandle
- tqReaderSetColIdList(pInfo->tqReader, pColIds);
+ pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds);
SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo);
- code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList);
+ code = pAPI->tqReaderFn.tqReaderSetQueryTableList(pInfo->tqReader, tableIdList);
if (code != 0) {
taosArrayDestroy(tableIdList);
goto _error;
@@ -2470,13 +2458,15 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo->igExpired = pTableScanNode->igExpired;
pInfo->twAggSup.maxTs = INT64_MIN;
pInfo->pState = NULL;
+ pInfo->stateStore = pTaskInfo->storageAPI.stateStore;
+ pInfo->readerFn = pTaskInfo->storageAPI.tqReaderFn;
// for stream
if (pTaskInfo->streamInfo.pState) {
void* buff = NULL;
int32_t len = 0;
- streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len);
- streamScanOperatorDeocde(buff, len, pInfo);
+ pAPI->stateStore.streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len);
+ streamScanOperatorDecode(buff, len, pInfo);
}
setOperatorInfo(pOperator, STREAM_SCAN_OP_NAME, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo,
@@ -2502,18 +2492,18 @@ _error:
return NULL;
}
-static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr) {
+static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr, SStorageAPI* pAPI) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
STagScanInfo* pInfo = pOperator->info;
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0];
STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos);
- int32_t code = metaGetTableEntryByUid(mr, item->uid);
+ int32_t code = pAPI->metaReaderFn.getTableEntryByUid(mr, item->uid);
tDecoderClear(&(*mr).coder);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno),
GET_TASKID(pTaskInfo));
- metaReaderClear(mr);
+ pAPI->metaReaderFn.clearReader(mr);
T_LONG_JMP(pTaskInfo->env, terrno);
}
@@ -2528,7 +2518,7 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes,
} else { // it is a tag value
STagVal val = {0};
val.cid = pExprInfo[j].base.pParam[0].pCol->colId;
- const char* p = metaGetTableTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val);
+ const char* p = pAPI->metaFn.extractTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val);
char* data = NULL;
if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) {
@@ -2553,6 +2543,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
}
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
STagScanInfo* pInfo = pOperator->info;
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0];
@@ -2568,10 +2559,10 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
char str[512] = {0};
int32_t count = 0;
SMetaReader mr = {0};
- metaReaderInit(&mr, pInfo->readHandle.meta, 0);
+ pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) {
- doTagScanOneTable(pOperator, pRes, count, &mr);
+ doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI);
++count;
if (++pInfo->curPos >= size) {
setOperatorCompleted(pOperator);
@@ -2589,7 +2580,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
}
}
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
// qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
if (pOperator->status == OP_EXEC_DONE) {
@@ -2622,7 +2613,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
int32_t numOfExprs = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs);
- int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs);
+ int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -2661,6 +2652,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
SOperatorInfo* pOperator = source->pOperator;
STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
int32_t readIdx = source->readerIdx;
SSDataBlock* pBlock = source->inputBlock;
int32_t code = 0;
@@ -2672,7 +2665,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
SReadHandle* pHandle = &pInfo->base.readHandle;
if (NULL == source->dataReader || !source->multiReader) {
- code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false, NULL);
+ code = pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL);
if (code != 0) {
T_LONG_JMP(pTaskInfo->env, code);
}
@@ -2684,9 +2677,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
qTrace("tsdb/read-table-data: %p, enter next reader", reader);
while (true) {
- code = tsdbNextDataBlock(reader, &hasNext);
+ code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext);
if (code != 0) {
- tsdbReleaseDataBlock(reader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(reader);
pInfo->base.dataReader = NULL;
T_LONG_JMP(pTaskInfo->env, code);
}
@@ -2696,7 +2689,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
}
if (isTaskKilled(pTaskInfo)) {
- tsdbReleaseDataBlock(reader);
+ pAPI->tsdReader.tsdReaderReleaseDataBlock(reader);
pInfo->base.dataReader = NULL;
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
@@ -2736,7 +2729,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
qTrace("tsdb/read-table-data: %p, close reader", reader);
if (!source->multiReader) {
- tsdbReaderClose(pInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader);
source->dataReader = NULL;
}
pInfo->base.dataReader = NULL;
@@ -2744,7 +2737,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
}
if (!source->multiReader) {
- tsdbReaderClose(pInfo->base.dataReader);
+ pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader);
source->dataReader = NULL;
}
pInfo->base.dataReader = NULL;
@@ -2849,6 +2842,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
int32_t numOfTable = taosArrayGetSize(pInfo->queryConds);
@@ -2862,7 +2856,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
for (int32_t i = 0; i < numOfTable; ++i) {
STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
blockDataDestroy(param->inputBlock);
- tsdbReaderClose(param->dataReader);
+ pAPI->tsdReader.tsdReaderClose(param->dataReader);
param->dataReader = NULL;
}
taosArrayClear(pInfo->sortSourceParams);
@@ -2974,11 +2968,11 @@ void destroyTableMergeScanOperatorInfo(void* param) {
for (int32_t i = 0; i < numOfTable; i++) {
STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i);
blockDataDestroy(p->inputBlock);
- tsdbReaderClose(p->dataReader);
+ pTableScanInfo->base.readerAPI.tsdReaderClose(p->dataReader);
p->dataReader = NULL;
}
- tsdbReaderClose(pTableScanInfo->base.dataReader);
+ pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->base.dataReader = NULL;
taosArrayDestroy(pTableScanInfo->sortSourceParams);
@@ -2991,7 +2985,7 @@ void destroyTableMergeScanOperatorInfo(void* param) {
}
taosArrayDestroy(pTableScanInfo->queryConds);
- destroyTableScanBase(&pTableScanInfo->base);
+ destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI);
pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock);
pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock);
@@ -3040,7 +3034,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
if (pTableScanNode->scan.pScanPseudoCols != NULL) {
SExprSupp* pSup = &pInfo->base.pseudoSup;
pSup->pExprInfo = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->numOfExprs);
- pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset);
+ pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore);
}
pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]};
@@ -3051,6 +3045,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
goto _error;
}
+ pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader;
pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD;
pInfo->base.scanFlag = MAIN_SCAN;
pInfo->base.readHandle = *readHandle;
@@ -3102,9 +3097,9 @@ _error:
static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator);
static void destoryTableCountScanOperator(void* param);
static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
- SSDataBlock* pRes, char* dbName, tb_uid_t stbUid);
+ SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI);
static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
- SSDataBlock* pRes, char* dbName);
+ SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI);
static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName);
static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
@@ -3191,6 +3186,7 @@ int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList*
qError("%s get table count scan supp. get inputs error", GET_TASKID(taskInfo));
return code;
}
+
supp->dbNameSlotId = -1;
supp->stbNameSlotId = -1;
supp->tbCountSlotId = -1;
@@ -3200,6 +3196,7 @@ int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList*
qError("%s get table count scan supp. get group tags slot id error", GET_TASKID(taskInfo));
return code;
}
+
code = tblCountScanGetCountSlotId(pseudoCols, supp);
if (code != TSDB_CODE_SUCCESS) {
qError("%s get table count scan supp. get count error", GET_TASKID(taskInfo));
@@ -3359,9 +3356,11 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount
const char* db = NULL;
int32_t vgId = 0;
char dbName[TSDB_DB_NAME_LEN] = {0};
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
// get dbname
- vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+ pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
SName sn = {0};
tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&sn, dbName);
@@ -3376,20 +3375,23 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount
static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) {
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
if (pSupp->groupByStbName) {
if (pInfo->stbUidList == NULL) {
pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t));
- if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) {
+ if (pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, TSDB_SUPER_TABLE, pInfo->stbUidList) < 0) {
qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr());
}
}
if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) {
tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx);
- buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid);
+ buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid, pAPI);
pInfo->currGrpIdx++;
} else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) {
- buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName);
+ buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName, pAPI);
pInfo->currGrpIdx++;
} else {
@@ -3398,7 +3400,9 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca
} else {
uint64_t groupId = calcGroupId(dbName, strlen(dbName));
pRes->info.id.groupId = groupId;
- int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta);
+
+ int64_t dbTableCount = 0;
+ pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &dbTableCount, NULL);
fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes);
setOperatorCompleted(pOperator);
}
@@ -3406,26 +3410,34 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca
static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) {
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
if (strlen(pSupp->dbNameFilter) != 0) {
if (strlen(pSupp->stbNameFilter) != 0) {
- tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter);
- SMetaStbStats stats = {0};
- metaGetStbStats(pInfo->readHandle.meta, uid, &stats);
- int64_t ctbNum = stats.ctbNum;
- fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes);
+ uint64_t uid = 0;
+ pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid);
+
+ int64_t numOfChildTables = 0;
+ pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, uid, &numOfChildTables);
+
+ fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, numOfChildTables, pRes);
} else {
- int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta);
+ int64_t tbNumVnode = 0;
+ pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL);
fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
}
} else {
- int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta);
+ int64_t tbNumVnode = 0;
+ pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL);
fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
}
+
setOperatorCompleted(pOperator);
}
static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
- SSDataBlock* pRes, char* dbName) {
+ SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI) {
char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
if (pSupp->groupByDbName) {
snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
@@ -3433,32 +3445,33 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S
uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
pRes->info.id.groupId = groupId;
- int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta);
- if (ntbNum != 0) {
- fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes);
+
+ int64_t numOfTables = 0;//metaGetNtbNum(pInfo->readHandle.vnode);
+ pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, NULL, &numOfTables);
+
+ if (numOfTables != 0) {
+ fillTableCountScanDataBlock(pSupp, dbName, "", numOfTables, pRes);
}
}
static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
- SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) {
+ SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI) {
char stbName[TSDB_TABLE_NAME_LEN] = {0};
- metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName);
+ pAPI->metaFn.getTableNameByUid(pInfo->readHandle.vnode, stbUid, stbName);
char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
if (pSupp->groupByDbName) {
- snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName);
+ snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, varDataVal(stbName));
} else {
- snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", stbName);
+ snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", varDataVal(stbName));
}
uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
pRes->info.id.groupId = groupId;
- SMetaStbStats stats = {0};
- metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats);
- int64_t ctbNum = stats.ctbNum;
-
- fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes);
+ int64_t ctbNum = 0;
+ int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum);
+ fillTableCountScanDataBlock(pSupp, dbName, varDataVal(stbName), ctbNum, pRes);
}
static void destoryTableCountScanOperator(void* param) {
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 718bb596c5..0357828732 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -60,7 +60,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
}
pOperator->exprSupp.pCtx =
- createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
+ createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore);
initResultSizeInfo(&pOperator->resultInfo, 1024);
code = filterInitFromNode((SNode*)pSortNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
@@ -500,7 +500,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
pSup->numOfExprs = numOfCols;
initResultSizeInfo(&pOperator->resultInfo, 1024);
- pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
+ pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore);
pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c
index 04c6c619aa..fd766e21f8 100644
--- a/source/libs/executor/src/sysscanoperator.c
+++ b/source/libs/executor/src/sysscanoperator.c
@@ -21,27 +21,27 @@
#include "querynodes.h"
#include "systable.h"
#include "tname.h"
-#include "ttime.h"
#include "tdatablock.h"
#include "tmsg.h"
+#include "operator.h"
#include "query.h"
+#include "querytask.h"
+#include "storageapi.h"
#include "tcompare.h"
#include "thash.h"
#include "ttypes.h"
-#include "vnode.h"
-#include "operator.h"
-#include "querytask.h"
-
+#include "index.h"
typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype);
typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result);
typedef int32_t (*__sys_check)(SNode* cond);
typedef struct SSTabFltArg {
- void* pMeta;
- void* pVnode;
+ void* pMeta;
+ void* pVnode;
+ SStorageAPI* pAPI;
} SSTabFltArg;
typedef struct SSysTableIndex {
@@ -72,6 +72,7 @@ typedef struct SSysTableScanInfo {
SLoadRemoteDataInfo loadInfo;
SLimitInfo limitInfo;
int32_t tbnameSlotId;
+ SStorageAPI* pAPI;
} SSysTableScanInfo;
typedef struct {
@@ -153,10 +154,11 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t
SFilterInfo* pFilterInfo);
int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) {
- void* pVnode = ((SSTabFltArg*)arg)->pVnode;
+ SSTabFltArg* pArg = arg;
+ void* pVnode = pArg->pVnode;
const char* db = NULL;
- vnodeGetInfo(pVnode, &db, NULL);
+ pArg->pAPI->metaFn.getBasicInfo(pVnode, &db, NULL, NULL, NULL);
SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -180,10 +182,11 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) {
}
int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) {
- void* pVnode = ((SSTabFltArg*)arg)->pVnode;
+ SSTabFltArg* pArg = arg;
+ void* pVnode = ((SSTabFltArg*)arg)->pVnode;
int64_t vgId = 0;
- vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId);
+ pArg->pAPI->metaFn.getBasicInfo(pVnode, NULL, (int32_t*)&vgId, NULL, NULL);
SOperatorNode* pOper = (SOperatorNode*)pNode;
SValueNode* pVal = (SValueNode*)pOper->pRight;
@@ -200,7 +203,7 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) {
}
int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) {
- void* pMeta = ((SSTabFltArg*)arg)->pMeta;
+ SSTabFltArg* pArg = arg;
SOperatorNode* pOper = (SOperatorNode*)pNode;
SValueNode* pVal = (SValueNode*)pOper->pRight;
@@ -220,7 +223,8 @@ int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) {
}
int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) {
- void* pMeta = ((SSTabFltArg*)arg)->pMeta;
+ SSTabFltArg* pArg = arg;
+ SStorageAPI* pAPI = pArg->pAPI;
SOperatorNode* pOper = (SOperatorNode*)pNode;
SValueNode* pVal = (SValueNode*)pOper->pRight;
@@ -237,7 +241,7 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) {
.equal = equal,
.filterFunc = func};
- int32_t ret = metaFilterCreateTime(pMeta, ¶m, result);
+ int32_t ret = pAPI->metaFilter.metaFilterCreateTime(pArg->pVnode, ¶m, result);
return ret;
}
@@ -431,8 +435,9 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) {
}
static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
- qDebug("sysTableScanUserCols get cols start");
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SSysTableScanInfo* pInfo = pOperator->info;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
@@ -446,7 +451,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
const char* db = NULL;
int32_t vgId = 0;
- vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+ pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -461,18 +466,18 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
STR_TO_VARSTR(tableName, pInfo->req.filterTb);
SMetaReader smrTable = {0};
- metaReaderInit(&smrTable, pInfo->readHandle.meta, 0);
- int32_t code = metaGetTableEntryByName(&smrTable, pInfo->req.filterTb);
+ pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
+ int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb);
if (code != TSDB_CODE_SUCCESS) {
- // terrno has been set by metaGetTableEntryByName, therefore, return directly
- metaReaderClear(&smrTable);
+ // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
+ pAPI->metaReaderFn.clearReader(&smrTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
}
if (smrTable.me.type == TSDB_SUPER_TABLE) {
- metaReaderClear(&smrTable);
+ pAPI->metaReaderFn.clearReader(&smrTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
@@ -480,12 +485,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
if (smrTable.me.type == TSDB_CHILD_TABLE) {
int64_t suid = smrTable.me.ctbEntry.suid;
- metaReaderClear(&smrTable);
- metaReaderInit(&smrTable, pInfo->readHandle.meta, 0);
- code = metaGetTableEntryByUid(&smrTable, suid);
+ pAPI->metaReaderFn.clearReader(&smrTable);
+ pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
+ code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid);
if (code != TSDB_CODE_SUCCESS) {
- // terrno has been set by metaGetTableEntryByName, therefore, return directly
- metaReaderClear(&smrTable);
+ // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
+ pAPI->metaReaderFn.clearReader(&smrTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
@@ -503,7 +508,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
}
sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName);
- metaReaderClear(&smrTable);
+ pAPI->metaReaderFn.clearReader(&smrTable);
if (numOfRows > 0) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
@@ -517,7 +522,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
int32_t ret = 0;
if (pInfo->pCur == NULL) {
- pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta);
+ pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode);
}
if (pInfo->pSchema == NULL) {
@@ -535,8 +540,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
int32_t restore = pInfo->restore;
pInfo->restore = false;
- while (restore || ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) {
- if (restore) restore = false;
+
+ while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) {
+ if (restore) {
+ restore = false;
+ }
+
char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -560,12 +569,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
schemaRow = *(SSchemaWrapper**)schema;
} else {
SMetaReader smrSuperTable = {0};
- metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0);
- int code = metaGetTableEntryByUid(&smrSuperTable, suid);
+ pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
+ int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid);
if (code != TSDB_CODE_SUCCESS) {
- // terrno has been set by metaGetTableEntryByName, therefore, return directly
+ // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code);
- metaReaderClear(&smrSuperTable);
+ pAPI->metaReaderFn.clearReader(&smrSuperTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
@@ -573,7 +582,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow);
taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES);
schemaRow = schemaWrapper;
- metaReaderClear(&smrSuperTable);
+ pAPI->metaReaderFn.clearReader(&smrSuperTable);
}
} else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) {
qDebug("sysTableScanUserCols cursor get normal table");
@@ -605,7 +614,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
blockDataDestroy(dataBlock);
if (ret != 0) {
- metaCloseTbCursor(pInfo->pCur);
+ pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
pInfo->pCur = NULL;
setOperatorCompleted(pOperator);
}
@@ -618,6 +627,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SSysTableScanInfo* pInfo = pOperator->info;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
@@ -631,7 +642,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
const char* db = NULL;
int32_t vgId = 0;
- vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+ pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -647,37 +658,37 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
STR_TO_VARSTR(tableName, condTableName);
SMetaReader smrChildTable = {0};
- metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0);
- int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName);
+ pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
+ int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName);
if (code != TSDB_CODE_SUCCESS) {
- // terrno has been set by metaGetTableEntryByName, therefore, return directly
- metaReaderClear(&smrChildTable);
+ // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
+ pAPI->metaReaderFn.clearReader(&smrChildTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
}
if (smrChildTable.me.type != TSDB_CHILD_TABLE) {
- metaReaderClear(&smrChildTable);
+ pAPI->metaReaderFn.clearReader(&smrChildTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
}
SMetaReader smrSuperTable = {0};
- metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK);
- code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
+ pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn);
+ code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
if (code != TSDB_CODE_SUCCESS) {
- // terrno has been set by metaGetTableEntryByUid
- metaReaderClear(&smrSuperTable);
- metaReaderClear(&smrChildTable);
+ // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid
+ pAPI->metaReaderFn.clearReader(&smrSuperTable);
+ pAPI->metaReaderFn.clearReader(&smrChildTable);
blockDataDestroy(dataBlock);
return NULL;
}
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
- metaReaderClear(&smrSuperTable);
- metaReaderClear(&smrChildTable);
+ pAPI->metaReaderFn.clearReader(&smrSuperTable);
+ pAPI->metaReaderFn.clearReader(&smrChildTable);
if (numOfRows > 0) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
@@ -691,11 +702,11 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
int32_t ret = 0;
if (pInfo->pCur == NULL) {
- pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta);
+ pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode);
}
bool blockFull = false;
- while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
+ while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) {
continue;
}
@@ -704,27 +715,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name);
SMetaReader smrSuperTable = {0};
- metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0);
+ pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
- int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid);
+ int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno),
GET_TASKID(pTaskInfo));
- metaReaderClear(&smrSuperTable);
- metaCloseTbCursor(pInfo->pCur);
+ pAPI->metaReaderFn.clearReader(&smrSuperTable);
+ pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
pInfo->pCur = NULL;
T_LONG_JMP(pTaskInfo->env, terrno);
}
if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) {
- metaTbCursorPrev(pInfo->pCur, TSDB_TABLE_MAX);
+ pAPI->metaFn.cursorPrev(pInfo->pCur, TSDB_TABLE_MAX);
blockFull = true;
} else {
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows,
dataBlock);
}
- metaReaderClear(&smrSuperTable);
+ pAPI->metaReaderFn.clearReader(&smrSuperTable);
if (blockFull || numOfRows >= pOperator->resultInfo.capacity) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
@@ -745,7 +756,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
blockDataDestroy(dataBlock);
if (ret != 0) {
- metaCloseTbCursor(pInfo->pCur);
+ pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
pInfo->pCur = NULL;
setOperatorCompleted(pOperator);
}
@@ -1090,6 +1101,8 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) {
static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
SSysTableScanInfo* pInfo = pOperator->info;
SSysTableIndex* pIdx = pInfo->pIdx;
@@ -1100,7 +1113,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
const char* db = NULL;
int32_t vgId = 0;
- vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+ pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -1118,10 +1131,10 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
tb_uid_t* uid = taosArrayGet(pIdx->uids, i);
SMetaReader mr = {0};
- metaReaderInit(&mr, pInfo->readHandle.meta, 0);
- ret = metaGetTableEntryByUid(&mr, *uid);
+ pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, 0, &pAPI->metaFn);
+ ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid);
if (ret < 0) {
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
continue;
}
STR_TO_VARSTR(n, mr.me.name);
@@ -1146,15 +1159,15 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
SMetaReader mr1 = {0};
- metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK);
+ pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn);
int64_t suid = mr.me.ctbEntry.suid;
- int32_t code = metaGetTableEntryByUid(&mr1, suid);
+ int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name,
suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
- metaReaderClear(&mr1);
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr1);
+ pAPI->metaReaderFn.clearReader(&mr);
T_LONG_JMP(pTaskInfo->env, terrno);
}
pColInfoData = taosArrayGet(p->pDataBlock, 3);
@@ -1164,7 +1177,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
STR_TO_VARSTR(n, mr1.me.name);
pColInfoData = taosArrayGet(p->pDataBlock, 4);
colDataSetVal(pColInfoData, numOfRows, n, false);
- metaReaderClear(&mr1);
+ pAPI->metaReaderFn.clearReader(&mr1);
// table comment
pColInfoData = taosArrayGet(p->pDataBlock, 8);
@@ -1229,7 +1242,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
// impl later
}
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
pColInfoData = taosArrayGet(p->pDataBlock, 9);
colDataSetVal(pColInfoData, numOfRows, n, false);
@@ -1275,10 +1288,11 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info;
if (pInfo->pCur == NULL) {
- pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta);
+ pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode);
}
blockDataCleanup(pInfo->pRes);
@@ -1286,7 +1300,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
const char* db = NULL;
int32_t vgId = 0;
- vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+ pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@@ -1301,7 +1315,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
int32_t ret = 0;
- while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
+ while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
STR_TO_VARSTR(n, pInfo->pCur->mr.me.name);
// table name
@@ -1324,15 +1338,15 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
SMetaReader mr = {0};
- metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK);
+ pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn);
uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
- int32_t code = metaGetTableEntryByUid(&mr, suid);
+ int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name,
suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
- metaReaderClear(&mr);
- metaCloseTbCursor(pInfo->pCur);
+ pAPI->metaReaderFn.clearReader(&mr);
+ pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
pInfo->pCur = NULL;
T_LONG_JMP(pTaskInfo->env, terrno);
}
@@ -1345,7 +1359,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
STR_TO_VARSTR(n, mr.me.name);
pColInfoData = taosArrayGet(p->pDataBlock, 4);
colDataSetVal(pColInfoData, numOfRows, n, false);
- metaReaderClear(&mr);
+ pAPI->metaReaderFn.clearReader(&mr);
// table comment
pColInfoData = taosArrayGet(p->pDataBlock, 8);
@@ -1442,7 +1456,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
// todo temporarily free the cursor here, the true reason why the free is not valid needs to be found
if (ret != 0) {
- metaCloseTbCursor(pInfo->pCur);
+ pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
pInfo->pCur = NULL;
setOperatorCompleted(pOperator);
}
@@ -1471,7 +1485,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
} else {
if (pInfo->showRewrite == false) {
if (pCondition != NULL && pInfo->pIdx == NULL) {
- SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode};
+ SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI};
SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex));
idx->init = 0;
@@ -1708,7 +1722,7 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode,
const char* pUser, SExecTaskInfo* pTaskInfo) {
- int32_t code = TDB_CODE_SUCCESS;
+ int32_t code = TSDB_CODE_SUCCESS;
SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@@ -1726,6 +1740,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
extractTbnameSlotId(pInfo, pScanNode);
+ pInfo->pAPI = &pTaskInfo->storageAPI;
+
pInfo->accountId = pScanPhyNode->accountId;
pInfo->pUser = taosStrdup((void*)pUser);
pInfo->sysInfo = pScanPhyNode->sysInfo;
@@ -1798,9 +1814,13 @@ void destroySysScanOperator(void* param) {
if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 ||
strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 ||
strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) {
- metaCloseTbCursor(pInfo->pCur);
+ if (pInfo->pAPI->metaFn.closeTableMetaCursor != NULL) {
+ pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur);
+ }
+
pInfo->pCur = NULL;
}
+
if (pInfo->pIdx) {
taosArrayDestroy(pInfo->pIdx->uids);
taosMemoryFree(pInfo->pIdx);
@@ -2124,15 +2144,15 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
return -1;
}
-static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) {
+static int32_t doGetTableRowSize(SReadHandle *pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) {
*rowLen = 0;
SMetaReader mr = {0};
- metaReaderInit(&mr, pMeta, 0);
- int32_t code = metaGetTableEntryByUid(&mr, uid);
+ pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0, &pHandle->api.metaFn);
+ int32_t code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, uid);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr);
- metaReaderClear(&mr);
+ pHandle->api.metaReaderFn.clearReader(&mr);
return terrno;
}
@@ -2144,10 +2164,10 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
} else if (mr.me.type == TSDB_CHILD_TABLE) {
uint64_t suid = mr.me.ctbEntry.suid;
tDecoderClear(&mr.coder);
- code = metaGetTableEntryByUid(&mr, suid);
+ code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, suid);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr);
- metaReaderClear(&mr);
+ pHandle->api.metaReaderFn.clearReader(&mr);
return terrno;
}
@@ -2163,7 +2183,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
}
}
- metaReaderClear(&mr);
+ pHandle->api.metaReaderFn.clearReader(&mr);
return TSDB_CODE_SUCCESS;
}
@@ -2174,16 +2194,17 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
SBlockDistInfo* pBlockScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN};
- int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid,
+ int32_t code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid,
(int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
- tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
- blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle);
+ pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
+ blockDistInfo.numOfInmemRows = (int32_t) pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle);
SSDataBlock* pBlock = pBlockScanInfo->pResBlock;
@@ -2213,7 +2234,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
static void destroyBlockDistScanOperatorInfo(void* param) {
SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param;
blockDataDestroy(pDistInfo->pResBlock);
- tsdbReaderClose(pDistInfo->pHandle);
+ pDistInfo->readHandle.api.tsdReader.tsdReaderClose(pDistInfo->pHandle);
tableListDestroy(pDistInfo->pTableListInfo);
taosMemoryFreeClear(param);
}
@@ -2268,7 +2289,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
size_t num = tableListGetSize(pTableListInfo);
void* pList = tableListGetInfo(pTableListInfo, 0);
- code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false, NULL);
+ code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL);
cleanupQueryTableDataCond(&cond);
if (code != 0) {
goto _error;
@@ -2280,7 +2301,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
- int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
+ int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c
index 432a627957..3e4055876d 100644
--- a/source/libs/executor/src/timesliceoperator.c
+++ b/source/libs/executor/src/timesliceoperator.c
@@ -18,6 +18,7 @@
#include "functionMgt.h"
#include "operator.h"
#include "querytask.h"
+#include "storageapi.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
@@ -873,7 +874,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
int32_t numOfExprs = 0;
SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
- int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs);
+ int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -881,7 +882,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
if (pInterpPhyNode->pExprs != NULL) {
int32_t num = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
- code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
+ code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 9d882a90ce..aa0aa9799b 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1334,10 +1334,12 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf,
}
static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) {
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SWinKey key = {.ts = ts, .groupId = groupId};
tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));
- streamStateDel(pInfo->pState, &key);
+ pAPI->stateStore.streamStateDel(pInfo->pState, &key);
return true;
}
@@ -1474,7 +1476,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
for (int32_t i = *index; i < size; i++) {
SWinKey* pWin = taosArrayGet(pWins, i);
void* tbname = NULL;
- streamStateGetParName(pInfo->pState, pWin->groupId, &tbname);
+ pInfo->statestore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname);
if (tbname == NULL) {
appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);
} else {
@@ -1482,7 +1484,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);
}
- streamFreeVal(tbname);
+ pInfo->statestore.streamStateFreeVal(tbname);
(*index)++;
}
}
@@ -1536,7 +1538,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
blockDataDestroy(pInfo->pPullDataRes);
taosArrayDestroy(pInfo->pDelWins);
blockDataDestroy(pInfo->pDelRes);
- streamFileStateDestroy(pInfo->pState->pFileState);
+ pInfo->statestore.streamFileStateDestroy(pInfo->pState->pFileState);
taosMemoryFreeClear(pInfo->pState);
nodesDestroyNode((SNode*)pInfo->pPhyNode);
@@ -1611,16 +1613,20 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
}
void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo) {
+ SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;
+
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
initIntervalDownStream(downstream->pDownstream[0], type, pInfo);
return;
}
+
SStreamScanInfo* pScanInfo = downstream->info;
pScanInfo->windowSup.parentType = type;
pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;
if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) {
- pScanInfo->pUpdateInfo = updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark);
+ pScanInfo->pUpdateInfo = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark);
}
+
pScanInfo->interval = pInfo->interval;
pScanInfo->twAggSup = pInfo->twAggSup;
pScanInfo->pState = pInfo->pState;
@@ -1653,7 +1659,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
int32_t code =
- initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState);
+ initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -1681,7 +1687,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
if (pPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar);
- code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+ code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -1881,7 +1887,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
if (pStateNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr);
- int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
+ int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -1907,7 +1913,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
+ pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -1976,7 +1982,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
initBasicInfo(&pInfo->binfo, pResBlock);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
+ pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -2048,33 +2054,33 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3
}
}
-bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return streamStateCheck(pState, pKey); }
+bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { return pStore->streamStateCheck(pState, pKey); }
-int32_t setIntervalOutputBuf(SStreamState* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,
+int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,
SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
- SAggSupporter* pAggSup) {
- SWinKey key = {
- .ts = win->skey,
- .groupId = groupId,
- };
+ SAggSupporter* pAggSup, SStateStore* pStore) {
+
+ SWinKey key = { .ts = win->skey, .groupId = groupId };
char* value = NULL;
int32_t size = pAggSup->resultRowSize;
- if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) {
+ if (pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) {
return TSDB_CODE_OUT_OF_MEMORY;
}
+
*pResult = (SRowBuffPos*)value;
SResultRow* res = (SResultRow*)((*pResult)->pRowBuff);
+
// set time window for current result
res->win = (*win);
setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset);
return TSDB_CODE_SUCCESS;
}
-bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) {
+bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore) {
if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {
SWinKey key = {.ts = pWin->skey, .groupId = groupId};
- if (!hasIntervalWindow(pState, &key)) {
+ if (!hasIntervalWindow(pState, &key, pStore)) {
return true;
}
return false;
@@ -2105,7 +2111,7 @@ static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {
clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
initResultRowInfo(&pInfo->binfo.resultRowInfo);
pInfo->aggSup.currentPageId = -1;
- streamStateClear(pInfo->pState);
+ pInfo->statestore.streamStateClear(pInfo->pState);
}
static void clearSpecialDataBlock(SSDataBlock* pBlock) {
@@ -2201,13 +2207,15 @@ static void clearFunctionContext(SExprSupp* pSup) {
}
}
-int32_t getOutputBuf(SStreamState* pState, SRowBuffPos* pPos, SResultRow** pResult) {
- return streamStateGetByPos(pState, pPos, (void**)pResult);
+int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) {
+ return pStore->streamStateGetByPos(pState, pPos, (void**)pResult);
}
-int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
+int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
SExprInfo* pExprInfo = pSup->pExprInfo;
int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
@@ -2218,7 +2226,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {
SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i);
SResultRow* pRow = NULL;
- int32_t code = getOutputBuf(pState, pPos, &pRow);
+ int32_t code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore);
uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId;
ASSERT(code == 0);
doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
@@ -2230,12 +2238,12 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = groupId;
void* tbname = NULL;
- if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
+ if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
}
- streamFreeVal(tbname);
+ pAPI->stateStore.streamStateFreeVal(tbname);
} else {
// current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.id.groupId != groupId) {
@@ -2282,7 +2290,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
return TSDB_CODE_SUCCESS;
}
-void doBuildStreamIntervalResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock,
+void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
SGroupResInfo* pGroupResInfo) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// set output datablock version
@@ -2357,7 +2365,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
.groupId = groupId,
};
void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
- if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && isClosed && !chIds) {
+ if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->statestore) && isClosed && !chIds) {
SPullWindowInfo pull = {
.window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
// add pull data request
@@ -2388,7 +2396,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
}
int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,
- pSup->rowEntryInfoOffset, &pInfo->aggSup);
+ pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->statestore);
pResult = (SResultRow*)pResPos->pRowBuff;
if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
@@ -2470,6 +2478,8 @@ static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) {
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
SOperatorInfo* downstream = pOperator->pDownstream[0];
SExprSupp* pSup = &pOperator->exprSupp;
@@ -2508,8 +2518,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
} else {
if (pInfo->twAggSup.maxTs > 0 &&
pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) {
- streamStateCommit(pInfo->pState);
- streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
+ pAPI->stateStore.streamStateCommit(pInfo->pState);
+ pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs;
}
qDebug("===stream===interval final close");
@@ -2710,6 +2720,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
}
pOperator->pTaskInfo = pTaskInfo;
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
+
pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
.sliding = pIntervalPhyNode->sliding,
.intervalUnit = pIntervalPhyNode->intervalUnit,
@@ -2735,7 +2747,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
if (pIntervalPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
- int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+ int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -2748,18 +2760,18 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
*(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
- streamStateSetNumber(pInfo->pState, -1);
+
+ pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
- pInfo->pState);
+ pInfo->pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs);
-
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
-
initResultRowInfo(&pInfo->binfo.resultRowInfo);
+
pInfo->numOfChild = numOfChild;
pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
@@ -2776,6 +2788,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
}
+
pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
pInfo->pullIndex = 0;
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
@@ -2792,9 +2805,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pInfo->pUpdated = NULL;
pInfo->pUpdatedMap = NULL;
int32_t funResSize= getMaxFunResSize(&pOperator->exprSupp, numOfCols);
- pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
+ pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
compareTs, pInfo->pState, pInfo->twAggSup.deleteMark);
pInfo->dataVersion = 0;
+ pInfo->statestore = pTaskInfo->storageAPI.stateStore;
pOperator->operatorType = pPhyNode->type;
pOperator->blocking = true;
@@ -2851,9 +2865,9 @@ void destroyStreamSessionAggOperatorInfo(void* param) {
}
int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,
- SSDataBlock* pResultBlock) {
+ SSDataBlock* pResultBlock, SFunctionStateStore* pStore) {
initBasicInfo(pBasicInfo, pResultBlock);
- int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
+ int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@@ -2890,13 +2904,13 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin
pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
pScanInfo->pState = pAggSup->pState;
if ((!pScanInfo->igCheckUpdate || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) && !pScanInfo->pUpdateInfo) {
- pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark);
+ pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark);
}
pScanInfo->twAggSup = *pTwSup;
}
int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap,
- SStreamState* pState, int32_t keySize, int16_t keyType) {
+ SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore) {
pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput);
pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR);
pSup->gap = gap;
@@ -2907,10 +2921,12 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx,
return TSDB_CODE_OUT_OF_MEMORY;
}
+ pSup->stateStore = *pStore;
+
initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput);
pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState));
*(pSup->pState) = *pState;
- streamStateSetNumber(pSup->pState, -1);
+ pSup->stateStore.streamStateSetNumber(pSup->pState, -1);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pSup->pResultRows = tSimpleHashInit(32, hashFn);
@@ -2924,11 +2940,13 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx,
if (bufSize <= pageSize) {
bufSize = pageSize * 4;
}
+
if (!osTempSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE;
qError("Init stream agg supporter failed since %s, tempDir:%s", terrstr(), tsTempDir);
return terrno;
}
+
int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir);
for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].saveHandle.pBuf = pSup->pResultBuf;
@@ -2953,7 +2971,7 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT
pKey->win.skey = startTs;
pKey->win.ekey = endTs;
pKey->groupId = groupId;
- int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);
+ int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);
if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pKey);
}
@@ -2967,8 +2985,8 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT
pCurWin->sessionWin.win.skey = startTs;
pCurWin->sessionWin.win.ekey = endTs;
int32_t size = pAggSup->resultRowSize;
- int32_t code =
- streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size);
+ int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin,
+ pAggSup->gap, &pCurWin->pOutputBuf, &size);
if (code == TSDB_CODE_SUCCESS) {
pCurWin->isOutput = true;
} else {
@@ -2979,11 +2997,12 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT
int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) {
int32_t size = 0;
- int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size);
+ int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
- streamStateCurNext(pAggSup->pState, pCur);
+
+ pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);
return TSDB_CODE_SUCCESS;
}
void saveDeleteInfo(SArray* pWins, SSessionKey key) {
@@ -3068,7 +3087,7 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo
}
static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {
- streamStateSessionDel(pAggSup->pState, pKey);
+ pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey);
SSessionKey hashKey = {0};
getSessionHashKey(pKey, &hashKey);
tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey));
@@ -3086,12 +3105,12 @@ static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo*
SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
SResultWindowInfo* pNextWin) {
- SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);
+ SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);
pNextWin->isOutput = true;
setSessionWinOutputInfo(pStUpdated, pNextWin);
int32_t size = 0;
pNextWin->sessionWin = pCurWin->sessionWin;
- int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size);
+ int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pNextWin->pOutputBuf);
SET_SESSION_WIN_INVALID(*pNextWin);
@@ -3103,6 +3122,8 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC
SSHashObj* pStDeleted) {
SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
@@ -3114,7 +3135,7 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC
SStreamStateCur* pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);
if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) {
taosMemoryFree(winInfo.pOutputBuf);
- streamStateFreeCur(pCur);
+ pAPI->stateStore.streamStateFreeCur(pCur);
break;
}
SResultRow* pWinResult = NULL;
@@ -3128,13 +3149,13 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC
}
removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin);
doDeleteSessionWindow(pAggSup, &winInfo.sessionWin);
- streamStateFreeCur(pCur);
+ pAPI->stateStore.streamStateFreeCur(pCur);
taosMemoryFree(winInfo.pOutputBuf);
}
}
int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) {
- saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize);
+ saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize, &pAggSup->stateStore);
return TSDB_CODE_SUCCESS;
}
@@ -3254,6 +3275,8 @@ static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) {
}
void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
+ SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;
+
blockDataCleanup(pBlock);
int32_t size = tSimpleHashGetSize(pStDeleted);
if (size == 0) {
@@ -3282,14 +3305,14 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo
SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
void* tbname = NULL;
- streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname);
+ pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname);
if (tbname == NULL) {
colDataSetNULL(pTableCol, pBlock->info.rows);
} else {
char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
- streamFreeVal(tbname);
+ pAPI->stateStore.streamStateFreeVal(tbname);
}
pBlock->info.rows += 1;
}
@@ -3301,6 +3324,8 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo
static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) {
SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
int32_t size = taosArrayGetSize(pWinArray);
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
@@ -3317,7 +3342,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS
SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup;
SSessionKey chWinKey = {0};
getSessionHashKey(pWinKey, &chWinKey);
- SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);
+ SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);
SResultRow* pResult = NULL;
SResultRow* pChResult = NULL;
while (1) {
@@ -3344,7 +3369,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS
break;
}
}
- streamStateFreeCur(pCur);
+ pAPI->stateStore.streamStateFreeCur(pCur);
}
if (num > 0) {
saveSessionOutputBuf(pAggSup, &parentWin);
@@ -3409,7 +3434,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL
pGroupResInfo->pBuf = NULL;
}
-void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo,
+void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo,
SSDataBlock* pBlock) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// set output datablock version
@@ -3568,7 +3593,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
if (pSessionNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar);
- code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+ code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -3577,13 +3602,13 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
- code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
+ code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap,
- pTaskInfo->streamInfo.pState, 0, 0);
+ pTaskInfo->streamInfo.pState, 0, 0, &pTaskInfo->storageAPI.stateStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -3639,7 +3664,7 @@ _error:
static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
tSimpleHashClear(pInfo->streamAggSup.pResultRows);
- streamStateSessionClear(pInfo->streamAggSup.pState);
+ pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState);
}
static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
@@ -3765,6 +3790,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
goto _error;
}
+ SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION);
@@ -3789,7 +3815,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
}
SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
- streamStateSetNumber(pChInfo->streamAggSup.pState, i);
+ pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i);
taosArrayPush(pInfo->pChildren, &pChildOp);
}
}
@@ -3852,7 +3878,7 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
pCurWin->winInfo.sessionWin.win.skey = ts;
pCurWin->winInfo.sessionWin.win.ekey = ts;
int32_t code =
- streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize,
+ pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize,
compareStateKey, &pCurWin->winInfo.pOutputBuf, &size);
pCurWin->pStateKey =
(SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize));
@@ -3873,12 +3899,12 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
pNextWin->winInfo.pOutputBuf = NULL;
- SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
- code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0);
+ SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
+ code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_INVALID(pNextWin->winInfo);
}
- streamStateFreeCur(pCur);
+ pAggSup->stateStore.streamStateFreeCur(pCur);
}
int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId,
@@ -3917,7 +3943,9 @@ int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNex
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,
SSHashObj* pStDeleted) {
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
SStreamStateAggOperatorInfo* pInfo = pOperator->info;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
uint64_t groupId = pSDataBlock->info.id.groupId;
@@ -3959,7 +3987,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);
- releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf);
+ releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf, &pAPI->stateStore);
continue;
}
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
@@ -4103,7 +4131,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
if (pStateNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
- code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+ code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -4122,14 +4150,14 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
- code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock);
+ code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes;
int16_t type = pColNode->node.resType.type;
code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize,
- type);
+ type, &pTaskInfo->storageAPI.stateStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -4410,7 +4438,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
+ pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -4696,7 +4724,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
initResultSizeInfo(&pOperator->resultInfo, 4096);
int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
- pTaskInfo->streamInfo.pState);
+ pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -4739,7 +4767,9 @@ _error:
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
- SExprSupp* pSup = &pOperator->exprSupp;
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
+ SExprSupp* pSup = &pOperator->exprSupp;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
@@ -4760,8 +4790,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator);
if (pInfo->twAggSup.maxTs > 0 &&
pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) {
- streamStateCommit(pInfo->pState);
- streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
+ pAPI->stateStore.streamStateCommit(pInfo->pState);
+ pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId);
pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs;
}
@@ -4903,6 +4933,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
pOperator->pTaskInfo = pTaskInfo;
+ SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
pInfo->ignoreExpiredDataSaved = false;
pInfo->isFinal = false;
@@ -4917,11 +4949,11 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));
*(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
- streamStateSetNumber(pInfo->pState, -1);
+ pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
- pInfo->pState);
+ pInfo->pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -4929,7 +4961,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
if (pIntervalPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
- code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+ code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@@ -4955,7 +4987,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->pUpdated = NULL;
pInfo->pUpdatedMap = NULL;
int32_t funResSize= getMaxFunResSize(pSup, numOfCols);
- pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
+ pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
compareTs, pInfo->pState, pInfo->twAggSup.deleteMark);
setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,
@@ -4963,6 +4995,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL,
destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL);
+ pInfo->statestore = pTaskInfo->storageAPI.stateStore;
initIntervalDownStream(downstream, pPhyNode->type, pInfo);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt
index 3e1e84bdc5..3a68648d49 100644
--- a/source/libs/function/CMakeLists.txt
+++ b/source/libs/function/CMakeLists.txt
@@ -34,7 +34,6 @@ target_link_libraries(
PRIVATE scalar
PRIVATE geometry
PRIVATE transport
- PRIVATE stream ${LINK_JEMALLOC}
PUBLIC uv_a
)
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 1a43802e6b..5c47daca5b 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -18,7 +18,6 @@
#include "function.h"
#include "query.h"
#include "querynodes.h"
-#include "streamState.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tdigest.h"
@@ -1697,7 +1696,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
GET_TYPED_DATA(v, double, pVal->nType, &pVal->i);
- int32_t code = getPercentile(pMemBucket, v, &ppInfo->result);
+ code = getPercentile(pMemBucket, v, &ppInfo->result);
if (code != TSDB_CODE_SUCCESS) {
goto _fin_error;
}
@@ -3120,7 +3119,7 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid
}
static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, SWinKey* key,
- STuplePos* pPos) {
+ STuplePos* pPos, SFunctionStateStore* pStore) {
STuplePos p = {0};
if (pHandle->pBuf != NULL) {
SFilePage* pPage = NULL;
@@ -3153,9 +3152,8 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf,
pPage->num += length;
setBufPageDirty(pPage, true);
releaseBufPage(pHandle->pBuf, pPage);
- } else {
- // other tuple save policy
- if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) {
+ } else { // other tuple save policy
+ if (pStore->streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) {
p.streamTupleKey = *key;
}
}
@@ -3179,10 +3177,10 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock*
}
char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf);
- return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos);
+ return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos, pCtx->pStore);
}
-static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) {
+static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos, SFunctionStateStore* pStore) {
if (pHandle->pBuf != NULL) {
SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId);
if (pPage == NULL) {
@@ -3192,7 +3190,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf
setBufPageDirty(pPage, true);
releaseBufPage(pHandle->pBuf, pPage);
} else {
- streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length);
+ pStore->streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length);
}
return TSDB_CODE_SUCCESS;
@@ -3202,10 +3200,10 @@ int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc
prepareBuf(pCtx);
char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf);
- return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos);
+ return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos, pCtx->pStore);
}
-static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos) {
+static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos, SFunctionStateStore* pStore) {
if (pHandle->pBuf != NULL) {
SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId);
if (pPage == NULL) {
@@ -3217,13 +3215,13 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo
} else {
void* value = NULL;
int32_t vLen;
- streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen);
+ pStore->streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen);
return (char*)value;
}
}
const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) {
- return doLoadTupleData(&pCtx->saveHandle, pPos);
+ return doLoadTupleData(&pCtx->saveHandle, pPos, pCtx->pStore);
}
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
@@ -4991,7 +4989,7 @@ static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pI
memcpy(pInfo->buf, data, pInfo->colBytes);
}
- return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos);
+ return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos, pCtx->pStore);
}
static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) {
@@ -5020,7 +5018,7 @@ static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCt
} else {
pHashItem->count += 1;
if (pCtx->subsidiaries.num > 0) {
- int32_t code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos);
+ code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt
index 0c2ce37c40..6f3f48610c 100644
--- a/source/libs/index/CMakeLists.txt
+++ b/source/libs/index/CMakeLists.txt
@@ -12,7 +12,6 @@ target_link_libraries(
PUBLIC os
PUBLIC util
PUBLIC common
- PUBLIC vnode
PUBLIC nodes
PUBLIC scalar
PUBLIC function
diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c
index 02ed0d2d05..2c12c84081 100644
--- a/source/libs/index/src/indexFilter.c
+++ b/source/libs/index/src/indexFilter.c
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
+#include "filter.h"
#include "index.h"
#include "indexComm.h"
#include "indexInt.h"
@@ -20,7 +21,6 @@
#include "querynodes.h"
#include "scalar.h"
#include "tdatablock.h"
-#include "vnode.h"
// clang-format off
#define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
@@ -67,9 +67,8 @@ typedef union {
typedef struct SIFParam {
SHashObj *pFilter;
-
- SArray *result;
- char *condValue;
+ SArray *result;
+ char *condValue;
SIdxFltStatus status;
uint8_t colValType;
@@ -79,6 +78,7 @@ typedef struct SIFParam {
char colName[TSDB_COL_NAME_LEN * 2 + 4];
SIndexMetaArg arg;
+ SMetaDataFilterAPI api;
} SIFParam;
typedef struct SIFCtx {
@@ -86,6 +86,7 @@ typedef struct SIFCtx {
SHashObj *pRes; /* element is SIFParam */
bool noExec; // true: just iterate condition tree, and add hint to executor plan
SIndexMetaArg arg;
+ SMetaDataFilterAPI *pAPI;
} SIFCtx;
static FORCE_INLINE int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
@@ -288,6 +289,8 @@ static int32_t sifInitParamValByCol(SNode *r, SNode *l, SIFParam *param, SIFCtx
}
static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
param->status = SFLT_COARSE_INDEX;
+ param->api = *ctx->pAPI;
+
switch (nodeType(node)) {
case QUERY_NODE_VALUE: {
SValueNode *vn = (SValueNode *)node;
@@ -364,6 +367,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
}
+
SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam));
if (NULL == paramList) {
@@ -659,7 +663,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
} else {
if (sifSetFltParam(left, right, &typedata, ¶m) != 0) return -1;
}
- ret = metaFilterTableIds(arg->metaEx, ¶m, output->result);
+ ret = left->api.metaFilterTableIds(arg->metaEx, ¶m, output->result);
}
return ret;
}
@@ -972,8 +976,9 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
if (pNode == NULL || pDst == NULL) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
+
int32_t code = 0;
- SIFCtx ctx = {.code = 0, .noExec = false, .arg = pDst->arg};
+ SIFCtx ctx = {.code = 0, .noExec = false, .arg = pDst->arg, .pAPI = &pDst->api};
ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (NULL == ctx.pRes) {
@@ -1006,13 +1011,13 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
return code;
}
-static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) {
+static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) {
int32_t code = TSDB_CODE_SUCCESS;
if (pNode == NULL) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
- SIFCtx ctx = {.code = 0, .noExec = true};
+ SIFCtx ctx = {.code = 0, .noExec = true, .pAPI = pAPI};
ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (NULL == ctx.pRes) {
indexError("index-filter failed to taosHashInit");
@@ -1044,8 +1049,8 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) {
return code;
}
-int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status) {
- SIdxFltStatus st = idxGetFltStatus(pFilterNode);
+int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SIdxFltStatus *status, SMetaDataFilterAPI* pAPI) {
+ SIdxFltStatus st = idxGetFltStatus(pFilterNode, pAPI);
if (st == SFLT_NOT_INDEX) {
*status = st;
return 0;
@@ -1054,7 +1059,7 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result,
SFilterInfo *filter = NULL;
SArray *output = taosArrayInit(8, sizeof(uint64_t));
- SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX};
+ SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX, .api = *pAPI};
int32_t code = sifCalculate((SNode *)pFilterNode, ¶m);
if (code != 0) {
sifFreeParam(¶m);
@@ -1071,13 +1076,13 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result,
return TSDB_CODE_SUCCESS;
}
-SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) {
+SIdxFltStatus idxGetFltStatus(SNode *pFilterNode, SMetaDataFilterAPI* pAPI) {
SIdxFltStatus st = SFLT_NOT_INDEX;
if (pFilterNode == NULL) {
return SFLT_NOT_INDEX;
}
- if (sifGetFltHint((SNode *)pFilterNode, &st) != TSDB_CODE_SUCCESS) {
+ if (sifGetFltHint((SNode *)pFilterNode, &st, pAPI) != TSDB_CODE_SUCCESS) {
st = SFLT_NOT_INDEX;
}
return st;
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 864513f15d..39dab556f3 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -8008,6 +8008,9 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
isJson = true;
code = buildJsonTagVal(pCxt, pTagSchema, pVal, pTagArray, ppTag);
+ if (TSDB_CODE_SUCCESS != code) {
+ nodesDestroyNode((SNode*)pVal);
+ }
taosArrayPush(tagName, pTagSchema->name);
} else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) {
char* tmpVal = nodesGetValueFromNode(pVal);
@@ -8328,13 +8331,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
SArray* pTagVals = taosArrayInit(1, sizeof(STagVal));
int32_t code = TSDB_CODE_SUCCESS;
STag* pTag = NULL;
- do {
- code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf);
- if (TSDB_CODE_SUCCESS != code) {
- break;
- }
- } while (0);
-
+ code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf);
taosArrayDestroy(pTagVals);
if (code != TSDB_CODE_SUCCESS) {
return code;
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 5597bd3df8..8fa92a9d2e 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -416,7 +416,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi
end:
taosHashCleanup(keyHash);
if (retCode == TSDB_CODE_SUCCESS) {
- tTagNew(pTagVals, 1, true, ppTag);
+ retCode = tTagNew(pTagVals, 1, true, ppTag);
}
for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) {
STagVal* p = (STagVal*)taosArrayGet(pTagVals, i);
diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt
index 92ccde3163..8ba8b79ab8 100644
--- a/source/libs/qworker/CMakeLists.txt
+++ b/source/libs/qworker/CMakeLists.txt
@@ -7,9 +7,15 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
-target_link_libraries(qworker
- PRIVATE os util transport nodes planner qcom executor
- )
+IF (TD_GRANT)
+ TARGET_LINK_LIBRARIES(qworker
+ PRIVATE os util transport nodes planner qcom executor index grant
+ )
+ELSE ()
+ TARGET_LINK_LIBRARIES(qworker
+ PRIVATE os util transport nodes planner qcom executor index
+ )
+ENDIF()
if(${BUILD_TEST})
ADD_SUBDIRECTORY(test)
diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c
index 1a3a740b34..231e597724 100644
--- a/source/libs/qworker/src/qwMsg.c
+++ b/source/libs/qworker/src/qwMsg.c
@@ -440,11 +440,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
int64_t rId = msg.refId;
int32_t eId = msg.execId;
- SQWMsg qwMsg = {.node = node,
- .msg = msg.msg,
- .msgLen = msg.msgLen,
- .connInfo = pMsg->info,
- .msgType = pMsg->msgType};
+ SQWMsg qwMsg = {.node = node, .msg = msg.msg, .msgLen = msg.msgLen, .connInfo = pMsg->info, .msgType = pMsg->msgType};
qwMsg.msgInfo.explain = msg.explain;
qwMsg.msgInfo.taskType = msg.taskType;
qwMsg.msgInfo.needFetch = msg.needFetch;
diff --git a/source/libs/qworker/test/CMakeLists.txt b/source/libs/qworker/test/CMakeLists.txt
index 780f5ae84b..22870ea94d 100644
--- a/source/libs/qworker/test/CMakeLists.txt
+++ b/source/libs/qworker/test/CMakeLists.txt
@@ -8,7 +8,7 @@ IF(NOT TD_DARWIN)
ADD_EXECUTABLE(qworkerTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
qworkerTest
- PUBLIC os util common transport gtest qcom nodes planner qworker executor
+ PUBLIC os util common transport gtest qcom nodes planner qworker executor index
)
TARGET_INCLUDE_DIRECTORIES(
diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt
index 193a6971e5..30c68cb512 100644
--- a/source/libs/scalar/CMakeLists.txt
+++ b/source/libs/scalar/CMakeLists.txt
@@ -14,6 +14,7 @@ target_link_libraries(scalar
PRIVATE nodes
PRIVATE function
PRIVATE qcom
+ PRIVATE parser
)
if(${BUILD_TEST})
diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt
index 3288120b67..fafc2a27e0 100644
--- a/source/libs/scheduler/CMakeLists.txt
+++ b/source/libs/scheduler/CMakeLists.txt
@@ -9,7 +9,7 @@ target_include_directories(
target_link_libraries(
scheduler
- PUBLIC os util nodes planner qcom common catalog transport command qworker executor
+ PUBLIC os util nodes planner qcom common catalog transport command qworker executor index
)
if(${BUILD_TEST})
diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt
index 2edbc44aae..89b82ef9ba 100644
--- a/source/libs/stream/CMakeLists.txt
+++ b/source/libs/stream/CMakeLists.txt
@@ -10,7 +10,7 @@ if(${BUILD_WITH_ROCKSDB})
target_link_libraries(
stream
PUBLIC rocksdb tdb
- PRIVATE os util transport qcom executor wal
+ PRIVATE os util transport qcom executor wal index
)
target_include_directories(
diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h
index 0f39cf817b..1cbd7b042c 100644
--- a/source/libs/stream/inc/streamBackendRocksdb.h
+++ b/source/libs/stream/inc/streamBackendRocksdb.h
@@ -17,7 +17,6 @@
#define _STREAM_BACKEDN_ROCKSDB_H_
#include "rocksdb/c.h"
-// #include "streamInc.h"
#include "streamState.h"
#include "tcoding.h"
#include "tcommon.h"
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index c9538f0b2a..326c081a5b 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -767,10 +767,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
if (ppInst != NULL && *ppInst != NULL) {
RocksdbCfInst* inst = *ppInst;
pState->pTdbState->rocksdb = inst->db;
- pState->pTdbState->pHandle = inst->pHandle;
+ pState->pTdbState->pHandle = (void**)inst->pHandle;
pState->pTdbState->writeOpts = inst->wOpt;
pState->pTdbState->readOpts = inst->rOpt;
- pState->pTdbState->cfOpts = inst->cfOpt;
+ pState->pTdbState->cfOpts = (void**)(inst->cfOpt);
pState->pTdbState->dbOpt = handle->dbOpt;
pState->pTdbState->param = inst->param;
pState->pTdbState->pBackend = handle;
@@ -809,10 +809,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
}
rocksdb_column_family_handle_t** cfHandle = taosMemoryMalloc(cfLen * sizeof(rocksdb_column_family_handle_t*));
pState->pTdbState->rocksdb = handle->db;
- pState->pTdbState->pHandle = cfHandle;
+ pState->pTdbState->pHandle = (void**)cfHandle;
pState->pTdbState->writeOpts = rocksdb_writeoptions_create();
pState->pTdbState->readOpts = rocksdb_readoptions_create();
- pState->pTdbState->cfOpts = (rocksdb_options_t**)cfOpt;
+ pState->pTdbState->cfOpts = (void**)cfOpt;
pState->pTdbState->dbOpt = handle->dbOpt;
pState->pTdbState->param = param;
pState->pTdbState->pBackend = handle;
@@ -1205,7 +1205,8 @@ SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWin
}
pCur->number = pState->number;
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "state", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
SStateKey sKey = {.key = *key, .opNum = pState->number};
char buf[128] = {0};
@@ -1245,7 +1246,8 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) return NULL;
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "state", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
rocksdb_iter_seek(pCur->iter, buf, (size_t)klen);
rocksdb_iter_prev(pCur->iter);
@@ -1267,7 +1269,8 @@ SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey*
if (pCur == NULL) return NULL;
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "state", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
SStateKey sKey = {.key = *key, .opNum = pState->number};
char buf[128] = {0};
@@ -1359,7 +1362,8 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta
}
pCur->number = pState->number;
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
char buf[128] = {0};
SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
@@ -1398,7 +1402,8 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta
return NULL;
}
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
pCur->number = pState->number;
char buf[128] = {0};
@@ -1434,7 +1439,8 @@ SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, con
return NULL;
}
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
pCur->number = pState->number;
SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
@@ -1526,7 +1532,8 @@ SStreamStateCur* streamStateFillGetCur_rocksdb(SStreamState* pState, const SWinK
if (pCur == NULL) return NULL;
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "fill", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
char buf[128] = {0};
int len = winKeyEncode((void*)key, buf);
@@ -1585,7 +1592,8 @@ SStreamStateCur* streamStateFillSeekKeyNext_rocksdb(SStreamState* pState, const
}
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "fill", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
char buf[128] = {0};
int len = winKeyEncode((void*)key, buf);
@@ -1620,7 +1628,8 @@ SStreamStateCur* streamStateFillSeekKeyPrev_rocksdb(SStreamState* pState, const
}
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "fill", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "fill", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
char buf[128] = {0};
int len = winKeyEncode((void*)key, buf);
@@ -1655,7 +1664,8 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes
}
pCur->number = pState->number;
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "sess", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "sess", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
int32_t c = 0;
@@ -1925,7 +1935,8 @@ void* streamDefaultIterCreate_rocksdb(SStreamState* pState) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
pCur->db = pState->pTdbState->rocksdb;
- pCur->iter = streamStateIterCreate(pState, "default", &pCur->snapshot, &pCur->readOpt);
+ pCur->iter = streamStateIterCreate(pState, "default", (rocksdb_snapshot_t**)&pCur->snapshot,
+ (rocksdb_readoptions_t**)&pCur->readOpt);
return pCur;
}
int32_t streamDefaultIterValid_rocksdb(void* iter) {
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 55474541ed..95b97e080a 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -163,7 +163,6 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
int32_t code = 0;
ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
-
void* exec = pTask->exec.pExecutor;
qSetStreamOpOpen(exec);
@@ -328,7 +327,11 @@ int32_t streamExecForAll(SStreamTask* pTask) {
while (1) {
if (streamTaskShouldPause(&pTask->status)) {
- return 0;
+ if (batchSize > 1) {
+ break;
+ } else {
+ return 0;
+ }
}
SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue);
@@ -404,7 +407,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
{
// set input
- void* pExecutor = pTask->exec.pExecutor;
+ void* pExecutor = pTask->exec.pExecutor;
const SStreamQueueItem* pItem = pInput;
if (pItem->type == STREAM_INPUT__GET_RES) {
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 33375fe921..98e63f7f51 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -268,12 +268,14 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
if (ppTask) {
SStreamTask* pTask = *ppTask;
+
taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t));
tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn);
atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
-
int32_t num = taosArrayGetSize(pMeta->pTaskList);
+
+ qDebug("s-task:%s set the drop task flag, remain running s-task:%d", pTask->id.idStr, num - 1);
for (int32_t i = 0; i < num; ++i) {
int32_t* pTaskId = taosArrayGet(pMeta->pTaskList, i);
if (*pTaskId == taskId) {
@@ -283,6 +285,8 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
}
streamMetaReleaseTask(pMeta, pTask);
+ } else {
+ qDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId);
}
taosWUnLockLatch(&pMeta->lock);
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 7e2c62f73a..ba24a581e9 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -23,7 +23,6 @@
#include "tcommon.h"
#include "tcompare.h"
#include "tref.h"
-#include "ttimer.h"
#define MAX_TABLE_NAME_NUM 200000
@@ -91,13 +90,14 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) {
return winKeyCmprImpl(&pWin1->key, &pWin2->key);
}
-SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages) {
- qWarn("open stream state, %s", path);
+SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) {
+ qDebug("open stream state, %s", path);
SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
if (pState == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
+
pState->pTdbState = taosMemoryCalloc(1, sizeof(STdbState));
if (pState->pTdbState == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -105,29 +105,33 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
return NULL;
}
+ SStreamTask* pStreamTask = pTask;
char statePath[1024];
if (!specPath) {
- sprintf(statePath, "%s/%d", path, pTask->id.taskId);
+ sprintf(statePath, "%s/%d", path, pStreamTask->id.taskId);
} else {
memset(statePath, 0, 1024);
tstrncpy(statePath, path, 1024);
}
- pState->taskId = pTask->id.taskId;
- pState->streamId = pTask->id.streamId;
+
+ pState->taskId = pStreamTask->id.taskId;
+ pState->streamId = pStreamTask->id.streamId;
+
#ifdef USE_ROCKSDB
- // qWarn("open stream state1");
- taosAcquireRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid);
- int code = streamStateOpenBackend(pTask->pMeta->streamBackend, pState);
+ SStreamMeta* pMeta = pStreamTask->pMeta;
+ taosAcquireRef(pMeta->streamBackendId, pMeta->streamBackendRid);
+ int code = streamStateOpenBackend(pMeta->streamBackend, pState);
if (code == -1) {
- taosReleaseRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid);
+ taosReleaseRef(pMeta->streamBackendId, pMeta->streamBackendRid);
taosMemoryFree(pState);
pState = NULL;
}
+
pState->pTdbState->pOwner = pTask;
pState->pFileState = NULL;
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT);
- pState->parNameMap = tSimpleHashInit(1024, hashFn);
+ pState->parNameMap = tSimpleHashInit(1024, hashFn);
return pState;
#else
@@ -449,7 +453,7 @@ int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pV
#ifdef USE_ROCKSDB
taosMemoryFree(pVal);
#else
- streamFreeVal(pVal);
+ streamStateFreeVal(pVal);
#endif
return 0;
}
@@ -700,7 +704,7 @@ void streamStateFreeCur(SStreamStateCur* pCur) {
taosMemoryFree(pCur);
}
-void streamFreeVal(void* val) {
+void streamStateFreeVal(void* val) {
#ifdef USE_ROCKSDB
taosMemoryFree(val);
#else
diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c
index ad66bb5a27..f531f65565 100644
--- a/source/libs/stream/src/tstreamFileState.c
+++ b/source/libs/stream/src/tstreamFileState.c
@@ -21,6 +21,7 @@
#include "tcommon.h"
#include "thash.h"
#include "tsimplehash.h"
+#include "storageapi.h"
#define FLUSH_RATIO 0.5
#define FLUSH_NUM 4
diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt
index a0c1717690..049bfbbb3a 100644
--- a/source/libs/stream/test/CMakeLists.txt
+++ b/source/libs/stream/test/CMakeLists.txt
@@ -8,10 +8,20 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
# bloomFilterTest
ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp")
-TARGET_LINK_LIBRARIES(
- streamUpdateTest
- PUBLIC os util common gtest gtest_main stream
-)
+#TARGET_LINK_LIBRARIES(
+# streamUpdateTest
+# PUBLIC os util common gtest gtest_main stream executor
+#)
+
+IF (TD_GRANT)
+ TARGET_LINK_LIBRARIES(streamUpdateTest
+ PUBLIC os util common gtest gtest_main stream executor index grant
+ )
+ELSE ()
+ TARGET_LINK_LIBRARIES(streamUpdateTest
+ PUBLIC os util common gtest gtest_main stream executor index
+ )
+ENDIF()
TARGET_INCLUDE_DIRECTORIES(
streamUpdateTest
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 31727f7535..e28e67f83a 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -325,6 +325,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NOT_CATCH_UP, "Mnode didn't catch th
TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ALREADY_IS_VOTER, "Mnode already is a leader")
TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ONLY_TWO_MNODE, "Only two mnodes exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NO_NEED_RESTORE, "No need to restore on this dnode")
+TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE, "Please use this command when the dnode is offline")
// vnode
TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed")
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index e0d94c0c55..210f0b04e4 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -571,7 +571,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py
-#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py
@@ -1089,7 +1089,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4
-#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4
diff --git a/tests/script/sh/pycumsum.py b/tests/script/sh/pycumsum.py
new file mode 100644
index 0000000000..27d575aec4
--- /dev/null
+++ b/tests/script/sh/pycumsum.py
@@ -0,0 +1,29 @@
+import pickle
+import numpy as np
+
+def init():
+ pass
+
+def destroy():
+ pass
+
+def start():
+ return pickle.dumps(0.0)
+
+def finish(buf):
+ return pickle.loads(buf)
+
+def reduce(datablock, buf):
+ (rows, cols) = datablock.shape()
+ state = pickle.loads(buf)
+ row = []
+ for i in range(rows):
+ for j in range(cols):
+ cell = datablock.data(i, j)
+ if cell is not None:
+ row.append(datablock.data(i, j))
+ if len(row) > 1:
+ new_state = np.cumsum(row)[-1]
+ else:
+ new_state = state
+ return pickle.dumps(new_state)
diff --git a/tests/script/tsim/query/udfpy.sim b/tests/script/tsim/query/udfpy.sim
index 9e0492ffd9..2a4daedd5d 100644
--- a/tests/script/tsim/query/udfpy.sim
+++ b/tests/script/tsim/query/udfpy.sim
@@ -15,6 +15,7 @@ system sh/prepare_pyudf.sh
system mkdir -p /tmp/pyudf
system cp sh/pybitand.py /tmp/pyudf/
system cp sh/pyl2norm.py /tmp/pyudf/
+system cp sh/pycumsum.py /tmp/pyudf/
system ls /tmp/pyudf
sql create database udf vgroups 3;
@@ -280,6 +281,18 @@ if $data20 != 8.000000000 then
return -1
endi
+#sql create aggregate function pycumsum as '/tmp/pyudf/pycumsum.py' outputtype double bufSize 128 language 'python';
+#sql select pycumsum(f2) from udf.t2
+#print ======= pycumsum
+#print $rows $data00
+#if $rows != 1 then
+# return -1
+#endi
+#if $data00 != 20.000000000 then
+# return -1
+#endi
+#sql drop function pycumsum
+
sql create or replace function bit_and as '/tmp/udf/libbitand.so' outputtype int
sql select func_version from information_schema.ins_functions where name='bit_and'
if $data00 != 1 then
diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c
index ac5aff4727..94619339e9 100644
--- a/utils/test/c/sml_test.c
+++ b/utils/test/c/sml_test.c
@@ -1132,6 +1132,155 @@ int sml_td22900_Test() {
return code;
}
+int sml_td24070_Test() {
+ TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+
+ TAOS_RES *pRes = taos_query(taos, "CREATE user test_db pass 'test'");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS td24070_read");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "grant read on td24070_read to test_db");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS td24070_write");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "grant write on td24070_write to test_db");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ taos_close(taos);
+
+
+ // test db privilege
+ taos = taos_connect("localhost", "test_db", "test", NULL, 0);
+ const char* sql[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299372000"};
+
+ pRes = taos_query(taos, "use td24070_read");
+ taos_free_result(pRes);
+
+ pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL,
+ TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+
+ printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+ int code = taos_errno(pRes);
+ ASSERT(code != 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "use td24070_write");
+ taos_free_result(pRes);
+
+ pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL,
+ TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+
+ printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+ code = taos_errno(pRes);
+ ASSERT(code == 0);
+ taos_free_result(pRes);
+ taos_close(taos);
+ // test db privilege end
+
+
+ // test stable privilege
+ taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+
+ pRes = taos_query(taos, "CREATE user test_stb_read pass 'test'");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "CREATE user test_stb_write pass 'test'");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "grant read on td24070_write.stb2 to test_stb_read");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "grant write on td24070_write.stb2 to test_stb_write");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+ taos_close(taos);
+
+ taos = taos_connect("localhost", "test_stb_read", "test", "td24070_write", 0);
+ const char* sql1[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"};
+
+ pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL,
+ TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+
+ printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+ code = taos_errno(pRes);
+ ASSERT(code != 0);
+ taos_free_result(pRes);
+ taos_close(taos);
+
+ taos = taos_connect("localhost", "test_stb_write", "test", "td24070_write", 0);
+ const char* sql2[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299373000"};
+
+ pRes = taos_schemaless_insert(taos, (char **)sql2, sizeof(sql2) / sizeof(sql2[0]), TSDB_SML_LINE_PROTOCOL,
+ TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+
+ printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+ code = taos_errno(pRes);
+ ASSERT(code == 0);
+ taos_free_result(pRes);
+ taos_close(taos);
+ // test stable privilege
+
+ // test table privilege
+ taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+
+ pRes = taos_query(taos, "CREATE user test_tb_read pass 'test'");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "CREATE user test_tb_write pass 'test'");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "grant read on td24070_write.stb2 with t1=1 to test_tb_read");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "grant write on td24070_write.stb2 with t1=1 to test_tb_write");
+ ASSERT(taos_errno(pRes) == 0);
+ taos_free_result(pRes);
+ taos_close(taos);
+
+ taos = taos_connect("localhost", "test_tb_read", "test", "td24070_write", 0);
+ const char* sql3[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"};
+
+
+ pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_LINE_PROTOCOL,
+ TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+
+ printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+ code = taos_errno(pRes);
+ ASSERT(code != 0);
+ taos_free_result(pRes);
+ taos_close(taos);
+
+ taos = taos_connect("localhost", "test_tb_write", "test", "td24070_write", 0);
+ const char* sql4[] = {"stb2,t1=1,dataModelName=t0 f1=283i32 1632299374000"};
+
+ pRes = taos_schemaless_insert(taos, (char **)sql4, sizeof(sql4) / sizeof(sql4[0]), TSDB_SML_LINE_PROTOCOL,
+ TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+
+ printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+ code = taos_errno(pRes);
+ ASSERT(code == 0);
+ taos_free_result(pRes);
+ taos_close(taos);
+ // test table privilege
+
+ return code;
+}
+
int sml_td23881_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -1379,6 +1528,8 @@ int main(int argc, char *argv[]) {
}
int ret = 0;
+ ret = sml_td24070_Test();
+ ASSERT(!ret);
ret = sml_td23881_Test();
ASSERT(ret);
ret = sml_escape_Test();