diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in
index cc46ef9938..2a7b76d768 100644
--- a/cmake/taosadapter_CMakeLists.txt.in
+++ b/cmake/taosadapter_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
- GIT_TAG ff7de07
+ GIT_TAG e07f41b
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index ed2ec0b6da..db727907b7 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
- GIT_TAG d5df76d
+ GIT_TAG 823fae5
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx b/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx
new file mode 100644
index 0000000000..ffb969a8a6
--- /dev/null
+++ b/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx
@@ -0,0 +1,46 @@
+---
+title: Write from Kafka
+---
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+import PyKafka from "./_py_kafka.mdx";
+
+## About Kafka
+
+Apache Kafka is an open-source distributed event streaming platform, used by thousands of companies for high-performance data pipelines, streaming analytics, data integration, and mission-critical applications. For the key concepts of kafka, please refer to [kafka documentation](https://kafka.apache.org/documentation/#gettingStarted).
+
+### kafka topic
+
+Messages in Kafka are organized by topics. A topic may have one or more partitions. We can manage kafka topics through `kafka-topics`.
+
+create a topic named `kafka-events`:
+
+```
+bin/kafka-topics.sh --create --topic kafka-events --bootstrap-server localhost:9092
+```
+
+Alter `kafka-events` topic to set partitions to 3:
+
+```
+bin/kafka-topics.sh --alter --topic kafka-events --partitions 3 --bootstrap-server=localhost:9092
+```
+
+Show all topics and partitions in Kafka:
+
+```
+bin/kafka-topics.sh --bootstrap-server=localhost:9092 --describe
+```
+
+## Insert into TDengine
+
+We can write data into TDengine via SQL or Schemaless. For more information, please refer to [Insert Using SQL](/develop/insert-data/sql-writing/) or [High Performance Writing](/develop/insert-data/high-volume/) or [Schemaless Writing](/reference/schemaless/).
+
+## Examples
+
+
+
+
+
+
+
diff --git a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx
similarity index 100%
rename from docs/en/07-develop/03-insert-data/02-influxdb-line.mdx
rename to docs/en/07-develop/03-insert-data/30-influxdb-line.mdx
diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx
similarity index 100%
rename from docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx
rename to docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx
diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx
similarity index 100%
rename from docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx
rename to docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx
diff --git a/docs/en/07-develop/03-insert-data/05-high-volume.md b/docs/en/07-develop/03-insert-data/60-high-volume.md
similarity index 100%
rename from docs/en/07-develop/03-insert-data/05-high-volume.md
rename to docs/en/07-develop/03-insert-data/60-high-volume.md
diff --git a/docs/en/07-develop/03-insert-data/_py_kafka.mdx b/docs/en/07-develop/03-insert-data/_py_kafka.mdx
new file mode 100644
index 0000000000..dc43a0d415
--- /dev/null
+++ b/docs/en/07-develop/03-insert-data/_py_kafka.mdx
@@ -0,0 +1,60 @@
+### python Kafka 客户端
+
+For python kafka client, please refer to [kafka client](https://cwiki.apache.org/confluence/display/KAFKA/Clients#Clients-Python). In this document, we use [kafka-python](http://github.com/dpkp/kafka-python).
+
+### consume from Kafka
+
+The simple way to consume messages from Kafka is to read messages one by one. The demo is as follows:
+
+```
+from kafka import KafkaConsumer
+consumer = KafkaConsumer('my_favorite_topic')
+for msg in consumer:
+ print (msg)
+```
+
+For higher performance, we can consume message from kafka in batch. The demo is as follows:
+
+```
+from kafka import KafkaConsumer
+consumer = KafkaConsumer('my_favorite_topic')
+while True:
+ msgs = consumer.poll(timeout_ms=500, max_records=1000)
+ if msgs:
+ print (msgs)
+```
+
+### multi-threading
+
+For more higher performance we can process data from kafka in multi-thread. We can use python's ThreadPoolExecutor to achieve multithreading. The demo is as follows:
+
+```
+from concurrent.futures import ThreadPoolExecutor, Future
+pool = ThreadPoolExecutor(max_workers=10)
+pool.submit(...)
+```
+
+### multi-process
+
+For more higher performance, sometimes we use multiprocessing. In this case, the number of Kafka Consumers should not be greater than the number of Kafka Topic Partitions. The demo is as follows:
+
+```
+from multiprocessing import Process
+
+ps = []
+for i in range(5):
+ p = Process(target=Consumer().consume())
+ p.start()
+ ps.append(p)
+
+for p in ps:
+ p.join()
+```
+
+In addition to python's built-in multithreading and multiprocessing library, we can also use the third-party library gunicorn.
+
+### Examples
+
+```py
+{{#include docs/examples/python/kafka_example.py}}
+```
diff --git a/docs/en/20-third-party/01-grafana.mdx b/docs/en/20-third-party/01-grafana.mdx
index e0fbefd5a8..ca32ce8afc 100644
--- a/docs/en/20-third-party/01-grafana.mdx
+++ b/docs/en/20-third-party/01-grafana.mdx
@@ -76,7 +76,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource
You can also download zip files from [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) or [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) and install manually. The commands are as follows:
```bash
-GF_VERSION=3.2.2
+GF_VERSION=3.2.7
# from GitHub
wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip
# from Grafana
diff --git a/docs/en/25-application/01-telegraf.md b/docs/en/25-application/01-telegraf.md
index f700326449..65fb08ee67 100644
--- a/docs/en/25-application/01-telegraf.md
+++ b/docs/en/25-application/01-telegraf.md
@@ -38,15 +38,9 @@ Download the latest TDengine-server from the [Downloads](http://tdengine.com/en/
## Data Connection Setup
-### Download TDengine plug-in to grafana plug-in directory
+### Install Grafana Plugin and Configure Data Source
-```bash
-1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip
-2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/
-3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine
-4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini
-5. sudo systemctl restart grafana-server.service
-```
+Please refer to [Install Grafana Plugin and Configure Data Source](/third-party/grafana/#install-grafana-plugin-and-configure-data-source)
### Modify /etc/telegraf/telegraf.conf
diff --git a/docs/en/25-application/02-collectd.md b/docs/en/25-application/02-collectd.md
index 692cd8d929..97412b2309 100644
--- a/docs/en/25-application/02-collectd.md
+++ b/docs/en/25-application/02-collectd.md
@@ -41,15 +41,9 @@ Download the latest TDengine-server from the [Downloads](http://tdengine.com/en/
## Data Connection Setup
-### Copy the TDengine plugin to the grafana plugin directory
+### Install Grafana Plugin and Configure Data Source
-```bash
-1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip
-2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/
-3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine
-4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini
-5. sudo systemctl restart grafana-server.service
-```
+Please refer to [Install Grafana Plugin and Configure Data Source](/third-party/grafana/#install-grafana-plugin-and-configure-data-source)
### Configure collectd
diff --git a/docs/examples/python/kafka_example.py b/docs/examples/python/kafka_example.py
new file mode 100644
index 0000000000..735059eec0
--- /dev/null
+++ b/docs/examples/python/kafka_example.py
@@ -0,0 +1,192 @@
+#! encoding = utf-8
+import json
+import time
+from json import JSONDecodeError
+from typing import Callable
+import logging
+from concurrent.futures import ThreadPoolExecutor, Future
+
+import taos
+from kafka import KafkaConsumer
+from kafka.consumer.fetcher import ConsumerRecord
+
+
+class Consumer(object):
+ DEFAULT_CONFIGS = {
+ 'kafka_brokers': 'localhost:9092',
+ 'kafka_topic': 'python_kafka',
+ 'kafka_group_id': 'taos',
+ 'taos_host': 'localhost',
+ 'taos_user': 'root',
+ 'taos_password': 'taosdata',
+ 'taos_database': 'power',
+ 'taos_port': 6030,
+ 'timezone': None,
+ 'clean_after_testing': False,
+ 'bath_consume': True,
+ 'batch_size': 1000,
+ 'async_model': True,
+ 'workers': 10
+ }
+
+ LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose',
+ 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale',
+ 'California.SantaClara', 'California.Cupertino']
+
+ CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1'
+ USE_DATABASE_SQL = 'use {}'
+ DROP_TABLE_SQL = 'drop table if exists meters'
+ DROP_DATABASE_SQL = 'drop database if exists {}'
+ CREATE_STABLE_SQL = 'create stable meters (ts timestamp, current float, voltage int, phase float) ' \
+ 'tags (location binary(64), groupId int)'
+ CREATE_TABLE_SQL = 'create table if not exists {} using meters tags (\'{}\', {})'
+ INSERT_SQL_HEADER = "insert into "
+ INSERT_PART_SQL = 'power.{} values (\'{}\', {}, {}, {})'
+
+ def __init__(self, **configs):
+ self.config: dict = self.DEFAULT_CONFIGS
+ self.config.update(configs)
+ self.consumer = KafkaConsumer(
+ self.config.get('kafka_topic'), # topic
+ bootstrap_servers=self.config.get('kafka_brokers'),
+ group_id=self.config.get('kafka_group_id'),
+ )
+ self.taos = taos.connect(
+ host=self.config.get('taos_host'),
+ user=self.config.get('taos_user'),
+ password=self.config.get('taos_password'),
+ port=self.config.get('taos_port'),
+ timezone=self.config.get('timezone'),
+ )
+ if self.config.get('async_model'):
+ self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers'))
+ self.tasks: list[Future] = []
+ # tags and table mapping # key: {location}_{groupId} value:
+ self.tag_table_mapping = {}
+ i = 0
+ for location in self.LOCATIONS:
+ for j in range(1, 11):
+ table_name = 'd{}'.format(i)
+ self._cache_table(location=location, group_id=j, table_name=table_name)
+ i += 1
+
+ def init_env(self):
+ # create database and table
+ self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database')))
+ self.taos.execute(self.CREATE_DATABASE_SQL.format(self.config.get('taos_database')))
+ self.taos.execute(self.USE_DATABASE_SQL.format(self.config.get('taos_database')))
+ self.taos.execute(self.DROP_TABLE_SQL)
+ self.taos.execute(self.CREATE_STABLE_SQL)
+ for tags, table_name in self.tag_table_mapping.items():
+ location, group_id = _get_location_and_group(tags)
+ self.taos.execute(self.CREATE_TABLE_SQL.format(table_name, location, group_id))
+
+ def consume(self):
+ logging.warning('## start consumer topic-[%s]', self.config.get('kafka_topic'))
+ try:
+ if self.config.get('bath_consume'):
+ self._run_batch(self._to_taos_batch)
+ else:
+ self._run(self._to_taos)
+ except KeyboardInterrupt:
+ logging.warning("## caught keyboard interrupt, stopping")
+ finally:
+ self.stop()
+
+ def stop(self):
+ # close consumer
+ if self.consumer is not None:
+ self.consumer.commit()
+ self.consumer.close()
+
+ # multi thread
+ if self.config.get('async_model'):
+ for task in self.tasks:
+ while not task.done():
+ pass
+ if self.pool is not None:
+ self.pool.shutdown()
+
+ # clean data
+ if self.config.get('clean_after_testing'):
+ self.taos.execute(self.DROP_TABLE_SQL)
+ self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database')))
+ # close taos
+ if self.taos is not None:
+ self.taos.close()
+
+ def _run(self, f: Callable[[ConsumerRecord], bool]):
+ for message in self.consumer:
+ if self.config.get('async_model'):
+ self.pool.submit(f(message))
+ else:
+ f(message)
+
+ def _run_batch(self, f: Callable[[list[list[ConsumerRecord]]], None]):
+ while True:
+ messages = self.consumer.poll(timeout_ms=500, max_records=self.config.get('batch_size'))
+ if messages:
+ if self.config.get('async_model'):
+ self.pool.submit(f, messages.values())
+ else:
+ f(list(messages.values()))
+ if not messages:
+ time.sleep(0.1)
+
+ def _to_taos(self, message: ConsumerRecord) -> bool:
+ sql = self.INSERT_SQL_HEADER + self._build_sql(message.value)
+ if len(sql) == 0: # decode error, skip
+ return True
+ logging.info('## insert sql %s', sql)
+ return self.taos.execute(sql=sql) == 1
+
+ def _to_taos_batch(self, messages: list[list[ConsumerRecord]]):
+ sql = self._build_sql_batch(messages=messages)
+ if len(sql) == 0: # decode error, skip
+ return
+ self.taos.execute(sql=sql)
+
+ def _build_sql(self, msg_value: str) -> str:
+ try:
+ data = json.loads(msg_value)
+ except JSONDecodeError as e:
+ logging.error('## decode message [%s] error ', msg_value, e)
+ return ''
+ location = data.get('location')
+ group_id = data.get('groupId')
+ ts = data.get('ts')
+ current = data.get('current')
+ voltage = data.get('voltage')
+ phase = data.get('phase')
+
+ table_name = self._get_table_name(location=location, group_id=group_id)
+ return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase)
+
+ def _build_sql_batch(self, messages: list[list[ConsumerRecord]]) -> str:
+ sql_list = []
+ for partition_messages in messages:
+ for message in partition_messages:
+ sql_list.append(self._build_sql(message.value))
+
+ return self.INSERT_SQL_HEADER + ' '.join(sql_list)
+
+ def _cache_table(self, location: str, group_id: int, table_name: str):
+ self.tag_table_mapping[_tag_table_mapping_key(location=location, group_id=group_id)] = table_name
+
+ def _get_table_name(self, location: str, group_id: int) -> str:
+ return self.tag_table_mapping.get(_tag_table_mapping_key(location=location, group_id=group_id))
+
+
+def _tag_table_mapping_key(location: str, group_id: int):
+ return '{}_{}'.format(location, group_id)
+
+
+def _get_location_and_group(key: str) -> (str, int):
+ fields = key.split('_')
+ return fields[0], fields[1]
+
+
+if __name__ == '__main__':
+ consumer = Consumer(async_model=True)
+ consumer.init_env()
+ consumer.consume()
\ No newline at end of file
diff --git a/docs/zh/07-develop/03-insert-data/20-kafka-writting.mdx b/docs/zh/07-develop/03-insert-data/20-kafka-writting.mdx
new file mode 100644
index 0000000000..32d3c2e5cb
--- /dev/null
+++ b/docs/zh/07-develop/03-insert-data/20-kafka-writting.mdx
@@ -0,0 +1,47 @@
+---
+title: 从 Kafka 写入
+---
+
+import Tabs from "@theme/Tabs";
+import TabItem from "@theme/TabItem";
+import PyKafka from "./_py_kafka.mdx";
+
+## Kafka 介绍
+
+Apache Kafka 是开源的分布式消息分发平台，被广泛应用于高性能数据管道、流式数据分析、数据集成和事件驱动类型的应用程序。Kafka 包含 Producer、Consumer 和 Topic，其中 Producer 是向 Kafka 发送消息的进程，Consumer 是从 Kafka 消费消息的进程。Kafka 相关概念可以参考[官方文档](https://kafka.apache.org/documentation/#gettingStarted)。
+
+
+### kafka topic
+
+Kafka 的消息按 topic 组织，每个 topic 会有一到多个 partition。可以通过 kafka 的 `kafka-topics` 管理 topic。
+
+创建名为 `kafka-events` 的topic:
+
+```
+bin/kafka-topics.sh --create --topic kafka-events --bootstrap-server localhost:9092
+```
+
+修改 `kafka-events` 的 partition 数量为 3:
+
+```
+bin/kafka-topics.sh --alter --topic kafka-events --partitions 3 --bootstrap-server=localhost:9092
+```
+
+展示所有的 topic 和 partition:
+
+```
+bin/kafka-topics.sh --bootstrap-server=localhost:9092 --describe
+```
+
+## 写入 TDengine
+
+TDengine 支持 Sql 方式和 Schemaless 方式的数据写入，Sql 方式数据写入可以参考 [TDengine SQL 写入](/develop/insert-data/sql-writing/) 和 [TDengine 高效写入](/develop/insert-data/high-volume/)。Schemaless 方式数据写入可以参考 [TDengine Schemaless 写入](/reference/schemaless/) 文档。
+
+## 示例代码
+
+
+
+
+
+
+
diff --git a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx
similarity index 100%
rename from docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx
rename to docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx
diff --git a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx
similarity index 100%
rename from docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx
rename to docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx
diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx
similarity index 100%
rename from docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx
rename to docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx
diff --git a/docs/zh/07-develop/03-insert-data/05-high-volume.md b/docs/zh/07-develop/03-insert-data/60-high-volume.md
similarity index 100%
rename from docs/zh/07-develop/03-insert-data/05-high-volume.md
rename to docs/zh/07-develop/03-insert-data/60-high-volume.md
diff --git a/docs/zh/07-develop/03-insert-data/_py_kafka.mdx b/docs/zh/07-develop/03-insert-data/_py_kafka.mdx
new file mode 100644
index 0000000000..cd7edf557d
--- /dev/null
+++ b/docs/zh/07-develop/03-insert-data/_py_kafka.mdx
@@ -0,0 +1,60 @@
+### python Kafka 客户端
+
+Kafka 的 python 客户端可以参考文档 [kafka client](https://cwiki.apache.org/confluence/display/KAFKA/Clients#Clients-Python)。推荐使用 [confluent-kafka-python](https://github.com/confluentinc/confluent-kafka-python) 和 [kafka-python](http://github.com/dpkp/kafka-python)。以下示例以 [kafka-python](http://github.com/dpkp/kafka-python) 为例。
+
+### 从 Kafka 消费数据
+
+Kafka 客户端采用 pull 的方式从 Kafka 消费数据，可以采用单条消费的方式或批量消费的方式读取数据。使用 [kafka-python](http://github.com/dpkp/kafka-python) 客户端单条消费数据的示例如下：
+
+```
+from kafka import KafkaConsumer
+consumer = KafkaConsumer('my_favorite_topic')
+for msg in consumer:
+ print (msg)
+```
+
+单条消费的方式在数据流量大的情况下往往存在性能瓶颈，导致 Kafka 消息积压，更推荐使用批量消费的方式消费数据。使用 [kafka-python](http://github.com/dpkp/kafka-python) 客户端批量消费数据的示例如下：
+
+```
+from kafka import KafkaConsumer
+consumer = KafkaConsumer('my_favorite_topic')
+while True:
+ msgs = consumer.poll(timeout_ms=500, max_records=1000)
+ if msgs:
+ print (msgs)
+```
+
+### Python 多线程
+
+为了提高数据写入效率，通常采用多线程的方式写入数据，可以使用 python 线程池 ThreadPoolExecutor 实现多线程。示例代码如下：
+
+```
+from concurrent.futures import ThreadPoolExecutor, Future
+pool = ThreadPoolExecutor(max_workers=10)
+pool.submit(...)
+```
+
+### Python 多进程
+
+单个python进程不能充分发挥多核 CPU 的性能，有时候我们会选择多进程的方式。在多进程的情况下，需要注意，Kafka Consumer 的数量应该小于等于 Kafka Topic Partition 数量。Python 多进程示例代码如下：
+
+```
+from multiprocessing import Process
+
+ps = []
+for i in range(5):
+ p = Process(target=Consumer().consume())
+ p.start()
+ ps.append(p)
+
+for p in ps:
+ p.join()
+```
+
+除了 Python 内置的多线程和多进程方式，还可以通过第三方库 gunicorn 实现并发。
+
+### 完整示例
+
+```py
+{{#include docs/examples/python/kafka_example.py}}
+```
diff --git a/docs/zh/25-application/01-telegraf.md b/docs/zh/25-application/01-telegraf.md
index 6338264d17..ec263d7792 100644
--- a/docs/zh/25-application/01-telegraf.md
+++ b/docs/zh/25-application/01-telegraf.md
@@ -39,15 +39,9 @@ IT 运维监测数据通常都是对时间特性比较敏感的数据，例如
## 数据链路设置
-### 下载 TDengine 插件到 Grafana 插件目录
+### 安装 Grafana Plugin 并配置数据源
-```bash
-1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip
-2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/
-3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine
-4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini
-5. sudo systemctl restart grafana-server.service
-```
+请参考[安装 Grafana Plugin 并配置数据源](/third-party/grafana/#%E5%AE%89%E8%A3%85-grafana-plugin-%E5%B9%B6%E9%85%8D%E7%BD%AE%E6%95%B0%E6%8D%AE%E6%BA%90)。
### 修改 /etc/telegraf/telegraf.conf
diff --git a/docs/zh/25-application/02-collectd.md b/docs/zh/25-application/02-collectd.md
index c6230f48ab..8b39a6431d 100644
--- a/docs/zh/25-application/02-collectd.md
+++ b/docs/zh/25-application/02-collectd.md
@@ -41,15 +41,9 @@ IT 运维监测数据通常都是对时间特性比较敏感的数据，例如
## 数据链路设置
-### 复制 TDengine 插件到 grafana 插件目录
+### 安装 Grafana Plugin 并配置数据源
-```bash
-1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip
-2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/
-3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine
-4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini
-5. sudo systemctl restart grafana-server.service
-```
+请参考[安装 Grafana Plugin 并配置数据源](/third-party/grafana/#%E5%AE%89%E8%A3%85-grafana-plugin-%E5%B9%B6%E9%85%8D%E7%BD%AE%E6%95%B0%E6%8D%AE%E6%BA%90)。
### 配置 collectd
diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md
index 9c2f1e790c..595b69b08b 100644
--- a/docs/zh/27-train-faq/01-faq.md
+++ b/docs/zh/27-train-faq/01-faq.md
@@ -201,3 +201,45 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端
OOM 是操作系统的保护机制，当操作系统内存(包括 SWAP )不足时，会杀掉某些进程，从而保证操作系统的稳定运行。通常内存不足主要是如下两个原因导致，一是剩余内存小于 vm.min_free_kbytes ；二是程序请求的内存大于剩余内存。还有一种情况是内存充足但程序占用了特殊的内存地址，也会触发 OOM 。
TDengine 会预先为每个 VNode 分配好内存，每个 Database 的 VNode 个数受 建库时的vgroups参数影响，每个 VNode 占用的内存大小受 buffer参数 影响。要防止 OOM，需要在项目建设之初合理规划内存，并合理设置 SWAP ，除此之外查询过量的数据也有可能导致内存暴涨，这取决于具体的查询语句。TDengine 企业版对内存管理做了优化，采用了新的内存分配器，对稳定性有更高要求的用户可以考虑选择企业版。
+
+### 19. 在macOS上遇到Too many open files怎么办？
+
+taosd日志文件报错Too many open file，是由于taosd打开文件数超过系统设置的上限所致。
+解决方案如下：
+1. 新建文件 /Library/LaunchDaemons/limit.maxfiles.plist，写入以下内容(以下示例将limit和maxfiles改为10万，可按需修改)：
+```
+
+
+
+
+Label
+ limit.maxfiles
+ProgramArguments
+
+ launchctl
+ limit
+ maxfiles
+ 100000
+ 100000
+
+RunAtLoad
+
+ServiceIPC
+
+
+
+```
+2. 修改文件权限
+```
+sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
+sudo chmod 644 /Library/LaunchDaemons/limit.maxfiles.plist
+```
+3. 加载 plist 文件 (或重启系统后生效。launchd在启动时会自动加载该目录的 plist)
+```
+sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist
+```
+4.确认更改后的限制
+```
+launchctl limit maxfiles
+```
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 77f1879b81..6ec3d5db10 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -249,10 +249,11 @@ typedef struct SColumnInfoData {
typedef struct SQueryTableDataCond {
uint64_t suid;
- int32_t order; // desc|asc order to iterate the data block
+ int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols;
SColumnInfo* colList;
- int32_t type; // data block load type:
+ int32_t* pSlotList; // the column output destation slot, and it may be null
+ int32_t type; // data block load type:
STimeWindow twindows;
int64_t startVersion;
int64_t endVersion;
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index db6ca65cf0..e1d3b01611 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -41,9 +41,9 @@ typedef struct SBlockOrderInfo {
BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
} while (0)
-#define colDataSetNotNull_f(bm_, r_) \
- do { \
- BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \
+#define colDataClearNull_f(bm_, r_) \
+ do { \
+ BMCharPos(bm_, r_) &= ((char)(~(1u << (7u - BitPos(r_))))); \
} while (0)
#define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1)
@@ -151,9 +151,6 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui
for (int32_t i = start; i < start + nRows; ++i) {
colDataSetNull_f(pColumnInfoData->nullbitmap, i);
}
-
- int32_t bytes = pColumnInfoData->info.bytes;
- memset(pColumnInfoData->pData + start * bytes, 0, nRows * bytes);
}
pColumnInfoData->hasNull = true;
@@ -234,9 +231,11 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload);
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
+int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows);
void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
void blockDataCleanup(SSDataBlock* pDataBlock);
+void blockDataEmpty(SSDataBlock* pDataBlock);
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 89ec9dc6c8..005cf36d5e 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -137,6 +137,7 @@ extern int64_t tsWalFsyncDataSizeLimit;
// internal
extern int32_t tsTransPullupInterval;
extern int32_t tsMqRebalanceInterval;
+extern int32_t tsStreamCheckpointTickInterval;
extern int32_t tsTtlUnit;
extern int32_t tsTtlPushInterval;
extern int32_t tsGrantHBInterval;
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index b6966c97b9..0a082a37e4 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -120,6 +120,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_VNODES,
TSDB_MGMT_TABLE_APPS,
TSDB_MGMT_TABLE_STREAM_TASKS,
+ TSDB_MGMT_TABLE_PRIVILEGES,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@@ -1061,6 +1062,7 @@ typedef struct {
int32_t vgId;
int8_t syncState;
int8_t syncRestore;
+ int8_t syncCanRead;
int64_t cacheUsage;
int64_t numOfTables;
int64_t numOfTimeSeries;
@@ -1146,6 +1148,13 @@ typedef struct {
int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
+typedef struct {
+ int64_t tick;
+} SMStreamTickReq;
+
+int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
+int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
+
typedef struct {
int32_t id;
uint16_t port; // node sync Port
@@ -1746,6 +1755,8 @@ typedef struct {
int64_t watermark;
int32_t numOfTags;
SArray* pTags; // array of SField
+ // 3.0.20
+ int64_t checkpointFreq; // ms
} SCMCreateStreamReq;
typedef struct {
@@ -1945,6 +1956,12 @@ typedef struct {
SHashObj* rebSubHash; // SHashObj
} SMqDoRebalanceMsg;
+typedef struct {
+ int64_t streamId;
+ int64_t checkpointId;
+ char streamName[TSDB_STREAM_FNAME_LEN];
+} SMStreamDoCheckpointMsg;
+
typedef struct {
int64_t status;
} SMVSubscribeRsp;
diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h
index b5b997dac0..32d00bb422 100644
--- a/include/common/tmsgcb.h
+++ b/include/common/tmsgcb.h
@@ -43,7 +43,6 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg);
typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
typedef void (*SendRspFp)(SRpcMsg* pMsg);
-typedef void (*SendRedirectRspFp)(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg);
typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
typedef void (*ReportStartup)(const char* name, const char* desc);
@@ -55,7 +54,6 @@ typedef struct {
GetQueueSizeFp qsizeFp;
SendReqFp sendReqFp;
SendRspFp sendRspFp;
- SendRedirectRspFp sendRedirectRspFp;
RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
ReleaseHandleFp releaseHandleFp;
ReportStartup reportStartupFp;
@@ -66,7 +64,6 @@ int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg);
int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype);
int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg);
void tmsgSendRsp(SRpcMsg* pMsg);
-void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg);
void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type);
void tmsgReportStartup(const char* name, const char* desc);
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index e80766d249..7833bdf139 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -172,6 +172,8 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_SERVER_VERSION, "server-version", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_VND_MSG)
@@ -241,8 +243,11 @@ enum {
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
- TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL)
- TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "stream-recover-finish", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "stream-task-check", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT, "stream-checkpoint", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESTORE_CHECKPOINT, "stream-restore-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_MON_MSG)
@@ -282,6 +287,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, "vnode-stream-recover1", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, "vnode-stream-recover2", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_VND_STREAM_CHECK_POINT_SOURCE, "vnode-stream-checkpoint-source", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)
diff --git a/include/common/ttypes.h b/include/common/ttypes.h
index 761ffd0f1c..6350057c1f 100644
--- a/include/common/ttypes.h
+++ b/include/common/ttypes.h
@@ -278,11 +278,9 @@ typedef struct {
#define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
#define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
#define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
-#define IS_VALID_BIGINT(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX)
#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX)
-#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) <= UINT64_MAX)
#define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
#define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index d210004760..412b4b4cf6 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -152,7 +152,7 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo);
* @param tinfo qhandle
* @return
*/
-int32_t qAsyncKillTask(qTaskInfo_t tinfo);
+int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode);
/**
* destroy query info structure
diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 13a8370dfd..32b8cc7389 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -137,22 +137,22 @@ typedef struct SqlFunctionCtx {
int16_t functionId; // function id
char *pOutput; // final result output buffer, point to sdata->data
int32_t numOfParams;
- SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
- SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
- int32_t offset;
- struct SResultRowEntryInfo *resultInfo;
- SSubsidiaryResInfo subsidiaries;
- SPoint1 start;
- SPoint1 end;
- SFuncExecFuncs fpSet;
- SScalarFuncExecFuncs sfp;
- struct SExprInfo *pExpr;
- struct SSDataBlock *pSrcBlock;
- struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity
- SSerializeDataHandle saveHandle;
- bool isStream;
-
- char udfName[TSDB_FUNC_NAME_LEN];
+ // input parameter, e.g., top(k, 20), the number of results of top query is kept in param
+ SFunctParam *param;
+ // corresponding output buffer for timestamp of each result, e.g., diff/csum
+ SColumnInfoData *pTsOutput;
+ int32_t offset;
+ SResultRowEntryInfo *resultInfo;
+ SSubsidiaryResInfo subsidiaries;
+ SPoint1 start;
+ SPoint1 end;
+ SFuncExecFuncs fpSet;
+ SScalarFuncExecFuncs sfp;
+ struct SExprInfo *pExpr;
+ struct SSDataBlock *pSrcBlock;
+ struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity
+ SSerializeDataHandle saveHandle;
+ char udfName[TSDB_FUNC_NAME_LEN];
} SqlFunctionCtx;
typedef struct tExprNode {
@@ -163,6 +163,7 @@ typedef struct tExprNode {
int32_t functionId;
int32_t num;
struct SFunctionNode *pFunctNode;
+ int32_t functionType;
} _function;
struct {
@@ -182,7 +183,6 @@ struct SScalarParam {
};
void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell);
-//int32_t getNumOfResult(SqlFunctionCtx *pCtx, int32_t num, SSDataBlock *pResBlock);
bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry);
bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry);
@@ -194,32 +194,6 @@ typedef struct SPoint {
int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2,
int32_t inputType);
-///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-// udf api
-/**
- * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf
- * @return error code
- */
-int32_t udfcOpen();
-
-/**
- * destroy udfd proxy
- * @return error code
- */
-int32_t udfcClose();
-
-/**
- * start udfd that serves udf function invocation under dnode startDnodeId
- * @param startDnodeId
- * @return
- */
-int32_t udfStartUdfd(int32_t startDnodeId);
-/**
- * stop udfd
- * @return
- */
-int32_t udfStopUdfd();
-
#ifdef __cplusplus
}
#endif
diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h
index 81f63537e5..9ca6a7a9fa 100644
--- a/include/libs/function/functionMgt.h
+++ b/include/libs/function/functionMgt.h
@@ -130,6 +130,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_GROUP_KEY,
FUNCTION_TYPE_CACHE_LAST_ROW,
FUNCTION_TYPE_CACHE_LAST,
+ FUNCTION_TYPE_TABLE_COUNT,
// distributed splitting functions
FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000,
diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h
index 31cc53bb9f..b71d50d43c 100644
--- a/include/libs/function/tudf.h
+++ b/include/libs/function/tudf.h
@@ -85,6 +85,32 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols,
int32_t cleanUpUdfs();
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// udf api
+/**
+ * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf
+ * @return error code
+ */
+int32_t udfcOpen();
+
+/**
+ * destroy udfd proxy
+ * @return error code
+ */
+int32_t udfcClose();
+
+/**
+ * start udfd that serves udf function invocation under dnode startDnodeId
+ * @param startDnodeId
+ * @return
+ */
+int32_t udfStartUdfd(int32_t startDnodeId);
+/**
+ * stop udfd
+ * @return
+ */
+int32_t udfStopUdfd();
+
#ifdef __cplusplus
}
#endif
diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h
index cd7cedb6c1..f94e0c98dc 100644
--- a/include/libs/nodes/nodes.h
+++ b/include/libs/nodes/nodes.h
@@ -60,6 +60,12 @@ extern "C" {
for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
(NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
+#define NODES_DESTORY_NODE(node) \
+ do { \
+ nodesDestroyNode((node)); \
+ (node) = NULL; \
+ } while (0)
+
#define NODES_DESTORY_LIST(list) \
do { \
nodesDestroyList((list)); \
@@ -228,6 +234,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN,
QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN,
+ QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN,
QUERY_NODE_PHYSICAL_PLAN_PROJECT,
QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN,
QUERY_NODE_PHYSICAL_PLAN_HASH_AGG,
diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h
index 6a6fd50f2a..574d41db58 100644
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@@ -62,7 +62,8 @@ typedef enum EScanType {
SCAN_TYPE_STREAM,
SCAN_TYPE_TABLE_MERGE,
SCAN_TYPE_BLOCK_INFO,
- SCAN_TYPE_LAST_ROW
+ SCAN_TYPE_LAST_ROW,
+ SCAN_TYPE_TABLE_COUNT
} EScanType;
typedef struct SScanLogicNode {
@@ -323,6 +324,8 @@ typedef struct SLastRowScanPhysiNode {
bool ignoreNull;
} SLastRowScanPhysiNode;
+typedef SLastRowScanPhysiNode STableCountScanPhysiNode;
+
typedef struct SSystemTableScanPhysiNode {
SScanPhysiNode scan;
SEpSet mgmtEpSet;
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index f51aa88485..97ce5ae032 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -260,24 +260,24 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
-#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR)
-#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR)
+#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED)
+#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR)
#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later
-#define NEED_REDIRECT_ERROR(_code) \
- ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
- (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
- SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
- (_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK)
+#define NEED_REDIRECT_ERROR(_code) \
+ ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
+ (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
+ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
+ (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
(_type) == TDMT_MND_DROP_STB)
-#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \
- ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || \
- SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \
- SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_APP_NOT_READY)
+#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \
+ ((_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
+ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
+ (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
#define REQUEST_TOTAL_EXEC_TIMES 2
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 4099551188..0196dcb0a8 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -275,31 +275,6 @@ typedef struct {
SEpSet epSet;
} SStreamChildEpInfo;
-typedef struct {
- int32_t srcNodeId;
- int32_t srcChildId;
- int64_t stateSaveVer;
- int64_t stateProcessedVer;
-} SStreamCheckpointInfo;
-
-typedef struct {
- int64_t streamId;
- int64_t checkTs;
- int32_t checkpointId; // incremental
- int32_t taskId;
- SArray* checkpointVer; // SArray
-} SStreamMultiVgCheckpointInfo;
-
-typedef struct {
- int32_t taskId;
- int32_t checkpointId; // incremental
-} SStreamCheckpointKey;
-
-typedef struct {
- int32_t taskId;
- SArray* checkpointVer;
-} SStreamRecoveringState;
-
typedef struct SStreamTask {
int64_t streamId;
int32_t taskId;
@@ -364,6 +339,10 @@ typedef struct SStreamTask {
int64_t checkReqId;
SArray* checkReqIds; // shuffle
int32_t refCnt;
+
+ int64_t checkpointingId;
+ int32_t checkpointAlignCnt;
+
} SStreamTask;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@@ -509,6 +488,60 @@ typedef struct {
int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq);
int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq);
+typedef struct {
+ int64_t streamId;
+ int64_t checkpointId;
+ int32_t taskId;
+ int32_t nodeId;
+ int64_t expireTime;
+} SStreamCheckpointSourceReq;
+
+typedef struct {
+ int64_t streamId;
+ int64_t checkpointId;
+ int32_t taskId;
+ int32_t nodeId;
+ int64_t expireTime;
+} SStreamCheckpointSourceRsp;
+
+int32_t tEncodeSStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq);
+int32_t tDecodeSStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSourceReq* pReq);
+
+int32_t tEncodeSStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckpointSourceRsp* pRsp);
+int32_t tDecodeSStreamCheckpointSourceRsp(SDecoder* pDecoder, SStreamCheckpointSourceRsp* pRsp);
+
+typedef struct {
+ SMsgHead msgHead;
+ int64_t streamId;
+ int64_t checkpointId;
+ int32_t downstreamTaskId;
+ int32_t downstreamNodeId;
+ int32_t upstreamTaskId;
+ int32_t upstreamNodeId;
+ int32_t childId;
+ int64_t expireTime;
+ int8_t taskLevel;
+} SStreamCheckpointReq;
+
+typedef struct {
+ SMsgHead msgHead;
+ int64_t streamId;
+ int64_t checkpointId;
+ int32_t downstreamTaskId;
+ int32_t downstreamNodeId;
+ int32_t upstreamTaskId;
+ int32_t upstreamNodeId;
+ int32_t childId;
+ int64_t expireTime;
+ int8_t taskLevel;
+} SStreamCheckpointRsp;
+
+int32_t tEncodeSStreamCheckpointReq(SEncoder* pEncoder, const SStreamCheckpointReq* pReq);
+int32_t tDecodeSStreamCheckpointReq(SDecoder* pDecoder, SStreamCheckpointReq* pReq);
+
+int32_t tEncodeSStreamCheckpointRsp(SEncoder* pEncoder, const SStreamCheckpointRsp* pRsp);
+int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pRsp);
+
typedef struct {
int64_t streamId;
int32_t downstreamTaskId;
@@ -598,18 +631,22 @@ void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen);
-int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId);
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
-void streamMetaRemoveTask1(SStreamMeta* pMeta, int32_t taskId);
+void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
int32_t streamMetaBegin(SStreamMeta* pMeta);
int32_t streamMetaCommit(SStreamMeta* pMeta);
int32_t streamMetaRollBack(SStreamMeta* pMeta);
int32_t streamLoadTasks(SStreamMeta* pMeta);
+// checkpoint
+int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
+int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointReq* pReq);
+int32_t streamProcessCheckpointRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointRsp* pRsp);
+
#ifdef __cplusplus
}
#endif
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 25d37020cc..490dbf7744 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -40,48 +40,64 @@ int32_t* taosGetErrno();
#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error
// rpc
-#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003)
-#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004)
+// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) // 2.x
+// #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) // 2.x
+// #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) // 2.x
+// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) // 2.x
+// #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) // 2.x
+// #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) // 2.x
+// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) // 2.x
+// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) // 2.x
+// #define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) // 2.x
+// #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) // 2.x
#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B)
+// #define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) // 2.x
+// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) // 2.x
+// #define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) // 2.x
+// #define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) // 2.x
+// #define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) // 2.x
+// #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) // 2.x
+// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) // 2.x
+#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) //
+// #define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) // 2.x
#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015)
-#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017)
-#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018)
-#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019)
+// #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) // 2.x
+#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) //
+#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) //
+#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) //
//common & util
-#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013)
-#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014)
-
-#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100)
-#define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101)
+#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) //
+// #define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) // 2.x
#define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0102)
-#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104)
-#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105)
-#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106)
-#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107)
-#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108)
-#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109)
-#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A)
+// #define TSDB_CODE_COM_INVALID_CFG_MSG TAOS_DEF_ERROR_CODE(0, 0x0103) // 2.x
+#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) //
+#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) // internal
+#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) // internal
+#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107) // internal
+#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108) // internal
+#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109) // internal
+#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) // internal
-#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0110)
-#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0111)
-#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0112)
-#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113)
-#define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114)
-#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115)
-#define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0116)
-#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117)
-#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118)
-#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119)
-#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x011A)
-#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x011B)
-#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x011C)
-#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x011D)
-#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x011E)
-#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F)
+#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0110) //
+#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0111) // internal
+#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0112) //
+// #define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113)
+// #define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114)
+#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115) //
+#define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0116) //
+#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117) // internal
+#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118) //
+#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119) // internal
+#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x011A) // internal
+#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x011B) // internal
+#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x011C) // internal
+#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x011D) // internal
+#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x011E) // internal
+#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) // internal
#define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0120)
-#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121)
+#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) //
#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0122)
#define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0123)
#define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0124)
@@ -94,6 +110,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012B)
#define TSDB_CODE_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x012C)
+#define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) //
+#define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) //
+
//client
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200)
#define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201)
@@ -279,6 +298,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3)
#define TSDB_CODE_MND_TRANS_CLOG_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x03D4)
#define TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL TAOS_DEF_ERROR_CODE(0, 0x03D5)
+#define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal
#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF)
// mnode-mq
@@ -311,19 +331,56 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482)
// dnode
-#define TSDB_CODE_NODE_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0408)
-#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0409)
-#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040A)
+// #define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) // 2.x
+// #define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) // 2.x
+// #define TSDB_CODE_DND_NO_WRITE_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0402) // 2.x
+// #define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0403) // 2.x
+// #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0404) // 2.x
+// #define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) // 2.x
+// #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) // 2.x
+// #define TSDB_CODE_DND_VNODE_OPEN_FAILED TAOS_DEF_ERROR_CODE(0, 0x0407) // 2.x
+#define TSDB_CODE_DNODE_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0408)
+#define TSDB_CODE_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0409)
+#define TSDB_CODE_MNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040A)
+#define TSDB_CODE_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040B)
+#define TSDB_CODE_QNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040C)
+#define TSDB_CODE_QNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040D)
+#define TSDB_CODE_QNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040E)
+#define TSDB_CODE_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040F)
+#define TSDB_CODE_SNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0410)
+#define TSDB_CODE_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411)
// vnode
+// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x
+// #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) // 2.x
+// #define TSDB_CODE_VND_ACTION_NEED_REPROCESS. TAOS_DEF_ERROR_CODE(0, 0x0502) // 2.x
#define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503)
+// #define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) // 2.x
+// #define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) // 2.x
+// #define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) // 2.x
+// #define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) // 2.x
+// #define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) // 2.x
+// #define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) // 2.x
+// #define TSDB_CODE_VND_INVALID_VRESION_FILE TAOS_DEF_ERROR_CODE(0, 0x050A) // 2.x
+// #define TSDB_CODE_VND_IS_FULL TAOS_DEF_ERROR_CODE(0, 0x050B) // 2.x
+// #define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) // 2.x
+// #define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) // 2.x
+// #define TSDB_CODE_VND_IS_BALANCING TAOS_DEF_ERROR_CODE(0, 0x050E) // 2.x
+// #define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0510) // 2.x
+// #define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) // 2.x
#define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512)
+// #define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) // 2.x
+// #define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) // 2.x
+// #define TSDB_CODE_WAIT_THREAD_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0515) // 2.x
+#define TSDB_CODE_VND_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0520) // internal
+#define TSDB_CODE_VND_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0521) // internal
#define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0522)
#define TSDB_CODE_VND_INVALID_TABLE_ACTION TAOS_DEF_ERROR_CODE(0, 0x0524)
#define TSDB_CODE_VND_COL_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0525)
#define TSDB_CODE_VND_COL_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0526)
#define TSDB_CODE_VND_COL_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x0527)
#define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528)
+#define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529)
// tsdb
#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600)
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 976d1dd1b0..8ee774ceee 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -146,7 +146,6 @@ void taos_close(TAOS *taos) {
int taos_errno(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
- if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
return terrno;
}
@@ -154,12 +153,11 @@ int taos_errno(TAOS_RES *res) {
return 0;
}
- return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code;
+ return ((SRequestObj *)res)->code;
}
const char *taos_errstr(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) {
- if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
return (const char *)tstrerror(terrno);
}
@@ -171,8 +169,7 @@ const char *taos_errstr(TAOS_RES *res) {
if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
return pRequest->msgBuf;
} else {
- return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY)
- : (const char *)tstrerror(pRequest->code);
+ return (const char *)tstrerror(pRequest->code);
}
}
diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index d811eb7fec..719a6f82d2 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -139,6 +139,8 @@ typedef struct {
int32_t numOfSTables;
int32_t numOfCTables;
int32_t numOfCreateSTables;
+ int32_t numOfAlterColSTables;
+ int32_t numOfAlterTagSTables;
int64_t parseTime;
int64_t schemaTime;
@@ -512,6 +514,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end;
}
+ info->cost.numOfAlterTagSTables++;
taosMemoryFreeClear(pTableMeta);
code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
if (code != TSDB_CODE_SUCCESS) {
@@ -559,6 +562,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end;
}
+ info->cost.numOfAlterColSTables++;
+ taosMemoryFreeClear(pTableMeta);
code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
if (code != TSDB_CODE_SUCCESS) {
goto end;
@@ -820,11 +825,6 @@ static int8_t smlGetTsTypeByPrecision(int8_t precision) {
}
static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) {
- void *tmp = taosMemoryCalloc(1, len + 1);
- memcpy(tmp, data, len);
- uDebug("SML:0x%" PRIx64 " smlParseInfluxTime tslen:%d, ts:%s", info->id, len, (char*)tmp);
- taosMemoryFree(tmp);
-
if (len == 0 || (len == 1 && data[0] == '0')) {
return taosGetTimestampNs();
}
@@ -877,7 +877,10 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra
}
uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts);
- if (ts == -1) return TSDB_CODE_INVALID_TIMESTAMP;
+ if (ts <= 0) {
+ uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts);
+ return TSDB_CODE_INVALID_TIMESTAMP;
+ }
// add ts to
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
@@ -2076,10 +2079,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *
static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) {
SSmlLineInfo elements = {0};
- void *tmp = taosMemoryCalloc(1, len + 1);
- memcpy(tmp, sql, len);
- uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? (char*)tmp : sql));
- taosMemoryFree(tmp);
+ uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : sql));
int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf);
if (ret != TSDB_CODE_SUCCESS) {
@@ -2372,11 +2372,12 @@ static int32_t smlInsertData(SSmlHandle *info) {
static void smlPrintStatisticInfo(SSmlHandle *info) {
uError("SML:0x%" PRIx64
- " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d \
+ " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
"",
info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables,
- info->cost.numOfCreateSTables, info->cost.schemaTime - info->cost.parseTime,
+ info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables,
+ info->cost.schemaTime - info->cost.parseTime,
info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
}
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index a4a185e5ee..c53e771f42 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1137,14 +1137,15 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
}
void blockDataCleanup(SSDataBlock* pDataBlock) {
+ blockDataEmpty(pDataBlock);
SDataBlockInfo* pInfo = &pDataBlock->info;
-
- pInfo->rows = 0;
pInfo->id.uid = 0;
pInfo->id.groupId = 0;
- pInfo->window.ekey = 0;
- pInfo->window.skey = 0;
+}
+void blockDataEmpty(SSDataBlock* pDataBlock) {
+ SDataBlockInfo* pInfo = &pDataBlock->info;
+ ASSERT(pInfo->rows <= pDataBlock->info.capacity);
if (pInfo->capacity == 0) {
return;
}
@@ -1154,6 +1155,10 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
colInfoDataCleanup(p, pInfo->capacity);
}
+
+ pInfo->rows = 0;
+ pInfo->window.ekey = 0;
+ pInfo->window.skey = 0;
}
// todo temporarily disable it
@@ -1249,6 +1254,25 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
return TSDB_CODE_SUCCESS;
}
+int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows) {
+ int32_t code = 0;
+ if (numOfRows == 0 || numOfRows <= pDataBlock->info.capacity) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
+ for (int32_t i = 0; i < numOfCols; ++i) {
+ SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
+ code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, false);
+ if (code) {
+ return code;
+ }
+ }
+
+ pDataBlock->info.capacity = numOfRows;
+ return TSDB_CODE_SUCCESS;
+}
+
void blockDataFreeRes(SSDataBlock* pBlock) {
int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfOutput; ++i) {
@@ -1621,6 +1645,8 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s
static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total);
+
+ // clear the offset value of the unused entries.
memset(&pColInfoData->varmeta.offset[total - n], 0, n);
} else {
int32_t bytes = pColInfoData->info.bytes;
@@ -1635,7 +1661,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
}
if (pBlock->info.rows <= n) {
- blockDataCleanup(pBlock);
+ blockDataEmpty(pBlock);
} else {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
@@ -1652,12 +1678,22 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
memset(&pColInfoData->varmeta.offset[n], 0, total - n);
+ } else { // reset the bitmap value
+ /*int32_t stopIndex = BitmapLen(n) * 8;
+ for(int32_t i = n; i < stopIndex; ++i) {
+ colDataClearNull_f(pColInfoData->nullbitmap, i);
+ }
+
+ int32_t remain = BitmapLen(total) - BitmapLen(n);
+ if (remain > 0) {
+ memset(pColInfoData->nullbitmap+BitmapLen(n), 0, remain);
+ }*/
}
}
int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
if (n == 0) {
- blockDataCleanup(pBlock);
+ blockDataEmpty(pBlock);
return TSDB_CODE_SUCCESS;
}
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index ab46ba24cf..2e8cb5e5f7 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -167,6 +167,7 @@ int64_t tsWalFsyncDataSizeLimit = (100 * 1024 * 1024L);
// internal
int32_t tsTransPullupInterval = 2;
int32_t tsMqRebalanceInterval = 2;
+int32_t tsStreamCheckpointTickInterval = 1;
int32_t tsTtlUnit = 86400;
int32_t tsTtlPushInterval = 86400;
int32_t tsGrantHBInterval = 60;
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 3f81dbcd81..5b16a55ae3 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -992,15 +992,20 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tEncodeI32(&encoder, vlen) < 0) return -1;
for (int32_t i = 0; i < vlen; ++i) {
SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i);
+ int64_t reserved = 0;
if (tEncodeI32(&encoder, pload->vgId) < 0) return -1;
if (tEncodeI8(&encoder, pload->syncState) < 0) return -1;
if (tEncodeI8(&encoder, pload->syncRestore) < 0) return -1;
+ if (tEncodeI8(&encoder, pload->syncCanRead) < 0) return -1;
if (tEncodeI64(&encoder, pload->cacheUsage) < 0) return -1;
if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1;
if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1;
if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1;
if (tEncodeI64(&encoder, pload->compStorage) < 0) return -1;
if (tEncodeI64(&encoder, pload->pointsWritten) < 0) return -1;
+ if (tEncodeI64(&encoder, reserved) < 0) return -1;
+ if (tEncodeI64(&encoder, reserved) < 0) return -1;
+ if (tEncodeI64(&encoder, reserved) < 0) return -1;
}
// mnode loads
@@ -1065,15 +1070,20 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
for (int32_t i = 0; i < vlen; ++i) {
SVnodeLoad vload = {0};
+ int64_t reserved = 0;
if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1;
if (tDecodeI8(&decoder, &vload.syncState) < 0) return -1;
if (tDecodeI8(&decoder, &vload.syncRestore) < 0) return -1;
+ if (tDecodeI8(&decoder, &vload.syncCanRead) < 0) return -1;
if (tDecodeI64(&decoder, &vload.cacheUsage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1;
if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1;
if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1;
+ if (tDecodeI64(&decoder, &reserved) < 0) return -1;
+ if (tDecodeI64(&decoder, &reserved) < 0) return -1;
+ if (tDecodeI64(&decoder, &reserved) < 0) return -1;
if (taosArrayPush(pReq->pVloads, &vload) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@@ -3738,6 +3748,31 @@ int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
return 0;
}
+int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
+ SEncoder encoder = {0};
+ tEncoderInit(&encoder, buf, bufLen);
+
+ if (tStartEncode(&encoder) < 0) return -1;
+ if (tEncodeI64(&encoder, pReq->tick) < 0) return -1;
+ tEndEncode(&encoder);
+
+ int32_t tlen = encoder.pos;
+ tEncoderClear(&encoder);
+ return tlen;
+}
+
+int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
+ SDecoder decoder = {0};
+ tDecoderInit(&decoder, buf, bufLen);
+
+ if (tStartDecode(&decoder) < 0) return -1;
+ if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1;
+ tEndDecode(&decoder);
+
+ tDecoderClear(&decoder);
+ return 0;
+}
+
int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) {
if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1;
if (tEncodeU16(pEncoder, pReplica->port) < 0) return -1;
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
index 07a612bb35..30ef7b9542 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
@@ -144,6 +144,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
break;
default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+ dGError("msg:%p, not processed in mgmt queue", pMsg);
break;
}
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
index ceb452c551..16fe6c1b91 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
@@ -196,7 +196,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
code = 0;
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
index 212de0bfb4..857fbcbce5 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
@@ -61,7 +61,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
pMsg->info.rsp = NULL;
}
- if (code == TSDB_CODE_RPC_REDIRECT) {
+ if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) {
mndPostProcessQueryMsg(pMsg);
}
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 743beb7f82..bc46772858 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -216,7 +216,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dDebug("vgId:%d, already exist", req.vgId);
tFreeSCreateVnodeReq(&req);
vmReleaseVnode(pMgmt, pVnode);
- terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+ terrno = TSDB_CODE_VND_ALREADY_EXIST;
code = terrno;
return 0;
}
@@ -307,7 +307,7 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId);
if (pVnode == NULL) {
dError("vgId:%d, failed to alter replica since %s", vgId, terrstr());
- terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+ terrno = TSDB_CODE_VND_NOT_EXIST;
return -1;
}
@@ -369,7 +369,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId);
if (pVnode == NULL) {
dDebug("vgId:%d, failed to drop since %s", vgId, terrstr());
- terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+ terrno = TSDB_CODE_VND_NOT_EXIST;
return -1;
}
@@ -464,7 +464,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
- if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
+ if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
code = 0;
diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
index 421d723202..e3bda5a3f0 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
@@ -152,7 +152,19 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper != NULL) {
dmReleaseWrapper(pWrapper);
- terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+ switch (ntype) {
+ case MNODE:
+ terrno = TSDB_CODE_MNODE_ALREADY_DEPLOYED;
+ break;
+ case QNODE:
+ terrno = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
+ break;
+ case SNODE:
+ terrno = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
+ break;
+ default:
+ terrno = TSDB_CODE_APP_ERROR;
+ }
dError("failed to create node since %s", terrstr());
return -1;
}
@@ -191,7 +203,20 @@ static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) {
- terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+ switch (ntype) {
+ case MNODE:
+ terrno = TSDB_CODE_MNODE_NOT_DEPLOYED;
+ break;
+ case QNODE:
+ terrno = TSDB_CODE_QNODE_NOT_DEPLOYED;
+ break;
+ case SNODE:
+ terrno = TSDB_CODE_SNODE_NOT_DEPLOYED;
+ break;
+ default:
+ terrno = TSDB_CODE_APP_ERROR;
+ }
+
dError("failed to drop node since %s", terrstr());
return -1;
}
diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
index 2c9020b668..02a268afda 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
@@ -189,7 +189,6 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) {
int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
// dTrace("node:%s, is acquired, ref:%d", pWrapper->name, refCount);
} else {
- terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
pRetWrapper = NULL;
}
taosThreadRwlockUnlock(&pWrapper->lock);
@@ -205,7 +204,20 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) {
int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
// dTrace("node:%s, is marked, ref:%d", pWrapper->name, refCount);
} else {
- terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+ switch (pWrapper->ntype) {
+ case MNODE:
+ terrno = TSDB_CODE_MNODE_NOT_FOUND;
+ break;
+ case QNODE:
+ terrno = TSDB_CODE_QNODE_NOT_FOUND;
+ break;
+ case SNODE:
+ terrno = TSDB_CODE_SNODE_NOT_FOUND;
+ break;
+ default:
+ terrno = TSDB_CODE_APP_IS_STOPPING;
+ break;
+ }
code = -1;
}
taosThreadRwlockUnlock(&pWrapper->lock);
diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 95656fd76c..12aba130d5 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -33,31 +33,16 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) {
}
}
-static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) {
- pMsg->info.hasEpSet = 1;
- SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info, .msgType = pMsg->msgType};
- int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet);
-
- rsp.pCont = rpcMallocCont(contLen);
- if (rsp.pCont == NULL) {
- pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
- } else {
- tSerializeSEpSet(rsp.pCont, contLen, pNewEpSet);
- rsp.contLen = contLen;
- }
- dmSendRsp(&rsp);
- rpcFreeCont(pMsg->pCont);
- pMsg->pCont = NULL;
-}
-
int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
+ const STraceId *trace = &pMsg->info.traceId;
+
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)];
if (msgFp == NULL) {
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+ dGError("msg:%p, not processed since no handler", pMsg);
return -1;
}
- const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name);
pMsg->info.wrapper = pWrapper;
return (*msgFp)(pWrapper->pMgmt, pMsg);
@@ -99,18 +84,23 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
dmProcessServerStartupStatus(pDnode, pRpc);
return;
} else {
- terrno = TSDB_CODE_APP_NOT_READY;
+ if (pDnode->status == DND_STAT_INIT) {
+ terrno = TSDB_CODE_APP_IS_STARTING;
+ } else {
+ terrno = TSDB_CODE_APP_IS_STOPPING;
+ }
goto _OVER;
}
}
- if (IsReq(pRpc) && pRpc->pCont == NULL) {
+ if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) {
dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER;
}
if (pHandle->defaultNtype == NODE_END) {
+ dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
goto _OVER;
}
@@ -165,8 +155,7 @@ _OVER:
if (IsReq(pRpc)) {
SRpcMsg rsp = {.code = code, .info = pRpc->info};
- if ((code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) && pRpc->msgType > TDMT_MND_MSG &&
- pRpc->msgType < TDMT_VND_MSG) {
+ if (code == TSDB_CODE_MNODE_NOT_FOUND) {
dmBuildMnodeRedirectRsp(pDnode, &rsp);
}
@@ -219,7 +208,11 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
- terrno = TSDB_CODE_NODE_OFFLINE;
+ if (pDnode->status == DND_STAT_INIT) {
+ terrno = TSDB_CODE_APP_IS_STARTING;
+ } else {
+ terrno = TSDB_CODE_APP_IS_STOPPING;
+ }
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle);
return -1;
} else {
@@ -233,8 +226,9 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLin
static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcReleaseHandle(pHandle, type); }
static bool rpcRfp(int32_t code, tmsg_t msgType) {
- if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
- code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) {
+ if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_MNODE_NOT_FOUND ||
+ code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED ||
+ code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) {
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH ||
msgType == TDMT_SCH_MERGE_FETCH) {
return false;
@@ -323,7 +317,6 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) {
.clientRpc = pDnode->trans.clientRpc,
.sendReqFp = dmSendReq,
.sendRspFp = dmSendRsp,
- .sendRedirectRspFp = dmSendRedirectRsp,
.registerBrokenLinkArgFp = dmRegisterBrokenLinkArg,
.releaseHandleFp = dmReleaseHandle,
.reportStartupFp = dmReportStartup,
diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h
index 8719e988e7..2124b387ec 100644
--- a/source/dnode/mgmt/node_util/inc/dmUtil.h
+++ b/source/dnode/mgmt/node_util/inc/dmUtil.h
@@ -39,7 +39,7 @@
#include "sync.h"
#include "wal.h"
-#include "libs/function/function.h"
+#include "libs/function/tudf.h"
#ifdef __cplusplus
extern "C" {
#endif
diff --git a/source/dnode/mgmt/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp
index 7b7eb97216..28cb376b37 100644
--- a/source/dnode/mgmt/test/mnode/dmnode.cpp
+++ b/source/dnode/mgmt/test/mnode/dmnode.cpp
@@ -40,7 +40,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_ALREADY_DEPLOYED);
}
{
@@ -57,7 +57,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_ALREADY_DEPLOYED);
}
{
@@ -77,7 +77,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_ALREADY_DEPLOYED);
}
}
@@ -171,7 +171,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_NOT_DEPLOYED);
}
{
@@ -188,7 +188,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_MNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_NOT_DEPLOYED);
}
{
diff --git a/source/dnode/mgmt/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp
index a2c6a2c28c..3beb57c516 100644
--- a/source/dnode/mgmt/test/qnode/dqnode.cpp
+++ b/source/dnode/mgmt/test/qnode/dqnode.cpp
@@ -62,7 +62,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED);
}
test.Restart();
@@ -77,7 +77,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED);
}
}
@@ -120,7 +120,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED);
}
test.Restart();
@@ -135,7 +135,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED);
}
{
diff --git a/source/dnode/mgmt/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp
index e3ad65d831..30d6a34813 100644
--- a/source/dnode/mgmt/test/snode/dsnode.cpp
+++ b/source/dnode/mgmt/test/snode/dsnode.cpp
@@ -62,7 +62,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED);
}
test.Restart();
@@ -77,7 +77,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED);
}
}
@@ -120,7 +120,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED);
}
test.Restart();
@@ -135,7 +135,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED);
}
{
diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp
index 8a8e332289..24e7affd88 100644
--- a/source/dnode/mgmt/test/vnode/vnode.cpp
+++ b/source/dnode/mgmt/test/vnode/vnode.cpp
@@ -63,7 +63,7 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
ASSERT_EQ(pRsp->code, 0);
test.Restart();
} else {
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_VND_ALREADY_EXIST);
}
}
}
@@ -285,7 +285,7 @@ TEST_F(DndTestVnode, 06_Drop_Vnode) {
ASSERT_EQ(pRsp->code, 0);
test.Restart();
} else {
- ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_VND_NOT_EXIST);
}
}
}
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h
index 04ac5aba49..9961828747 100644
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@@ -71,6 +71,9 @@ typedef enum {
MND_OPER_READ_DB,
MND_OPER_READ_OR_WRITE_DB,
MND_OPER_SHOW_VARIBALES,
+ MND_OPER_SUBSCRIBE,
+ MND_OPER_CREATE_TOPIC,
+ MND_OPER_DROP_TOPIC,
} EOperType;
typedef enum {
@@ -273,6 +276,7 @@ typedef struct {
int32_t authVersion;
SHashObj* readDbs;
SHashObj* writeDbs;
+ SHashObj* topics;
SRWLatch lock;
} SUserObj;
@@ -328,6 +332,7 @@ typedef struct {
int32_t dnodeId;
ESyncState syncState;
bool syncRestore;
+ bool syncCanRead;
} SVnodeGid;
typedef struct {
@@ -635,10 +640,14 @@ typedef struct {
SArray* tasks; // SArray>
SSchemaWrapper outputSchema;
SSchemaWrapper tagSchema;
+
+ // 3.0.20
+ int64_t checkpointFreq; // ms
+ int64_t currentTick; // do not serialize
} SStreamObj;
int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj);
-int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj);
+int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj, int32_t sver);
void tFreeStreamObj(SStreamObj* pObj);
typedef struct {
@@ -648,15 +657,6 @@ typedef struct {
SArray* childInfo; // SArray
} SStreamCheckpointObj;
-#if 0
-typedef struct {
- int64_t uid;
- int64_t streamId;
- int8_t status;
- int8_t stage;
-} SStreamRecoverObj;
-#endif
-
#ifdef __cplusplus
}
#endif
diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h
index dc88b25f51..dfde2f671e 100644
--- a/source/dnode/mnode/impl/inc/mndPrivilege.h
+++ b/source/dnode/mnode/impl/inc/mndPrivilege.h
@@ -28,6 +28,8 @@ void mndCleanupPrivilege(SMnode *pMnode);
int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType);
int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb);
int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname);
+int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic);
+int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName);
int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);
int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp);
diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h
index 9f516904ce..8cd669c769 100644
--- a/source/dnode/mnode/impl/inc/mndTopic.h
+++ b/source/dnode/mnode/impl/inc/mndTopic.h
@@ -25,7 +25,7 @@ extern "C" {
int32_t mndInitTopic(SMnode *pMnode);
void mndCleanupTopic(SMnode *pMnode);
-SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName);
+SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName);
void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic);
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic);
diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h
index 970d1db7db..cf7deba397 100644
--- a/source/dnode/mnode/impl/inc/mndUser.h
+++ b/source/dnode/mnode/impl/inc/mndUser.h
@@ -31,6 +31,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser);
// for trans test
SSdbRaw *mndUserActionEncode(SUserObj *pUser);
SHashObj *mndDupDbHash(SHashObj *pOld);
+SHashObj *mndDupTopicHash(SHashObj *pOld);
int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp,
int32_t *pRspLen);
diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c
index ffa5dc6d0b..148e21b507 100644
--- a/source/dnode/mnode/impl/src/mndAcct.c
+++ b/source/dnode/mnode/impl/src/mndAcct.c
@@ -147,6 +147,8 @@ _OVER:
static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SAcctObj *pAcct = NULL;
+ SSdbRow *pRow = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -156,10 +158,10 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SAcctObj));
+ pRow = sdbAllocRow(sizeof(SAcctObj));
if (pRow == NULL) goto _OVER;
- SAcctObj *pAcct = sdbGetRowObj(pRow);
+ pAcct = sdbGetRowObj(pRow);
if (pAcct == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -186,7 +188,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("acct:%s, failed to decode from raw:%p since %s", pAcct->acct, pRaw, terrstr());
+ mError("acct:%s, failed to decode from raw:%p since %s", pAcct == NULL ? "null" : pAcct->acct, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
@@ -218,7 +220,7 @@ static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) {
return -1;
}
- terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+ terrno = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr());
return -1;
}
@@ -228,7 +230,7 @@ static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) {
return -1;
}
- terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+ terrno = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr());
return -1;
}
@@ -238,7 +240,7 @@ static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) {
return -1;
}
- terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+ terrno = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr());
return -1;
}
\ No newline at end of file
diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c
index 348c8f4cb8..ca03207d2b 100644
--- a/source/dnode/mnode/impl/src/mndCluster.c
+++ b/source/dnode/mnode/impl/src/mndCluster.c
@@ -157,6 +157,8 @@ _OVER:
static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SClusterObj *pCluster = NULL;
+ SSdbRow *pRow = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -166,10 +168,10 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SClusterObj));
+ pRow = sdbAllocRow(sizeof(SClusterObj));
if (pRow == NULL) goto _OVER;
- SClusterObj *pCluster = sdbGetRowObj(pRow);
+ pCluster = sdbGetRowObj(pRow);
if (pCluster == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -184,7 +186,8 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster->id, pRaw, terrstr());
+ mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster == NULL ? 0 : pCluster->id, pRaw,
+ terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c
index 4397ea0751..76bff9a70f 100644
--- a/source/dnode/mnode/impl/src/mndConsumer.c
+++ b/source/dnode/mnode/impl/src/mndConsumer.c
@@ -712,7 +712,9 @@ CM_ENCODE_OVER:
SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
- void *buf = NULL;
+ SSdbRow *pRow = NULL;
+ SMqConsumerObj *pConsumer = NULL;
+ void *buf = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER;
@@ -722,10 +724,10 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
goto CM_DECODE_OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SMqConsumerObj));
+ pRow = sdbAllocRow(sizeof(SMqConsumerObj));
if (pRow == NULL) goto CM_DECODE_OVER;
- SMqConsumerObj *pConsumer = sdbGetRowObj(pRow);
+ pConsumer = sdbGetRowObj(pRow);
if (pConsumer == NULL) goto CM_DECODE_OVER;
int32_t dataPos = 0;
@@ -745,7 +747,8 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
CM_DECODE_OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
- mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr());
+ mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer == NULL ? 0 : pConsumer->consumerId,
+ pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index c5d5687ade..5fa86dffe2 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -147,6 +147,8 @@ _OVER:
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SDbObj *pDb = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -156,10 +158,10 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj));
+ pRow = sdbAllocRow(sizeof(SDbObj));
if (pRow == NULL) goto _OVER;
- SDbObj *pDb = sdbGetRowObj(pRow);
+ pDb = sdbGetRowObj(pRow);
if (pDb == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -232,7 +234,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("db:%s, failed to decode from raw:%p since %s", pDb->name, pRaw, terrstr());
+ mError("db:%s, failed to decode from raw:%p since %s", pDb == NULL ? "null" : pDb->name, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c
index 2e984212a1..b5c2fb05b3 100644
--- a/source/dnode/mnode/impl/src/mndDef.c
+++ b/source/dnode/mnode/impl/src/mndDef.c
@@ -76,11 +76,14 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1;
+ // 3.0.20
+ if (tEncodeI64(pEncoder, pObj->checkpointFreq) < 0) return -1;
+
tEndEncode(pEncoder);
return pEncoder->pos;
}
-int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
+int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
@@ -139,6 +142,10 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1;
+ // 3.0.20
+ if (sver >= 2) {
+ if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1;
+ }
tEndDecode(pDecoder);
return 0;
}
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index 2a3ecf1924..df25e44bee 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -154,8 +154,9 @@ _OVER:
}
static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
- SSdbRow *pRow = NULL;
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SDnodeObj *pDnode = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -166,7 +167,8 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
pRow = sdbAllocRow(sizeof(SDnodeObj));
if (pRow == NULL) goto _OVER;
- SDnodeObj *pDnode = sdbGetRowObj(pRow);
+
+ pDnode = sdbGetRowObj(pRow);
if (pDnode == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -181,7 +183,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("dnode:%d, failed to decode from raw:%p since %s", pDnode->id, pRaw, terrstr());
+ mError("dnode:%d, failed to decode from raw:%p since %s", pDnode == NULL ? 0 : pDnode->id, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
@@ -375,14 +377,18 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
}
bool roleChanged = false;
for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
- if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) {
- if (pVgroup->vnodeGid[vg].syncState != pVload->syncState ||
- pVgroup->vnodeGid[vg].syncRestore != pVload->syncRestore) {
- mInfo("vgId:%d, state changed by status msg, old state:%s restored:%d new state:%s restored:%d",
- pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore,
- syncStr(pVload->syncState), pVload->syncRestore);
- pVgroup->vnodeGid[vg].syncState = pVload->syncState;
- pVgroup->vnodeGid[vg].syncRestore = pVload->syncRestore;
+ SVnodeGid *pGid = &pVgroup->vnodeGid[vg];
+ if (pGid->dnodeId == statusReq.dnodeId) {
+ if (pGid->syncState != pVload->syncState || pGid->syncRestore != pVload->syncRestore ||
+ pGid->syncCanRead != pVload->syncCanRead) {
+ mInfo(
+ "vgId:%d, state changed by status msg, old state:%s restored:%d canRead:%d new state:%s restored:%d "
+ "canRead:%d",
+ pVgroup->vgId, syncStr(pGid->syncState), pGid->syncRestore, pGid->syncCanRead,
+ syncStr(pVload->syncState), pVload->syncRestore, pVload->syncCanRead);
+ pGid->syncState = pVload->syncState;
+ pGid->syncRestore = pVload->syncRestore;
+ pGid->syncCanRead = pVload->syncCanRead;
roleChanged = true;
}
break;
@@ -781,7 +787,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id);
if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !dropReq.force) {
if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
- terrno = TSDB_CODE_NODE_OFFLINE;
+ terrno = TSDB_CODE_DNODE_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(),
numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
goto _OVER;
diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c
index c9b22fad3a..31f31a15ba 100644
--- a/source/dnode/mnode/impl/src/mndFunc.c
+++ b/source/dnode/mnode/impl/src/mndFunc.c
@@ -101,6 +101,8 @@ _OVER:
static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SFuncObj *pFunc = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -110,10 +112,10 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SFuncObj));
+ pRow = sdbAllocRow(sizeof(SFuncObj));
if (pRow == NULL) goto _OVER;
- SFuncObj *pFunc = sdbGetRowObj(pRow);
+ pFunc = sdbGetRowObj(pRow);
if (pFunc == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -148,7 +150,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("func:%s, failed to decode from raw:%p since %s", pFunc->name, pRaw, terrstr());
+ mError("func:%s, failed to decode from raw:%p since %s", pFunc == NULL ? "null" : pFunc->name, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c
index 09172115f8..82294ac7bf 100644
--- a/source/dnode/mnode/impl/src/mndInfoSchema.c
+++ b/source/dnode/mnode/impl/src/mndInfoSchema.c
@@ -71,7 +71,7 @@ static int32_t mndInsInitMeta(SHashObj *hash) {
int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, bool sysinfo,
STableMetaRsp *pRsp) {
if (NULL == pMnode->infosMeta) {
- terrno = TSDB_CODE_APP_NOT_READY;
+ terrno = TSDB_CODE_APP_ERROR;
return -1;
}
@@ -97,7 +97,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
if (NULL == pMnode->infosMeta) {
- terrno = TSDB_CODE_APP_NOT_READY;
+ terrno = TSDB_CODE_APP_ERROR;
return -1;
}
diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index f533fafeee..eb6742a564 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -45,7 +45,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0;
taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) {
- terrno = TSDB_CODE_APP_NOT_READY;
+ terrno = TSDB_CODE_APP_IS_STOPPING;
code = -1;
} else if (!mndIsLeader(pMnode)) {
code = -1;
@@ -85,6 +85,21 @@ static void *mndBuildTimerMsg(int32_t *pContLen) {
return pReq;
}
+static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) {
+ SMStreamTickReq timerReq = {
+ .tick = sec,
+ };
+
+ int32_t contLen = tSerializeSMStreamTickMsg(NULL, 0, &timerReq);
+ if (contLen <= 0) return NULL;
+ void *pReq = rpcMallocCont(contLen);
+ if (pReq == NULL) return NULL;
+
+ tSerializeSMStreamTickMsg(pReq, contLen, &timerReq);
+ *pContLen = contLen;
+ return pReq;
+}
+
static void mndPullupTrans(SMnode *pMnode) {
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
@@ -105,7 +120,24 @@ static void mndCalMqRebalance(SMnode *pMnode) {
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) {
- SRpcMsg rpcMsg = {.msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen};
+ SRpcMsg rpcMsg = {
+ .msgType = TDMT_MND_TMQ_TIMER,
+ .pCont = pReq,
+ .contLen = contLen,
+ };
+ tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
+ }
+}
+
+static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) {
+ int32_t contLen = 0;
+ void *pReq = mndBuildCheckpointTickMsg(&contLen, sec);
+ if (pReq != NULL) {
+ SRpcMsg rpcMsg = {
+ .msgType = TDMT_MND_STREAM_CHECKPOINT_TIMER,
+ .pCont = pReq,
+ .contLen = contLen,
+ };
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
}
}
@@ -150,12 +182,16 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs)
bool roleChanged = false;
for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
- if (pVgroup->vnodeGid[vg].dnodeId == dnodeId) {
- if (pVgroup->vnodeGid[vg].syncState != TAOS_SYNC_STATE_OFFLINE) {
- mInfo("vgId:%d, state changed by offline check, old state:%s restored:%d new state:error restored:0",
- pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore);
- pVgroup->vnodeGid[vg].syncState = TAOS_SYNC_STATE_OFFLINE;
- pVgroup->vnodeGid[vg].syncRestore = 0;
+ SVnodeGid *pGid = &pVgroup->vnodeGid[vg];
+ if (pGid->dnodeId == dnodeId) {
+ if (pGid->syncState != TAOS_SYNC_STATE_OFFLINE) {
+ mInfo(
+ "vgId:%d, state changed by offline check, old state:%s restored:%d canRead:%d new state:error restored:0 "
+ "canRead:0",
+ pVgroup->vgId, syncStr(pGid->syncState), pGid->syncRestore, pGid->syncCanRead);
+ pGid->syncState = TAOS_SYNC_STATE_OFFLINE;
+ pGid->syncRestore = 0;
+ pGid->syncCanRead = 0;
roleChanged = true;
}
break;
@@ -220,6 +256,12 @@ static void *mndThreadFp(void *param) {
mndCalMqRebalance(pMnode);
}
+#if 0
+ if (sec % tsStreamCheckpointTickInterval == 0) {
+ mndStreamCheckpointTick(pMnode, sec);
+ }
+#endif
+
if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) {
mndPullupTelem(pMnode);
}
@@ -491,7 +533,7 @@ void mndPreClose(SMnode *pMnode) {
if (pMnode != NULL) {
syncLeaderTransfer(pMnode->syncMgmt.sync);
syncPreStop(pMnode->syncMgmt.sync);
-
+#if 0
while (syncSnapshotRecving(pMnode->syncMgmt.sync)) {
mInfo("vgId:1, snapshot is recving");
taosMsleep(300);
@@ -500,6 +542,7 @@ void mndPreClose(SMnode *pMnode) {
mInfo("vgId:1, snapshot is sending");
taosMsleep(300);
}
+#endif
}
}
@@ -556,11 +599,45 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) {
return 0;
}
- if (mndAcquireRpc(pMsg->info.node) == 0) return 0;
- SMnode *pMnode = pMsg->info.node;
+ SMnode *pMnode = pMsg->info.node;
+ taosThreadRwlockRdlock(&pMnode->lock);
+ if (pMnode->stopped) {
+ taosThreadRwlockUnlock(&pMnode->lock);
+ terrno = TSDB_CODE_APP_IS_STOPPING;
+ return -1;
+ }
+
+ terrno = 0;
SSyncState state = syncGetState(pMnode->syncMgmt.sync);
+ if (terrno != 0) {
+ taosThreadRwlockUnlock(&pMnode->lock);
+ return -1;
+ }
+ if (state.state != TAOS_SYNC_STATE_LEADER) {
+ taosThreadRwlockUnlock(&pMnode->lock);
+ terrno = TSDB_CODE_SYN_NOT_LEADER;
+ goto _OVER;
+ }
+
+ if (!state.restored || !pMnode->restored) {
+ taosThreadRwlockUnlock(&pMnode->lock);
+ terrno = TSDB_CODE_SYN_RESTORING;
+ goto _OVER;
+ }
+
+#if 1
+ atomic_add_fetch_32(&pMnode->rpcRef, 1);
+#else
+ int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1);
+ mTrace("mnode rpc is acquired, ref:%d", ref);
+#endif
+
+ taosThreadRwlockUnlock(&pMnode->lock);
+ return 0;
+
+_OVER:
if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER ||
pMsg->msgType == TDMT_MND_UPTIME_TIMER) {
@@ -573,42 +650,26 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
SEpSet epSet = {0};
mndGetMnodeEpSet(pMnode, &epSet);
- mDebug(
+ mGDebug(
"msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d "
"role:%s, redirect numOfEps:%d inUse:%d",
pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored,
syncStr(state.restored), epSet.numOfEps, epSet.inUse);
- if (epSet.numOfEps > 0) {
- for (int32_t i = 0; i < epSet.numOfEps; ++i) {
- mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
- }
+ if (epSet.numOfEps <= 0) return -1;
- int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
- pMsg->info.rsp = rpcMallocCont(contLen);
- if (pMsg->info.rsp != NULL) {
- tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet);
- pMsg->info.hasEpSet = 1;
- pMsg->info.rspLen = contLen;
- terrno = TSDB_CODE_RPC_REDIRECT;
- } else {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- }
- } else {
- terrno = TSDB_CODE_APP_NOT_READY;
+ for (int32_t i = 0; i < epSet.numOfEps; ++i) {
+ mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
}
- return -1;
-}
+ int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
+ pMsg->info.rsp = rpcMallocCont(contLen);
+ if (pMsg->info.rsp != NULL) {
+ tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet);
+ pMsg->info.hasEpSet = 1;
+ pMsg->info.rspLen = contLen;
+ }
-static int32_t mndCheckMsgContent(SRpcMsg *pMsg) {
- if (!IsReq(pMsg)) return 0;
- if (pMsg->contLen != 0 && pMsg->pCont != NULL) return 0;
-
- const STraceId *trace = &pMsg->info.traceId;
- mGError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen,
- pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
- terrno = TSDB_CODE_INVALID_MSG_LEN;
return -1;
}
@@ -623,7 +684,6 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
return -1;
}
- if (mndCheckMsgContent(pMsg) != 0) return -1;
if (mndCheckMnodeState(pMsg) != 0) return -1;
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c
index 7c86a5be22..9c847c1138 100644
--- a/source/dnode/mnode/impl/src/mndMnode.c
+++ b/source/dnode/mnode/impl/src/mndMnode.c
@@ -142,6 +142,8 @@ _OVER:
static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SMnodeObj *pObj = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
@@ -151,10 +153,10 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SMnodeObj));
+ pRow = sdbAllocRow(sizeof(SMnodeObj));
if (pRow == NULL) goto _OVER;
- SMnodeObj *pObj = sdbGetRowObj(pRow);
+ pObj = sdbGetRowObj(pRow);
if (pObj == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -167,7 +169,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("mnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr());
+ mError("mnode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
@@ -290,7 +292,7 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p
.pCont = pReq,
.contLen = contLen,
.msgType = TDMT_DND_CREATE_MNODE,
- .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED,
+ .acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@@ -331,7 +333,7 @@ static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDrop
.pCont = pReq,
.contLen = contLen,
.msgType = TDMT_DND_DROP_MNODE,
- .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED,
+ .acceptableCode = TSDB_CODE_MNODE_NOT_DEPLOYED,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@@ -438,7 +440,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
}
if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
- terrno = TSDB_CODE_NODE_OFFLINE;
+ terrno = TSDB_CODE_DNODE_OFFLINE;
goto _OVER;
}
@@ -488,7 +490,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
if (totalMnodes == 2) {
if (force) {
mError("cant't force drop dnode, since a mnode on it and replica is 2");
- terrno = TSDB_CODE_NODE_OFFLINE;
+ terrno = TSDB_CODE_DNODE_OFFLINE;
return -1;
}
mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
@@ -572,7 +574,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
}
if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) {
- terrno = TSDB_CODE_NODE_OFFLINE;
+ terrno = TSDB_CODE_DNODE_OFFLINE;
goto _OVER;
}
diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c
index e874c1a53e..cf463304a1 100644
--- a/source/dnode/mnode/impl/src/mndPerfSchema.c
+++ b/source/dnode/mnode/impl/src/mndPerfSchema.c
@@ -68,7 +68,7 @@ int32_t mndPerfsInitMeta(SHashObj *hash) {
int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) {
if (NULL == pMnode->perfsMeta) {
- terrno = TSDB_CODE_APP_NOT_READY;
+ terrno = TSDB_CODE_APP_ERROR;
return -1;
}
@@ -94,7 +94,7 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char
int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
if (NULL == pMnode->perfsMeta) {
- terrno = TSDB_CODE_APP_NOT_READY;
+ terrno = TSDB_CODE_APP_ERROR;
return -1;
}
diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c
index 151a2a6404..ccb4140b83 100644
--- a/source/dnode/mnode/impl/src/mndPrivilege.c
+++ b/source/dnode/mnode/impl/src/mndPrivilege.c
@@ -28,6 +28,10 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType
int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) {
return 0;
}
+int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic) { return 0; }
+int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName) {
+ return 0;
+}
int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) {
memcpy(pRsp->user, pUser->user, TSDB_USER_LEN);
pRsp->superAuth = 1;
diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c
index 434e6fbc52..28a5dee2db 100644
--- a/source/dnode/mnode/impl/src/mndQnode.c
+++ b/source/dnode/mnode/impl/src/mndQnode.c
@@ -100,6 +100,8 @@ _OVER:
static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SQnodeObj *pObj = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -109,10 +111,10 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SQnodeObj));
+ pRow = sdbAllocRow(sizeof(SQnodeObj));
if (pRow == NULL) goto _OVER;
- SQnodeObj *pObj = sdbGetRowObj(pRow);
+ pObj = sdbGetRowObj(pRow);
if (pObj == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -125,7 +127,7 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("qnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr());
+ mError("qnode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
@@ -203,7 +205,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_CREATE_QNODE;
- action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
@@ -230,7 +232,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_DROP_QNODE;
- action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
@@ -343,7 +345,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_DROP_QNODE;
- action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c
index 20c2ebb0a4..ec2e844ba5 100644
--- a/source/dnode/mnode/impl/src/mndShow.c
+++ b/source/dnode/mnode/impl/src/mndShow.c
@@ -108,6 +108,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_APPS;
} else if (strncasecmp(name, TSDB_INS_TABLE_STREAM_TASKS, len) == 0) {
type = TSDB_MGMT_TABLE_STREAM_TASKS;
+ } else if (strncasecmp(name, TSDB_INS_TABLE_USER_PRIVILEGES, len) == 0) {
+ type = TSDB_MGMT_TABLE_PRIVILEGES;
} else {
// ASSERT(0);
}
diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c
index 698c07d9bc..f9d0a9b9ba 100644
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@@ -132,6 +132,8 @@ _OVER:
static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SSmaObj *pSma = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -141,10 +143,10 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SSmaObj));
+ pRow = sdbAllocRow(sizeof(SSmaObj));
if (pRow == NULL) goto _OVER;
- SSmaObj *pSma = sdbGetRowObj(pRow);
+ pSma = sdbGetRowObj(pRow);
if (pSma == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -200,7 +202,7 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr());
+ mError("sma:%s, failed to decode from raw:%p since %s", pSma == NULL ? "null" : pSma->name, pRaw, terrstr());
taosMemoryFreeClear(pSma->expr);
taosMemoryFreeClear(pSma->tagsFilter);
taosMemoryFreeClear(pSma->sql);
@@ -461,7 +463,7 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans,
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_CREATE_VNODE;
- action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
@@ -778,7 +780,7 @@ static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SD
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_DROP_VNODE;
- action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_VND_NOT_EXIST;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c
index 7753e5b127..e6a253bcc0 100644
--- a/source/dnode/mnode/impl/src/mndSnode.c
+++ b/source/dnode/mnode/impl/src/mndSnode.c
@@ -105,6 +105,8 @@ _OVER:
static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SSnodeObj *pObj = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -114,10 +116,10 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SSnodeObj));
+ pRow = sdbAllocRow(sizeof(SSnodeObj));
if (pRow == NULL) goto _OVER;
- SSnodeObj *pObj = sdbGetRowObj(pRow);
+ pObj = sdbGetRowObj(pRow);
if (pObj == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -130,7 +132,7 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("snode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr());
+ mError("snode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
@@ -208,7 +210,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_CREATE_SNODE;
- action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
@@ -235,7 +237,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_DROP_SNODE;
- action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
@@ -350,7 +352,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_DROP_SNODE;
- action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 6dad7d74c8..2ed4fde509 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -162,6 +162,8 @@ _OVER:
static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SStbObj *pStb = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -171,10 +173,10 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SStbObj));
+ pRow = sdbAllocRow(sizeof(SStbObj));
if (pRow == NULL) goto _OVER;
- SStbObj *pStb = sdbGetRowObj(pRow);
+ pStb = sdbGetRowObj(pRow);
if (pStb == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -254,10 +256,12 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("stb:%s, failed to decode from raw:%p since %s", pStb->name, pRaw, terrstr());
- taosMemoryFreeClear(pStb->pColumns);
- taosMemoryFreeClear(pStb->pTags);
- taosMemoryFreeClear(pStb->comment);
+ mError("stb:%s, failed to decode from raw:%p since %s", pStb == NULL ? "null" : pStb->name, pRaw, terrstr());
+ if (pStb != NULL) {
+ taosMemoryFreeClear(pStb->pColumns);
+ taosMemoryFreeClear(pStb->pTags);
+ taosMemoryFreeClear(pStb->comment);
+ }
taosMemoryFreeClear(pRow);
return NULL;
}
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index d8cf7a837e..7ee688d220 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -28,7 +28,7 @@
#include "parser.h"
#include "tname.h"
-#define MND_STREAM_VER_NUMBER 1
+#define MND_STREAM_VER_NUMBER 2
#define MND_STREAM_RESERVE_SIZE 64
static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
@@ -36,6 +36,8 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream);
static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
+static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
+static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/
static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq);
static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
@@ -62,6 +64,10 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp);
+ mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr);
+ mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
+ mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
+
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
@@ -120,21 +126,22 @@ STREAM_ENCODE_OVER:
SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
- void *buf = NULL;
+ SSdbRow *pRow = NULL;
+ SStreamObj *pStream = NULL;
+ void *buf = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER;
- if (sver != MND_STREAM_VER_NUMBER) {
+ if (sver != 1 && sver != 2) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto STREAM_DECODE_OVER;
}
- int32_t size = sizeof(SStreamObj);
- SSdbRow *pRow = sdbAllocRow(size);
+ pRow = sdbAllocRow(sizeof(SStreamObj));
if (pRow == NULL) goto STREAM_DECODE_OVER;
- SStreamObj *pStream = sdbGetRowObj(pRow);
+ pStream = sdbGetRowObj(pRow);
if (pStream == NULL) goto STREAM_DECODE_OVER;
int32_t tlen;
@@ -146,7 +153,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
SDecoder decoder;
tDecoderInit(&decoder, buf, tlen + 1);
- if (tDecodeSStreamObj(&decoder, pStream) < 0) {
+ if (tDecodeSStreamObj(&decoder, pStream, sver) < 0) {
tDecoderClear(&decoder);
goto STREAM_DECODE_OVER;
}
@@ -157,7 +164,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
STREAM_DECODE_OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
- mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr());
+ mError("stream:%s, failed to decode from raw:%p since %s", pStream == NULL ? "null" : pStream->name, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
@@ -679,6 +686,164 @@ _OVER:
tFreeStreamObj(&streamObj);
return code;
}
+static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
+ SMnode *pMnode = pReq->info.node;
+ SSdb *pSdb = pMnode->pSdb;
+ void *pIter = NULL;
+ SStreamObj *pStream = NULL;
+
+ // iterate all stream obj
+ while (1) {
+ pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
+ if (pIter == NULL) break;
+ // incr tick
+ int64_t currentTick = atomic_add_fetch_64(&pStream->currentTick, 1);
+ // if >= checkpointFreq, build msg TDMT_MND_STREAM_BEGIN_CHECKPOINT, put into write q
+ if (currentTick >= pStream->checkpointFreq) {
+ atomic_store_64(&pStream->currentTick, 0);
+ SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
+
+ pMsg->streamId = pStream->uid;
+ pMsg->checkpointId = tGenIdPI64();
+ memcpy(pMsg->streamName, pStream->name, TSDB_STREAM_FNAME_LEN);
+
+ SRpcMsg rpcMsg = {
+ .msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT,
+ .pCont = pMsg,
+ .contLen = sizeof(SMStreamDoCheckpointMsg),
+ };
+
+ tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+ }
+ }
+
+ return 0;
+}
+
+static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, const SStreamTask *pTask,
+ SMStreamDoCheckpointMsg *pMsg) {
+ SStreamCheckpointSourceReq req = {0};
+ req.checkpointId = pMsg->checkpointId;
+ req.nodeId = pTask->nodeId;
+ req.expireTime = -1;
+ req.streamId = pTask->streamId;
+ req.taskId = pTask->taskId;
+
+ int32_t code;
+ int32_t blen;
+
+ tEncodeSize(tEncodeSStreamCheckpointSourceReq, &req, blen, code);
+ if (code < 0) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ int32_t tlen = sizeof(SMsgHead) + blen;
+
+ void *buf = taosMemoryMalloc(tlen);
+ if (buf == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+ SEncoder encoder;
+ tEncoderInit(&encoder, abuf, tlen);
+ tEncodeSStreamCheckpointSourceReq(&encoder, &req);
+
+ SMsgHead *pMsgHead = (SMsgHead *)buf;
+ pMsgHead->contLen = htonl(tlen);
+ pMsgHead->vgId = htonl(pTask->nodeId);
+
+ tEncoderClear(&encoder);
+
+ *pBuf = buf;
+ *pLen = tlen;
+
+ return 0;
+}
+
+static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
+ SMnode *pMnode = pReq->info.node;
+ SSdb *pSdb = pMnode->pSdb;
+
+ SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont;
+
+ SStreamObj *pStream = mndAcquireStream(pMnode, pMsg->streamName);
+
+ if (pStream == NULL || pStream->uid != pMsg->streamId) {
+ mError("start checkpointing failed since stream %s not found", pMsg->streamName);
+ return -1;
+ }
+
+ // build new transaction:
+ STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "stream-checkpoint");
+ if (pTrans == NULL) return -1;
+ mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
+ taosRLockLatch(&pStream->lock);
+ // 1. redo action: broadcast checkpoint source msg for all source vg
+ int32_t totLevel = taosArrayGetSize(pStream->tasks);
+ for (int32_t i = 0; i < totLevel; i++) {
+ SArray *pLevel = taosArrayGetP(pStream->tasks, i);
+ SStreamTask *pTask = taosArrayGetP(pLevel, 0);
+ if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
+ int32_t sz = taosArrayGetSize(pLevel);
+ for (int32_t j = 0; j < sz; j++) {
+ SStreamTask *pTask = taosArrayGetP(pLevel, j);
+ ASSERT(pTask->nodeId > 0);
+ SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId);
+ if (pVgObj == NULL) {
+ ASSERT(0);
+ taosRUnLockLatch(&pStream->lock);
+ mndReleaseStream(pMnode, pStream);
+ mndTransDrop(pTrans);
+ return -1;
+ }
+
+ void *buf;
+ int32_t tlen;
+ if (mndBuildStreamCheckpointSourceReq(&buf, &tlen, pTask, pMsg) < 0) {
+ taosRUnLockLatch(&pStream->lock);
+ mndReleaseStream(pMnode, pStream);
+ mndTransDrop(pTrans);
+ return -1;
+ }
+
+ STransAction action = {0};
+ action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
+ action.pCont = buf;
+ action.contLen = tlen;
+ action.msgType = TDMT_VND_STREAM_CHECK_POINT_SOURCE;
+
+ mndReleaseVgroup(pMnode, pVgObj);
+
+ if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+ taosMemoryFree(buf);
+ taosRUnLockLatch(&pStream->lock);
+ mndReleaseStream(pMnode, pStream);
+ mndTransDrop(pTrans);
+ return -1;
+ }
+ }
+ }
+ }
+ // 2. reset tick
+ atomic_store_64(&pStream->currentTick, 0);
+ // 3. commit log: stream checkpoint info
+ taosRUnLockLatch(&pStream->lock);
+
+ if (mndTransPrepare(pMnode, pTrans) != 0) {
+ mError("failed to prepare trans rebalance since %s", terrstr());
+ mndTransDrop(pTrans);
+ mndReleaseStream(pMnode, pStream);
+ return -1;
+ }
+
+ mndReleaseStream(pMnode, pStream);
+ mndTransDrop(pTrans);
+
+ return 0;
+}
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
@@ -747,71 +912,6 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return TSDB_CODE_ACTION_IN_PROGRESS;
}
-#if 0
-static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) {
- SMnode *pMnode = pReq->info.node;
- SStreamObj *pStream = NULL;
- /*SDbObj *pDb = NULL;*/
- /*SUserObj *pUser = NULL;*/
-
- SMRecoverStreamReq recoverReq = {0};
- if (tDeserializeSMRecoverStreamReq(pReq->pCont, pReq->contLen, &recoverReq) < 0) {
- ASSERT(0);
- terrno = TSDB_CODE_INVALID_MSG;
- return -1;
- }
-
- pStream = mndAcquireStream(pMnode, recoverReq.name);
-
- if (pStream == NULL) {
- if (recoverReq.igNotExists) {
- mInfo("stream:%s, not exist, ignore not exist is set", recoverReq.name);
- sdbRelease(pMnode->pSdb, pStream);
- return 0;
- } else {
- terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
- return -1;
- }
- }
-
- if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) {
- return -1;
- }
-
- STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq);
- if (pTrans == NULL) {
- mError("stream:%s, failed to recover since %s", recoverReq.name, terrstr());
- sdbRelease(pMnode->pSdb, pStream);
- return -1;
- }
- mInfo("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name);
-
- // broadcast to recover all tasks
- if (mndRecoverStreamTasks(pMnode, pTrans, pStream) < 0) {
- mError("stream:%s, failed to recover task since %s", recoverReq.name, terrstr());
- sdbRelease(pMnode->pSdb, pStream);
- return -1;
- }
-
- // update stream status
- if (mndSetStreamRecover(pMnode, pTrans, pStream) < 0) {
- sdbRelease(pMnode->pSdb, pStream);
- return -1;
- }
-
- if (mndTransPrepare(pMnode, pTrans) != 0) {
- mError("trans:%d, failed to prepare recover stream trans since %s", pTrans->id, terrstr());
- sdbRelease(pMnode->pSdb, pStream);
- mndTransDrop(pTrans);
- return -1;
- }
-
- sdbRelease(pMnode->pSdb, pStream);
-
- return TSDB_CODE_ACTION_IN_PROGRESS;
-}
-#endif
-
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
@@ -846,13 +946,6 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
}
}
-#if 0
- if (mndSetDropOffsetStreamLogs(pMnode, pTrans, pStream) < 0) {
- sdbRelease(pSdb, pStream);
- goto END;
- }
-#endif
-
sdbRelease(pSdb, pStream);
}
diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c
index ffb46e5f1b..55e073a8a4 100644
--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@@ -440,9 +440,9 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
}
static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) {
- STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "persist-reb");
- mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL);
+ STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
if (pTrans == NULL) return -1;
+ mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL);
// make txn:
// 1. redo action: action to all vg
@@ -523,28 +523,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
tDeleteSMqConsumerObj(pConsumerNew);
taosMemoryFree(pConsumerNew);
}
-#if 0
- if (consumerNum) {
- char topic[TSDB_TOPIC_FNAME_LEN];
- char cgroup[TSDB_CGROUP_LEN];
- mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true);
- SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
- if (pTopic) {
- // TODO make topic complete
- SMqTopicObj topicObj = {0};
- memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
- topicObj.refConsumerCnt = pTopic->refConsumerCnt - consumerNum;
- // TODO is that correct?
- pTopic->refConsumerCnt = topicObj.refConsumerCnt;
- mInfo("subscribe topic %s unref %d consumer cgroup %s, refcnt %d", pTopic->name, consumerNum, cgroup,
- topicObj.refConsumerCnt);
- if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) {
- ASSERT(0);
- goto REB_FAIL;
- }
- }
- }
-#endif
// 4. TODO commit log: modification log
@@ -743,7 +721,9 @@ SUB_ENCODE_OVER:
static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
- void *buf = NULL;
+ SSdbRow *pRow = NULL;
+ SMqSubscribeObj *pSub = NULL;
+ void *buf = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER;
@@ -753,11 +733,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
goto SUB_DECODE_OVER;
}
- int32_t size = sizeof(SMqSubscribeObj);
- SSdbRow *pRow = sdbAllocRow(size);
+ pRow = sdbAllocRow(sizeof(SMqSubscribeObj));
if (pRow == NULL) goto SUB_DECODE_OVER;
- SMqSubscribeObj *pSub = sdbGetRowObj(pRow);
+ pSub = sdbGetRowObj(pRow);
if (pSub == NULL) goto SUB_DECODE_OVER;
int32_t dataPos = 0;
@@ -777,7 +756,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
SUB_DECODE_OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
- mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr());
+ mError("subscribe:%s, failed to decode from raw:%p since %s", pSub == NULL ? "null" : pSub->key, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c
index 2e3c04eb78..895d7fedc4 100644
--- a/source/dnode/mnode/impl/src/mndSync.c
+++ b/source/dnode/mnode/impl/src/mndSync.c
@@ -321,7 +321,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
if (pMgmt->transId != 0) {
mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId);
taosWUnLockLatch(&pMgmt->lock);
- terrno = TSDB_CODE_APP_NOT_READY;
+ terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED;
return -1;
}
@@ -342,13 +342,11 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
sdbSetApplyInfo(pMnode->pSdb, req.info.conn.applyIndex, req.info.conn.applyTerm, SYNC_INDEX_INVALID);
code = 0;
} else {
- mInfo("trans:%d, failed to proposed since %s", transId, terrstr());
+ mError("trans:%d, failed to proposed since %s", transId, terrstr());
taosWLockLatch(&pMgmt->lock);
pMgmt->transId = 0;
taosWUnLockLatch(&pMgmt->lock);
- if (terrno == TSDB_CODE_SYN_NOT_LEADER) {
- terrno = TSDB_CODE_APP_NOT_READY;
- } else {
+ if (terrno == 0) {
terrno = TSDB_CODE_APP_ERROR;
}
}
@@ -384,20 +382,23 @@ void mndSyncStop(SMnode *pMnode) {
}
bool mndIsLeader(SMnode *pMnode) {
+ terrno = 0;
SSyncState state = syncGetState(pMnode->syncMgmt.sync);
- if (state.state != TAOS_SYNC_STATE_LEADER || !state.restored) {
- if (state.state != TAOS_SYNC_STATE_LEADER) {
- terrno = TSDB_CODE_SYN_NOT_LEADER;
- } else {
- terrno = TSDB_CODE_APP_NOT_READY;
- }
- mDebug("vgId:1, mnode not ready, state:%s, restore:%d", syncStr(state.state), state.restored);
+ if (terrno != 0) {
+ mDebug("vgId:1, mnode is stopping");
return false;
}
- if (!mndGetRestored(pMnode)) {
- terrno = TSDB_CODE_APP_NOT_READY;
+ if (state.state != TAOS_SYNC_STATE_LEADER) {
+ terrno = TSDB_CODE_SYN_NOT_LEADER;
+ mDebug("vgId:1, mnode not leader, state:%s", syncStr(state.state));
+ return false;
+ }
+
+ if (!state.restored || !pMnode->restored) {
+ terrno = TSDB_CODE_SYN_RESTORING;
+ mDebug("vgId:1, mnode not restored:%d:%d", state.restored, pMnode->restored);
return false;
}
diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c
index 6412761f0b..eea74c6dd8 100644
--- a/source/dnode/mnode/impl/src/mndTopic.c
+++ b/source/dnode/mnode/impl/src/mndTopic.c
@@ -152,8 +152,10 @@ TOPIC_ENCODE_OVER:
SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SMqTopicObj *pTopic = NULL;
+ void *buf = NULL;
- void *buf = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER;
@@ -162,11 +164,10 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
goto TOPIC_DECODE_OVER;
}
- int32_t size = sizeof(SMqTopicObj);
- SSdbRow *pRow = sdbAllocRow(size);
+ pRow = sdbAllocRow(sizeof(SMqTopicObj));
if (pRow == NULL) goto TOPIC_DECODE_OVER;
- SMqTopicObj *pTopic = sdbGetRowObj(pRow);
+ pTopic = sdbGetRowObj(pRow);
if (pTopic == NULL) goto TOPIC_DECODE_OVER;
int32_t len;
@@ -251,7 +252,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
TOPIC_DECODE_OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
- mError("topic:%s, failed to decode from raw:%p since %s", pTopic->name, pRaw, terrstr());
+ mError("topic:%s, failed to decode from raw:%p since %s", pTopic == NULL ? "null" : pTopic->name, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
@@ -288,7 +289,7 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic
return 0;
}
-SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName) {
+SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName) {
SSdb *pSdb = pMnode->pSdb;
SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName);
if (pTopic == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@@ -573,7 +574,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
goto _OVER;
}
- if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) {
+ if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC) != 0) {
goto _OVER;
}
@@ -633,6 +634,11 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
}
}
+ if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC) != 0) {
+ mndReleaseTopic(pMnode, pTopic);
+ return -1;
+ }
+
void *pIter = NULL;
SMqConsumerObj *pConsumer;
while (1) {
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 27c58dfba1..4b11884050 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -918,10 +918,13 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
sendRsp = true;
}
} else {
- if (pTrans->stage == TRN_STAGE_REDO_ACTION && ((code == TSDB_CODE_APP_NOT_READY && pTrans->failedTimes > 60) ||
- (code != TSDB_CODE_APP_NOT_READY && pTrans->failedTimes > 6))) {
+ if (pTrans->stage == TRN_STAGE_REDO_ACTION) {
+ if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_APP_IS_STARTING) {
+ if (pTrans->failedTimes > 60) sendRsp = true;
+ } else {
+ if (pTrans->failedTimes > 6) sendRsp = true;
+ }
if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
- sendRsp = true;
}
}
@@ -942,7 +945,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL;
}
if (i != 0 && code == 0) {
- code = TSDB_CODE_RPC_REDIRECT;
+ code = TSDB_CODE_MNODE_NOT_FOUND;
}
mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage),
pInfo->ahandle);
@@ -1039,8 +1042,8 @@ static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pA
pAction->rawWritten = 0;
pAction->msgSent = 0;
pAction->msgReceived = 0;
- if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR ||
- pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
+ if (pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR ||
+ pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps;
mInfo("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage),
pAction->id, pAction->epSet.inUse);
diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c
index 2f50ab04b8..806ba0c98e 100644
--- a/source/dnode/mnode/impl/src/mndUser.c
+++ b/source/dnode/mnode/impl/src/mndUser.c
@@ -18,10 +18,11 @@
#include "mndDb.h"
#include "mndPrivilege.h"
#include "mndShow.h"
+#include "mndTopic.h"
#include "mndTrans.h"
#include "tbase64.h"
-#define USER_VER_NUMBER 1
+#define USER_VER_NUMBER 2
#define USER_RESERVE_SIZE 64
static int32_t mndCreateDefaultUsers(SMnode *pMnode);
@@ -36,6 +37,8 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq);
static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq);
static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextUser(SMnode *pMnode, void *pIter);
+static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
+static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter);
int32_t mndInitUser(SMnode *pMnode) {
SSdbTable table = {
@@ -56,6 +59,8 @@ int32_t mndInitUser(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER, mndRetrieveUsers);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_USER, mndCancelGetNextUser);
+ mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_PRIVILEGES, mndRetrievePrivileges);
+ mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_PRIVILEGES, mndCancelGetNextPrivileges);
return sdbSetTable(pMnode->pSdb, table);
}
@@ -119,7 +124,9 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
- int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
+ int32_t numOfTopics = taosHashGetSize(pUser->topics);
+ int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN +
+ numOfTopics * TSDB_TOPIC_FNAME_LEN;
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER;
@@ -137,6 +144,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER)
+ SDB_SET_INT32(pRaw, dataPos, numOfTopics, _OVER)
char *db = taosHashIterate(pUser->readDbs, NULL);
while (db != NULL) {
@@ -150,6 +158,12 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
db = taosHashIterate(pUser->writeDbs, db);
}
+ char *topic = taosHashIterate(pUser->topics, NULL);
+ while (topic != NULL) {
+ SDB_SET_BINARY(pRaw, dataPos, topic, TSDB_TOPIC_FNAME_LEN, _OVER);
+ topic = taosHashIterate(pUser->topics, topic);
+ }
+
SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@@ -168,19 +182,21 @@ _OVER:
static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SUserObj *pUser = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
- if (sver != USER_VER_NUMBER) {
+ if (sver != 1 && sver != 2) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj));
+ pRow = sdbAllocRow(sizeof(SUserObj));
if (pRow == NULL) goto _OVER;
- SUserObj *pUser = sdbGetRowObj(pRow);
+ pUser = sdbGetRowObj(pRow);
if (pUser == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -197,12 +213,18 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
int32_t numOfReadDbs = 0;
int32_t numOfWriteDbs = 0;
+ int32_t numOfTopics = 0;
SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER)
+ if (sver >= 2) {
+ SDB_GET_INT32(pRaw, dataPos, &numOfTopics, _OVER)
+ }
+
pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->writeDbs =
taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
- if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto _OVER;
+ pUser->topics = taosHashInit(numOfTopics, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+ if (pUser->readDbs == NULL || pUser->writeDbs == NULL || pUser->topics == NULL) goto _OVER;
for (int32_t i = 0; i < numOfReadDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0};
@@ -218,6 +240,15 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN);
}
+ if (sver >= 2) {
+ for (int32_t i = 0; i < numOfTopics; ++i) {
+ char topic[TSDB_TOPIC_FNAME_LEN] = {0};
+ SDB_GET_BINARY(pRaw, dataPos, topic, TSDB_TOPIC_FNAME_LEN, _OVER)
+ int32_t len = strlen(topic) + 1;
+ taosHashPut(pUser->topics, topic, len, topic, TSDB_TOPIC_FNAME_LEN);
+ }
+ }
+
SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pUser->lock);
@@ -225,9 +256,12 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr());
- taosHashCleanup(pUser->readDbs);
- taosHashCleanup(pUser->writeDbs);
+ mError("user:%s, failed to decode from raw:%p since %s", pUser == NULL ? "null" : pUser->user, pRaw, terrstr());
+ if (pUser != NULL) {
+ taosHashCleanup(pUser->readDbs);
+ taosHashCleanup(pUser->writeDbs);
+ taosHashCleanup(pUser->topics);
+ }
taosMemoryFreeClear(pRow);
return NULL;
}
@@ -255,8 +289,10 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser);
taosHashCleanup(pUser->readDbs);
taosHashCleanup(pUser->writeDbs);
+ taosHashCleanup(pUser->topics);
pUser->readDbs = NULL;
pUser->writeDbs = NULL;
+ pUser->topics = NULL;
return 0;
}
@@ -270,6 +306,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN);
TSWAP(pOld->readDbs, pNew->readDbs);
TSWAP(pOld->writeDbs, pNew->writeDbs);
+ TSWAP(pOld->topics, pNew->topics);
taosWUnLockLatch(&pOld->lock);
return 0;
@@ -409,7 +446,7 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc
return 0;
}
-SHashObj *mndDupDbHash(SHashObj *pOld) {
+SHashObj *mndDupObjHash(SHashObj *pOld, int32_t dataLen) {
SHashObj *pNew =
taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pNew == NULL) {
@@ -420,7 +457,7 @@ SHashObj *mndDupDbHash(SHashObj *pOld) {
char *db = taosHashIterate(pOld, NULL);
while (db != NULL) {
int32_t len = strlen(db) + 1;
- if (taosHashPut(pNew, db, len, db, TSDB_DB_FNAME_LEN) != 0) {
+ if (taosHashPut(pNew, db, len, db, dataLen) != 0) {
taosHashCancelIterate(pOld, db);
taosHashCleanup(pNew);
terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -432,6 +469,10 @@ SHashObj *mndDupDbHash(SHashObj *pOld) {
return pNew;
}
+SHashObj *mndDupDbHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_DB_FNAME_LEN); }
+
+SHashObj *mndDupTopicHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_TOPIC_FNAME_LEN); }
+
static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
@@ -482,9 +523,10 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
taosRLockLatch(&pUser->lock);
newUser.readDbs = mndDupDbHash(pUser->readDbs);
newUser.writeDbs = mndDupDbHash(pUser->writeDbs);
+ newUser.topics = mndDupTopicHash(pUser->topics);
taosRUnLockLatch(&pUser->lock);
- if (newUser.readDbs == NULL || newUser.writeDbs == NULL) {
+ if (newUser.readDbs == NULL || newUser.writeDbs == NULL || newUser.topics == NULL) {
goto _OVER;
}
@@ -582,6 +624,26 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
}
}
+ if (alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) {
+ int32_t len = strlen(alterReq.objname) + 1;
+ SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
+ if (pTopic == NULL) {
+ mndReleaseTopic(pMnode, pTopic);
+ goto _OVER;
+ }
+ taosHashPut(newUser.topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
+ }
+
+ if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC) {
+ int32_t len = strlen(alterReq.objname) + 1;
+ SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
+ if (pTopic == NULL) {
+ mndReleaseTopic(pMnode, pTopic);
+ goto _OVER;
+ }
+ taosHashRemove(newUser.topics, alterReq.objname, len);
+ }
+
code = mndAlterUser(pMnode, pUser, &newUser, pReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
@@ -594,6 +656,7 @@ _OVER:
mndReleaseUser(pMnode, pUser);
taosHashCleanup(newUser.writeDbs);
taosHashCleanup(newUser.readDbs);
+ taosHashCleanup(newUser.topics);
return code;
}
@@ -756,6 +819,128 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) {
sdbCancelFetch(pSdb, pIter);
}
+static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
+ SMnode *pMnode = pReq->info.node;
+ SSdb *pSdb = pMnode->pSdb;
+ int32_t numOfRows = 0;
+ SUserObj *pUser = NULL;
+ int32_t cols = 0;
+ char *pWrite;
+
+ while (numOfRows < rows) {
+ pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser);
+ if (pShow->pIter == NULL) break;
+
+ int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
+ int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
+ int32_t numOfTopics = taosHashGetSize(pUser->topics);
+ if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics >= rows) break;
+
+ if (pUser->superUser) {
+ cols = 0;
+ SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
+ STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
+ colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
+
+ char privilege[20] = {0};
+ STR_WITH_MAXSIZE_TO_VARSTR(privilege, "all", pShow->pMeta->pSchemas[cols].bytes);
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
+
+ char objName[20] = {0};
+ STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes);
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
+
+ numOfRows++;
+ }
+
+ char *db = taosHashIterate(pUser->readDbs, NULL);
+ while (db != NULL) {
+ cols = 0;
+ SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
+ STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
+ colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
+
+ char privilege[20] = {0};
+ STR_WITH_MAXSIZE_TO_VARSTR(privilege, "read", pShow->pMeta->pSchemas[cols].bytes);
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
+
+ SName name = {0};
+ char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
+ tNameFromString(&name, db, T_NAME_ACCT | T_NAME_DB);
+ tNameGetDbName(&name, varDataVal(objName));
+ varDataSetLen(objName, strlen(varDataVal(objName)));
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
+
+ numOfRows++;
+ db = taosHashIterate(pUser->readDbs, db);
+ }
+
+ db = taosHashIterate(pUser->writeDbs, NULL);
+ while (db != NULL) {
+ cols = 0;
+ SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
+ STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
+ colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
+
+ char privilege[20] = {0};
+ STR_WITH_MAXSIZE_TO_VARSTR(privilege, "write", pShow->pMeta->pSchemas[cols].bytes);
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
+
+ SName name = {0};
+ char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
+ tNameFromString(&name, db, T_NAME_ACCT | T_NAME_DB);
+ tNameGetDbName(&name, varDataVal(objName));
+ varDataSetLen(objName, strlen(varDataVal(objName)));
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
+
+ numOfRows++;
+ db = taosHashIterate(pUser->writeDbs, db);
+ }
+
+ char *topic = taosHashIterate(pUser->topics, NULL);
+ while (topic != NULL) {
+ cols = 0;
+ SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
+ STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
+ colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
+
+ char privilege[20] = {0};
+ STR_WITH_MAXSIZE_TO_VARSTR(privilege, "subscribe", pShow->pMeta->pSchemas[cols].bytes);
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
+
+ char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0};
+ tstrncpy(varDataVal(topicName), mndGetDbStr(topic), TSDB_TOPIC_NAME_LEN - 2);
+ varDataSetLen(topicName, strlen(varDataVal(topicName)));
+ pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+ colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
+
+ numOfRows++;
+ topic = taosHashIterate(pUser->topics, topic);
+ }
+
+ sdbRelease(pSdb, pUser);
+ }
+
+ pShow->numOfRows += numOfRows;
+ return numOfRows;
+}
+
+static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter) {
+ SSdb *pSdb = pMnode->pSdb;
+ sdbCancelFetch(pSdb, pIter);
+}
+
int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp,
int32_t *pRspLen) {
SUserAuthBatchRsp batchRsp = {0};
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index d06853e470..161bf7536f 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -113,6 +113,8 @@ _OVER:
SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
+ SSdbRow *pRow = NULL;
+ SVgObj *pVgroup = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -122,10 +124,10 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
goto _OVER;
}
- SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj));
+ pRow = sdbAllocRow(sizeof(SVgObj));
if (pRow == NULL) goto _OVER;
- SVgObj *pVgroup = sdbGetRowObj(pRow);
+ pVgroup = sdbGetRowObj(pRow);
if (pVgroup == NULL) goto _OVER;
int32_t dataPos = 0;
@@ -152,7 +154,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
_OVER:
if (terrno != 0) {
- mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr());
+ mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup == NULL ? 0 : pVgroup->vgId, pRaw, terrstr());
taosMemoryFreeClear(pRow);
return NULL;
}
@@ -186,6 +188,7 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) {
if (pNewGid->dnodeId == pOldGid->dnodeId) {
pNewGid->syncState = pOldGid->syncState;
pNewGid->syncRestore = pOldGid->syncRestore;
+ pNewGid->syncCanRead = pOldGid->syncCanRead;
}
}
}
@@ -696,8 +699,16 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
if (!exist) {
strcpy(role, "dropping");
} else if (online) {
- bool show = (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER && !pVgroup->vnodeGid[i].syncRestore);
- snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), show ? "*" : "");
+ char *star = "";
+ if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER) {
+ if (!pVgroup->vnodeGid[i].syncRestore && !pVgroup->vnodeGid[i].syncCanRead) {
+ star = "**";
+ } else if (!pVgroup->vnodeGid[i].syncRestore && pVgroup->vnodeGid[i].syncCanRead) {
+ star = "*";
+ } else {
+ }
+ }
+ snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), star);
} else {
}
STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes);
@@ -987,7 +998,7 @@ int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVg
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_CREATE_VNODE;
- action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
@@ -1087,7 +1098,7 @@ int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_DROP_VNODE;
- action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_VND_NOT_EXIST;
if (isRedo) {
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp
index 46a9a465eb..1f59a7fcca 100644
--- a/source/dnode/mnode/impl/test/acct/acct.cpp
+++ b/source/dnode/mnode/impl/test/acct/acct.cpp
@@ -32,7 +32,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
}
TEST_F(MndTestAcct, 02_Alter_Acct) {
@@ -42,7 +42,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
}
TEST_F(MndTestAcct, 03_Drop_Acct) {
@@ -52,5 +52,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
- ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
+ ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
}
diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp
index 60be7cfbc0..89c2b6931a 100644
--- a/source/dnode/mnode/impl/test/trans/trans2.cpp
+++ b/source/dnode/mnode/impl/test/trans/trans2.cpp
@@ -173,7 +173,7 @@ class MndTestTrans2 : public ::testing::Test {
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_CREATE_MNODE;
- action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED;
mndTransAppendRedoAction(pTrans, &action);
}
@@ -190,7 +190,7 @@ class MndTestTrans2 : public ::testing::Test {
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_DND_CREATE_MNODE;
- action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+ action.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED;
mndTransAppendUndoAction(pTrans, &action);
}
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index aa55204ae5..b133226ed3 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -168,7 +168,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) {
SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg;
- streamMetaRemoveTask1(pSnode->pMeta, pReq->taskId);
+ streamMetaRemoveTask(pSnode->pMeta, pReq->taskId);
return 0;
}
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 7ef3207b4d..c4040644b1 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -106,14 +106,24 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList,
int32_t metaReadNext(SMetaReader *pReader);
const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
-int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
-int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
-bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
-int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
- bool *acquired);
-int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
- int32_t payloadLen, double selectivityRatio);
-int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
+
+int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
+int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
+int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
+bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
+int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
+ bool *acquired);
+int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
+ int32_t payloadLen, double selectivityRatio);
+int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
+tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
+int64_t metaGetTbNum(SMeta *pMeta);
+int64_t metaGetNtbNum(SMeta *pMeta);
+typedef struct {
+ int64_t uid;
+ int64_t ctbNum;
+} SMetaStbStats;
+int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo);
typedef struct SMetaFltParam {
tb_uid_t suid;
@@ -159,20 +169,19 @@ typedef struct STsdbReader STsdbReader;
int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
- STsdbReader **ppReader, const char *idstr);
+ SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr);
-void tsdbReaderClose(STsdbReader *pReader);
-bool tsdbNextDataBlock(STsdbReader *pReader);
-bool tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid);
-void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
-int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
-SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
-int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
-int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
-int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
-void *tsdbGetIdx(SMeta *pMeta);
-void *tsdbGetIvtIdx(SMeta *pMeta);
-uint64_t getReaderMaxVersion(STsdbReader *pReader);
+void tsdbReaderClose(STsdbReader *pReader);
+bool tsdbNextDataBlock(STsdbReader *pReader);
+void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
+int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockSMA, bool *allHave);
+SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
+int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
+int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
+int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
+void *tsdbGetIdx(SMeta *pMeta);
+void *tsdbGetIvtIdx(SMeta *pMeta);
+uint64_t getReaderMaxVersion(STsdbReader *pReader);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
uint64_t suid, void **pReader);
diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h
index e2b7327e8f..15e64ffaed 100644
--- a/source/dnode/vnode/src/inc/vnd.h
+++ b/source/dnode/vnode/src/inc/vnd.h
@@ -100,7 +100,7 @@ int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
int32_t vnodeSyncStart(SVnode* pVnode);
void vnodeSyncPreClose(SVnode* pVnode);
void vnodeSyncClose(SVnode* pVnode);
-void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg);
+void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code);
bool vnodeIsLeader(SVnode* pVnode);
bool vnodeIsRoleLeader(SVnode* pVnode);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index f229b3b127..6451d15969 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -116,8 +116,6 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in
int metaGetTableEntryByName(SMetaReader* pReader, const char* name);
int metaAlterCache(SMeta* pMeta, int32_t nPage);
-tb_uid_t metaGetTableEntryUidByName(SMeta* pMeta, const char* name);
-int64_t metaGetTbNum(SMeta* pMeta);
int64_t metaGetTimeSeriesNum(SMeta* pMeta);
SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid, int lock);
void metaCloseCtbCursor(SMCtbCursor* pCtbCur, int lock);
@@ -144,12 +142,6 @@ typedef struct SMetaInfo {
} SMetaInfo;
int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader);
-typedef struct {
- int64_t uid;
- int64_t ctbNum;
-} SMetaStbStats;
-int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo);
-
// tsdb
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback);
int tsdbClose(STsdb** pTsdb);
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index 0257aede3d..cfdb4ab8d1 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -223,6 +223,23 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) {
return 0;
}
+
+int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) {
+ int code = 0;
+ SMetaReader mr = {0};
+ metaReaderInit(&mr, (SMeta *)meta, 0);
+ code = metaGetTableEntryByUid(&mr, uid);
+ if (code < 0) {
+ metaReaderClear(&mr);
+ return -1;
+ }
+ strncpy(tbName, mr.me.name, TSDB_TABLE_NAME_LEN);
+ metaReaderClear(&mr);
+
+ return 0;
+}
+
+
int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) {
int code = 0;
SMetaReader mr = {0};
@@ -739,6 +756,10 @@ int64_t metaGetTimeSeriesNum(SMeta *pMeta) {
return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries;
}
+int64_t metaGetNtbNum(SMeta *pMeta) {
+ return pMeta->pVnode->config.vndStats.numOfNTables;
+}
+
typedef struct {
SMeta *pMeta;
TBC *pCur;
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 3d9ebec4c9..003b387838 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -1425,7 +1425,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
- streamMetaRemoveTask1(pTq->pStreamMeta, pReq->taskId);
+ streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId);
return 0;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index d71eb33951..7309ea3407 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -228,23 +228,23 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST
invalidate = true;
break;
- }
- } else {
- SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
- if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
- SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
- taosMemoryFree(pLastCol->colVal.value.pData);
+ } else { // new inserting key is greater than cached, update cached entry
+ SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
+ if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
+ SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
+ taosMemoryFree(pLastCol->colVal.value.pData);
- lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
- if (lastCol.colVal.value.pData == NULL) {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _invalidate;
+ lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
+ if (lastCol.colVal.value.pData == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _invalidate;
+ }
+ memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
}
- memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
- }
- taosArraySet(pLast, iCol, &lastCol);
+ taosArraySet(pLast, iCol, &lastCol);
+ }
}
}
}
@@ -253,65 +253,10 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST
taosMemoryFreeClear(pTSchema);
taosLRUCacheRelease(pCache, h, invalidate);
- /*
- cacheRow = (STSRow *)taosLRUCacheValue(pCache, h);
- if (row->ts >= cacheRow->ts) {
- if (row->ts == cacheRow->ts) {
- STSRow *mergedRow = NULL;
- SRowMerger merger = {0};
- STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1);
-
- tRowMergerInit(&merger, &tsdbRowFromTSRow(0, cacheRow), pTSchema);
-
- tRowMerge(&merger, &tsdbRowFromTSRow(1, row));
-
- tRowMergerGetRow(&merger, &mergedRow);
- tRowMergerClear(&merger);
-
- taosMemoryFreeClear(pTSchema);
-
- row = mergedRow;
- dup = false;
- }
-
- if (TD_ROW_LEN(row) <= TD_ROW_LEN(cacheRow)) {
- tdRowCpy(cacheRow, row);
- if (!dup) {
- taosMemoryFree(row);
- }
-
- taosLRUCacheRelease(pCache, h, false);
- } else {
- taosLRUCacheRelease(pCache, h, true);
- // tsdbCacheDeleteLastrow(pCache, uid, TSKEY_MAX);
- if (dup) {
- cacheRow = tdRowDup(row);
- } else {
- cacheRow = row;
- }
- _taos_lru_deleter_t deleter = deleteTableCacheLastrow;
- LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL,
- TAOS_LRU_PRIORITY_LOW);
- if (status != TAOS_LRU_STATUS_OK) {
- code = -1;
- }
- // tsdbCacheInsertLastrow(pCache, uid, row, dup);
- }
- }*/
- } /*else {
- if (dup) {
- cacheRow = tdRowDup(row);
- } else {
- cacheRow = row;
+ if (invalidate) {
+ taosLRUCacheErase(pCache, key, keyLen);
}
-
- _taos_lru_deleter_t deleter = deleteTableCacheLastrow;
- LRUStatus status =
- taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW);
- if (status != TAOS_LRU_STATUS_OK) {
- code = -1;
- }
- }*/
+ }
return code;
}
@@ -349,28 +294,28 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
SColVal colVal = {0};
tTSRowGetVal(row, pTSchema, iCol, &colVal);
- if (!COL_VAL_IS_VALUE(&colVal)) {
+ if (COL_VAL_IS_VALUE(&colVal)) {
if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) {
invalidate = true;
break;
- }
- } else {
- SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
- if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
- SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
- taosMemoryFree(pLastCol->colVal.value.pData);
+ } else {
+ SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
+ if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
+ SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
+ taosMemoryFree(pLastCol->colVal.value.pData);
- lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
- if (lastCol.colVal.value.pData == NULL) {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _invalidate;
+ lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
+ if (lastCol.colVal.value.pData == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _invalidate;
+ }
+ memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
}
- memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
- }
- taosArraySet(pLast, iCol, &lastCol);
+ taosArraySet(pLast, iCol, &lastCol);
+ }
}
}
}
@@ -379,9 +324,9 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
taosMemoryFreeClear(pTSchema);
taosLRUCacheRelease(pCache, h, invalidate);
-
- // clear last cache anyway, lazy load when get last lookup
- // taosLRUCacheRelease(pCache, h, true);
+ if (invalidate) {
+ taosLRUCacheErase(pCache, key, keyLen);
+ }
}
return code;
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 83532c2a20..d9f0c55d24 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -84,8 +84,8 @@ typedef struct SIOCostSummary {
typedef struct SBlockLoadSuppInfo {
SArray* pColAgg;
SColumnDataAgg tsColAgg;
- SColumnDataAgg** plist;
- int16_t* colIds; // column ids for loading file block data
+ int16_t* colId;
+ int16_t* slotId;
int32_t numOfCols;
char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated.
bool smaValid; // the sma on all queried columns are activated
@@ -158,6 +158,7 @@ struct STsdbReader {
STsdb* pTsdb;
uint64_t suid;
int16_t order;
+ bool freeBlock;
STimeWindow window; // the primary query time window that applies to all queries
SSDataBlock* pResBlock;
int32_t capacity;
@@ -214,25 +215,25 @@ static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFil
static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
-static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SSDataBlock* pBlock) {
- size_t numOfCols = blockDataGetNumOfCols(pBlock);
-
+static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) {
pSupInfo->smaValid = true;
pSupInfo->numOfCols = numOfCols;
- pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
- pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES);
- if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL) {
- taosMemoryFree(pSupInfo->colIds);
- taosMemoryFree(pSupInfo->buildBuf);
+ pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t)*2 + POINTER_BYTES));
+ if (pSupInfo->colId == NULL) {
+ taosMemoryFree(pSupInfo->colId);
return TSDB_CODE_OUT_OF_MEMORY;
}
+ pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols));
+ pSupInfo->buildBuf = (char**) ((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols));
for (int32_t i = 0; i < numOfCols; ++i) {
- SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
- pSupInfo->colIds[i] = pCol->info.colId;
+ pSupInfo->colId[i] = pCols[i].colId;
+ pSupInfo->slotId[i] = pSlotIdList[i];
- if (IS_VAR_DATA_TYPE(pCol->info.type)) {
- pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes);
+ if (IS_VAR_DATA_TYPE(pCols[i].type)) {
+ pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes);
+ } else {
+ pSupInfo->buildBuf[i] = NULL;
}
}
@@ -244,7 +245,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) {
STColumn* pTCol = &pSchema->columns[i];
- if (pTCol->colId == pSupInfo->colIds[j]) {
+ if (pTCol->colId == pSupInfo->colId[j]) {
if (!IS_BSMA_ON(pTCol)) {
pSupInfo->smaValid = false;
return;
@@ -252,7 +253,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
i += 1;
j += 1;
- } else if (pTCol->colId < pSupInfo->colIds[j]) {
+ } else if (pTCol->colId < pSupInfo->colId[j]) {
// do nothing
i += 1;
} else {
@@ -454,7 +455,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
if (pLReader->pInfo == NULL) {
// here we ignore the first column, which is always be the primary timestamp column
pLReader->pInfo =
- tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1);
+ tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1);
if (pLReader->pInfo == NULL) {
tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
return terrno;
@@ -566,7 +567,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity)
}
static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity,
- const char* idstr) {
+ SSDataBlock* pResBlock, const char* idstr) {
int32_t code = 0;
int8_t level = 0;
STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader));
@@ -585,6 +586,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
pReader->suid = pCond->suid;
pReader->order = pCond->order;
pReader->capacity = capacity;
+ pReader->pResBlock = pResBlock;
pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL;
pReader->verRange = getQueryVerRange(pVnode, pCond, level);
pReader->type = pCond->type;
@@ -592,13 +594,22 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket
ASSERT(pCond->numOfCols > 0);
+ if (pReader->pResBlock == NULL) {
+ pReader->freeBlock = true;
+ pReader->pResBlock = createResBlock(pCond, pReader->capacity);
+ if (pReader->pResBlock == NULL) {
+ code = terrno;
+ goto _end;
+ }
+ }
+
+ // todo refactor.
limitOutputBufferSize(pCond, &pReader->capacity);
// allocate buffer in order to load data blocks from file
SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
pSup->pColAgg = taosArrayInit(pCond->numOfCols, sizeof(SColumnDataAgg));
- pSup->plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES);
- if (pSup->pColAgg == NULL || pSup->plist == NULL) {
+ if (pSup->pColAgg == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
@@ -611,13 +622,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
goto _end;
}
- pReader->pResBlock = createResBlock(pCond, pReader->capacity);
- if (pReader->pResBlock == NULL) {
- code = terrno;
- goto _end;
- }
-
- setColumnIdSlotList(&pReader->suppInfo, pReader->pResBlock);
+ setColumnIdSlotList(&pReader->suppInfo, pCond->colList, pCond->pSlotList, pCond->numOfCols);
*ppReader = pReader;
return code;
@@ -1044,17 +1049,16 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo
}
static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) {
- SReaderStatus* pStatus = &pReader->status;
- SDataBlockIter* pBlockIter = &pStatus->blockIter;
+ SReaderStatus* pStatus = &pReader->status;
+ SDataBlockIter* pBlockIter = &pStatus->blockIter;
+ SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
+ SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SBlockData* pBlockData = &pStatus->fileBlockData;
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
SDataBlk* pBlock = getCurrentBlock(pBlockIter);
SSDataBlock* pResBlock = pReader->pResBlock;
- int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock);
-
- SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
- SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
+ int32_t numOfOutputCols = pSupInfo->numOfCols;
SColVal cv = {0};
int64_t st = taosGetTimestampUs();
@@ -1090,8 +1094,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
int32_t i = 0;
int32_t rowIndex = 0;
- SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
- if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+ SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
+ if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
i += 1;
}
@@ -1100,12 +1104,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
int32_t num = pBlockData->nColData;
while (i < numOfOutputCols && colIndex < num) {
rowIndex = 0;
- pColData = taosArrayGet(pResBlock->pDataBlock, i);
SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
- if (pData->cid < pColData->info.colId) {
+ if (pData->cid < pSupInfo->colId[i]) {
colIndex += 1;
- } else if (pData->cid == pColData->info.colId) {
+ } else if (pData->cid == pSupInfo->colId[i]) {
+ pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
+
if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
colDataAppendNNULL(pColData, 0, dumpedRows);
} else {
@@ -1122,6 +1127,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
colIndex += 1;
i += 1;
} else { // the specified column does not exist in file block, fill with null data
+ pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
colDataAppendNNULL(pColData, 0, dumpedRows);
i += 1;
}
@@ -1129,7 +1135,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
// fill the mis-matched columns with null value
while (i < numOfOutputCols) {
- pColData = taosArrayGet(pResBlock->pDataBlock, i);
+ pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
colDataAppendNNULL(pColData, 0, dumpedRows);
i += 1;
}
@@ -1167,7 +1173,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
tBlockDataReset(pBlockData);
TABLEID tid = {.suid = pReader->suid, .uid = uid};
int32_t code =
- tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1);
+ tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@@ -1626,7 +1632,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
int64_t st = taosGetTimestampUs();
int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader);
- blockDataUpdateTsWindow(pBlock, 0);
+ blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]);
pBlock->info.id.uid = pBlockScanInfo->uid;
setComposedBlockFlag(pReader, true);
@@ -2498,7 +2504,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
_end:
pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
- blockDataUpdateTsWindow(pResBlock, 0);
+ blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]);
setComposedBlockFlag(pReader, true);
double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -3539,8 +3545,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow,
STableBlockScanInfo* pScanInfo) {
- int32_t numOfRows = pBlock->info.rows;
- int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
+ int32_t outputRowIndex = pBlock->info.rows;
int64_t uid = pScanInfo->uid;
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
@@ -3549,23 +3554,26 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
SColVal colVal = {0};
int32_t i = 0, j = 0;
- SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
- if (pColInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
- colDataAppend(pColInfoData, numOfRows, (const char*)&pTSRow->ts, false);
+ if (pSupInfo->colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+ SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
+ ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;
i += 1;
}
- while (i < numOfCols && j < pSchema->numOfCols) {
- pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
- col_id_t colId = pColInfoData->info.colId;
+ while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) {
+ col_id_t colId = pSupInfo->colId[i];
if (colId == pSchema->columns[j].colId) {
+ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
+
tTSRowGetVal(pTSRow, pSchema, j, &colVal);
- doCopyColVal(pColInfoData, numOfRows, i, &colVal, pSupInfo);
+ doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
i += 1;
j += 1;
} else if (colId < pSchema->columns[j].colId) {
- colDataAppendNULL(pColInfoData, numOfRows);
+ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
+
+ colDataAppendNULL(pColInfoData, outputRowIndex);
i += 1;
} else if (colId > pSchema->columns[j].colId) {
j += 1;
@@ -3573,9 +3581,9 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
}
// set null value since current column does not exist in the "pSchema"
- while (i < numOfCols) {
- pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
- colDataAppendNULL(pColInfoData, numOfRows);
+ while (i < pSupInfo->numOfCols) {
+ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
+ colDataAppendNULL(pColInfoData, outputRowIndex);
i += 1;
}
@@ -3590,27 +3598,25 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
int32_t outputRowIndex = pResBlock->info.rows;
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-
- SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
- if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
- colDataAppendInt64(pColData, outputRowIndex, &pBlockData->aTSKEY[rowIndex]);
+ if (pReader->suppInfo.colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+ SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
+ ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
i += 1;
}
SColVal cv = {0};
int32_t numOfInputCols = pBlockData->nColData;
- int32_t numOfOutputCols = pResBlock->pDataBlock->size;
+ int32_t numOfOutputCols = pSupInfo->numOfCols;
while (i < numOfOutputCols && j < numOfInputCols) {
- SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, i);
- SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
-
- if (pData->cid < pCol->info.colId) {
+ SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
+ if (pData->cid < pSupInfo->colId[i]) {
j += 1;
continue;
}
- if (pData->cid == pCol->info.colId) {
+ SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]);
+ if (pData->cid == pSupInfo->colId[i]) {
tColDataGetValue(pData, rowIndex, &cv);
doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
j += 1;
@@ -3623,7 +3629,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
}
while (i < numOfOutputCols) {
- SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i);
+ SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
colDataAppendNULL(pCol, outputRowIndex);
i += 1;
}
@@ -3722,14 +3728,21 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
// ====================================== EXPOSED APIs ======================================
int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
- STsdbReader** ppReader, const char* idstr) {
+ SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) {
STimeWindow window = pCond->twindows;
if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) {
pCond->twindows.skey += 1;
pCond->twindows.ekey -= 1;
}
- int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, 4096, idstr);
+ int32_t capacity = 0;
+ if (pResBlock == NULL) {
+ capacity = 4096;
+ } else {
+ capacity = pResBlock->info.capacity;
+ }
+
+ int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
@@ -3755,7 +3768,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
}
// here we only need one more row, so the capacity is set to be ONE.
- code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, idstr);
+ code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, pResBlock, idstr);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
@@ -3769,7 +3782,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
}
pCond->order = order;
- code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, idstr);
+ code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, pResBlock, idstr);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
@@ -3841,10 +3854,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
return code;
-_err:
+ _err:
tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr);
return code;
-}
+ }
void tsdbReaderClose(STsdbReader* pReader) {
if (pReader == NULL) {
@@ -3874,19 +3887,19 @@ void tsdbReaderClose(STsdbReader* pReader) {
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
- taosMemoryFreeClear(pSupInfo->plist);
- taosMemoryFree(pSupInfo->colIds);
-
taosArrayDestroy(pSupInfo->pColAgg);
- for (int32_t i = 0; i < blockDataGetNumOfCols(pReader->pResBlock); ++i) {
+ for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) {
if (pSupInfo->buildBuf[i] != NULL) {
taosMemoryFreeClear(pSupInfo->buildBuf[i]);
}
}
- taosMemoryFree(pSupInfo->buildBuf);
- tBlockDataDestroy(&pReader->status.fileBlockData, true);
+ if (pReader->freeBlock) {
+ pReader->pResBlock = blockDataDestroy(pReader->pResBlock);
+ }
+ taosMemoryFree(pSupInfo->colId);
+ tBlockDataDestroy(&pReader->status.fileBlockData, true);
cleanupDataBlockIterator(&pReader->status.blockIter);
size_t numOfTables = taosHashGetSize(pReader->status.pTableMap);
@@ -3895,8 +3908,6 @@ void tsdbReaderClose(STsdbReader* pReader) {
clearBlockScanInfoBuf(&pReader->blockInfoBuf);
}
- blockDataDestroy(pReader->pResBlock);
-
if (pReader->pFileReader != NULL) {
tsdbDataFReaderClose(&pReader->pFileReader);
}
@@ -4011,16 +4022,6 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
return false;
}
-bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) {
- STableBlockScanInfo* pBlockScanInfo =
- *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid));
- if (pBlockScanInfo == NULL) { // no data block for the table of given uid
- return false;
- }
-
- return true;
-}
-
static void setBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) {
ASSERT(pReader != NULL);
*rows = pReader->pResBlock->info.rows;
@@ -4042,28 +4043,27 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64
}
}
-int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) {
+int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockSMA, bool* allHave) {
int32_t code = 0;
*allHave = false;
if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
- *pBlockStatis = NULL;
+ *pBlockSMA = NULL;
return TSDB_CODE_SUCCESS;
}
// there is no statistics data for composed block
if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) {
- *pBlockStatis = NULL;
+ *pBlockSMA = NULL;
return TSDB_CODE_SUCCESS;
}
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
-
- SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
- // int64_t stime = taosGetTimestampUs();
-
SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
+ ASSERT(pReader->pResBlock->info.id.uid == pFBlock->uid);
+
+ SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
if (tDataBlkHasSma(pBlock)) {
code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg);
if (code != TSDB_CODE_SUCCESS) {
@@ -4072,7 +4072,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
return code;
}
} else {
- *pBlockStatis = NULL;
+ *pBlockSMA = NULL;
return TSDB_CODE_SUCCESS;
}
@@ -4085,80 +4085,53 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
pTsAgg->min = pReader->pResBlock->info.window.skey;
pTsAgg->max = pReader->pResBlock->info.window.ekey;
- pSup->plist[0] = pTsAgg;
// update the number of NULL data rows
- size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock);
+ size_t numOfCols = pSup->numOfCols;
int32_t i = 0, j = 0;
size_t size = taosArrayGetSize(pSup->pColAgg);
-#if 0
- while (j < numOfCols && i < size) {
- SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
- if (pAgg->colId == pSup->colIds[j]) {
- if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) {
- pSup->plist[j] = pAgg;
- } else {
- *allHave = false;
- break;
- }
- i += 1;
- j += 1;
- } else if (pAgg->colId < pSup->colIds[j]) {
- i += 1;
- } else if (pSup->colIds[j] < pAgg->colId) {
- j += 1;
- }
+
+ SSDataBlock* pResBlock = pReader->pResBlock;
+ if (pResBlock->pBlockAgg == NULL) {
+ size_t num = taosArrayGetSize(pResBlock->pDataBlock);
+ pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg));
}
-#else
-
- // fill the all null data column
- SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg));
while (j < numOfCols && i < size) {
SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
- if (pAgg->colId == pSup->colIds[j]) {
- taosArrayPush(pNewAggList, pAgg);
+ if (pAgg->colId == pSup->colId[j]) {
+ pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg;
i += 1;
j += 1;
- } else if (pAgg->colId < pSup->colIds[j]) {
+ } else if (pAgg->colId < pSup->colId[j]) {
i += 1;
- } else if (pSup->colIds[j] < pAgg->colId) {
- if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) {
- taosArrayPush(pNewAggList, &pSup->tsColAgg);
+ } else if (pSup->colId[j] < pAgg->colId) {
+ if (pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID) {
+ pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg;
} else {
// all date in this block are null
- SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow};
- taosArrayPush(pNewAggList, &nullColAgg);
+ SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = pBlock->nRow};
+ taosArrayPush(pSup->pColAgg, &nullColAgg);
+
+ pResBlock->pBlockAgg[pSup->slotId[j]] = taosArrayGetLast(pSup->pColAgg);
}
j += 1;
}
}
- taosArrayClear(pSup->pColAgg);
- taosArrayAddAll(pSup->pColAgg, pNewAggList);
-
- size_t num = taosArrayGetSize(pSup->pColAgg);
- for(int32_t k = 0; k < num; ++k) {
- pSup->plist[k] = taosArrayGet(pSup->pColAgg, k);
- }
-
- taosArrayDestroy(pNewAggList);
-
-#endif
-
+ *pBlockSMA = pResBlock->pBlockAgg;
pReader->cost.smaDataLoad += 1;
- *pBlockStatis = pSup->plist;
tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr);
return code;
}
-static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
+static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status;
if (pStatus->composedDataBlock) {
- return pReader->pResBlock->pDataBlock;
+ return pReader->pResBlock;
}
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter);
@@ -4179,10 +4152,10 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
}
copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
- return pReader->pResBlock->pDataBlock;
+ return pReader->pResBlock;
}
-SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
+SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
if (pReader->step == EXTERNAL_ROWS_PREV) {
return doRetrieveDataBlock(pReader->innerReader[0]);
@@ -4209,7 +4182,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
// allocate buffer in order to load data blocks from file
memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg));
- memset(pReader->suppInfo.plist, 0, POINTER_BYTES);
pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID;
tsdbDataFReaderClose(&pReader->pFileReader);
diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index 8e9aab0afd..1199127f6d 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -380,6 +380,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
pLoad->vgId = TD_VID(pVnode);
pLoad->syncState = state.state;
pLoad->syncRestore = state.restored;
+ pLoad->syncCanRead = state.canRead;
pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index d63c45fc0e..fc6b126cc5 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -344,7 +344,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
vTrace("message in vnode query queue is processing");
// if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsLeader(pVnode)) {
if ((pMsg->msgType == TDMT_SCH_QUERY) && !syncIsReadyForRead(pVnode->sync)) {
- vnodeRedirectRpcMsg(pVnode, pMsg);
+ vnodeRedirectRpcMsg(pVnode, pMsg, terrno);
return 0;
}
@@ -367,12 +367,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
pMsg->msgType == TDMT_VND_BATCH_META) &&
!syncIsReadyForRead(pVnode->sync)) {
// !vnodeIsLeader(pVnode)) {
- vnodeRedirectRpcMsg(pVnode, pMsg);
+ vnodeRedirectRpcMsg(pVnode, pMsg, terrno);
return 0;
}
if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !pVnode->restored) {
- vnodeRedirectRpcMsg(pVnode, pMsg);
+ vnodeRedirectRpcMsg(pVnode, pMsg, TSDB_CODE_SYN_RESTORING);
return 0;
}
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index 55ba0d3e6f..20ae16ae43 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "vnd.h"
-#define BATCH_DISABLE 1
+#define BATCH_ENABLE 0
static inline bool vnodeIsMsgBlock(tmsg_t type) {
return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) ||
@@ -53,7 +53,7 @@ static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
}
}
-void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) {
+void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) {
SEpSet newEpSet = {0};
syncGetRetryEpSet(pVnode->sync, &newEpSet);
@@ -66,8 +66,20 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) {
}
pMsg->info.hasEpSet = 1;
- SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1};
- tmsgSendRedirectRsp(&rsp, &newEpSet);
+ if (code == 0) code = TSDB_CODE_SYN_NOT_LEADER;
+
+ SRpcMsg rsp = {.code = code, .info = pMsg->info, .msgType = pMsg->msgType + 1};
+ int32_t contLen = tSerializeSEpSet(NULL, 0, &newEpSet);
+
+ rsp.pCont = rpcMallocCont(contLen);
+ if (rsp.pCont == NULL) {
+ pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
+ } else {
+ tSerializeSEpSet(rsp.pCont, contLen, &newEpSet);
+ rsp.contLen = contLen;
+ }
+
+ tmsgSendRsp(&rsp);
}
static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
@@ -87,8 +99,8 @@ static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
}
static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) {
- if (code == TSDB_CODE_SYN_NOT_LEADER) {
- vnodeRedirectRpcMsg(pVnode, pMsg);
+ if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) {
+ vnodeRedirectRpcMsg(pVnode, pMsg, code);
} else {
const STraceId *trace = &pMsg->info.traceId;
vGError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", pVnode->config.vgId, pMsg, tstrerror(code), code);
@@ -99,15 +111,12 @@ static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code)
}
}
+#if BATCH_ENABLE
+
static void inline vnodeProposeBatchMsg(SVnode *pVnode, SRpcMsg **pMsgArr, bool *pIsWeakArr, int32_t *arrSize) {
if (*arrSize <= 0) return;
-#if BATCH_DISABLE
- int32_t code = syncPropose(pVnode->sync, pMsgArr[0], pIsWeakArr[0]);
-#else
int32_t code = syncProposeBatch(pVnode->sync, pMsgArr, pIsWeakArr, *arrSize);
-#endif
-
if (code > 0) {
for (int32_t i = 0; i < *arrSize; ++i) {
vnodeHandleWriteMsg(pVnode, pMsgArr[i]);
@@ -153,8 +162,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
if (!pVnode->restored) {
vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg);
- terrno = TSDB_CODE_APP_NOT_READY;
- vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_APP_NOT_READY);
+ terrno = TSDB_CODE_SYN_RESTORING;
+ vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
continue;
@@ -177,7 +186,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
continue;
}
- if (isBlock || BATCH_DISABLE) {
+ if (isBlock) {
vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos);
}
@@ -185,7 +194,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
pIsWeakArr[arrayPos] = isWeak;
arrayPos++;
- if (isBlock || msg == numOfMsgs - 1 || BATCH_DISABLE) {
+ if (isBlock || msg == numOfMsgs - 1) {
vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos);
}
}
@@ -194,6 +203,65 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
taosMemoryFree(pIsWeakArr);
}
+#else
+
+static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) {
+ int32_t code = syncPropose(pVnode->sync, pMsg, isWeak);
+ if (code > 0) {
+ vnodeHandleWriteMsg(pVnode, pMsg);
+ } else if (code == 0) {
+ vnodeWaitBlockMsg(pVnode, pMsg);
+ } else {
+ if (terrno != 0) code = terrno;
+ vnodeHandleProposeError(pVnode, pMsg, code);
+ }
+
+ return code;
+}
+
+void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+ SVnode *pVnode = pInfo->ahandle;
+ int32_t vgId = pVnode->config.vgId;
+ int32_t code = 0;
+ SRpcMsg *pMsg = NULL;
+ vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs);
+
+ for (int32_t msg = 0; msg < numOfMsgs; msg++) {
+ if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
+ bool isWeak = vnodeIsMsgWeak(pMsg->msgType);
+
+ const STraceId *trace = &pMsg->info.traceId;
+ vGTrace("vgId:%d, msg:%p get from vnode-write queue, weak:%d block:%d msg:%d:%d, handle:%p", vgId, pMsg, isWeak,
+ vnodeIsMsgBlock(pMsg->msgType), msg, numOfMsgs, pMsg->info.handle);
+
+ if (!pVnode->restored) {
+ vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg);
+ vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING);
+ rpcFreeCont(pMsg->pCont);
+ taosFreeQitem(pMsg);
+ continue;
+ }
+
+ code = vnodePreProcessWriteMsg(pVnode, pMsg);
+ if (code != 0) {
+ vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr());
+ if (terrno != 0) code = terrno;
+ vnodeHandleProposeError(pVnode, pMsg, code);
+ rpcFreeCont(pMsg->pCont);
+ taosFreeQitem(pMsg);
+ continue;
+ }
+
+ code = vnodeProposeMsg(pVnode, pMsg, isWeak);
+
+ vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
+ rpcFreeCont(pMsg->pCont);
+ taosFreeQitem(pMsg);
+ }
+}
+
+#endif
+
void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnode *pVnode = pInfo->ahandle;
int32_t vgId = pVnode->config.vgId;
@@ -515,7 +583,7 @@ void vnodeSyncPreClose(SVnode *pVnode) {
vInfo("vgId:%d, pre close sync", pVnode->config.vgId);
syncLeaderTransfer(pVnode->sync);
syncPreStop(pVnode->sync);
-
+#if 0
while (syncSnapshotRecving(pVnode->sync)) {
vInfo("vgId:%d, snapshot is recving", pVnode->config.vgId);
taosMsleep(300);
@@ -524,7 +592,7 @@ void vnodeSyncPreClose(SVnode *pVnode) {
vInfo("vgId:%d, snapshot is sending", pVnode->config.vgId);
taosMsleep(300);
}
-
+#endif
taosThreadMutexLock(&pVnode->lock);
if (pVnode->blocked) {
vInfo("vgId:%d, post block after close sync", pVnode->config.vgId);
@@ -545,21 +613,23 @@ bool vnodeIsRoleLeader(SVnode *pVnode) {
}
bool vnodeIsLeader(SVnode *pVnode) {
+ terrno = 0;
SSyncState state = syncGetState(pVnode->sync);
- if (state.state != TAOS_SYNC_STATE_LEADER || !state.restored) {
- if (state.state != TAOS_SYNC_STATE_LEADER) {
- terrno = TSDB_CODE_SYN_NOT_LEADER;
- } else {
- terrno = TSDB_CODE_APP_NOT_READY;
- }
- vInfo("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored);
+ if (terrno != 0) {
+ vInfo("vgId:%d, vnode is stopping", pVnode->config.vgId);
return false;
}
- if (!pVnode->restored) {
- vInfo("vgId:%d, vnode not restored", pVnode->config.vgId);
- terrno = TSDB_CODE_APP_NOT_READY;
+ if (state.state != TAOS_SYNC_STATE_LEADER) {
+ terrno = TSDB_CODE_SYN_NOT_LEADER;
+ vInfo("vgId:%d, vnode not leader, state:%s", pVnode->config.vgId, syncStr(state.state));
+ return false;
+ }
+
+ if (!state.restored || !pVnode->restored) {
+ terrno = TSDB_CODE_SYN_RESTORING;
+ vInfo("vgId:%d, vnode not restored:%d:%d", pVnode->config.vgId, state.restored, pVnode->restored);
return false;
}
diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h
index 4d0c5389e1..6acf19218d 100644
--- a/source/libs/command/inc/commandInt.h
+++ b/source/libs/command/inc/commandInt.h
@@ -34,6 +34,7 @@ extern "C" {
#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s"
#define EXPLAIN_DISTBLK_SCAN_FORMAT "Block Dist Scan on %s"
#define EXPLAIN_LASTROW_SCAN_FORMAT "Last Row Scan on %s"
+#define EXPLAIN_TABLE_COUNT_SCAN_FORMAT "Table Count Row Scan on %s"
#define EXPLAIN_PROJECTION_FORMAT "Projection"
#define EXPLAIN_JOIN_FORMAT "%s"
#define EXPLAIN_AGG_FORMAT "Aggragate"
diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c
index 410e62a18b..03c7249294 100644
--- a/source/libs/command/src/explain.c
+++ b/source/libs/command/src/explain.c
@@ -19,6 +19,7 @@
#include "query.h"
#include "tcommon.h"
#include "tdatablock.h"
+#include "systable.h"
int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes);
int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel);
@@ -212,6 +213,11 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo
pPhysiChildren = lastRowPhysiNode->scan.node.pChildren;
break;
}
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: {
+ STableCountScanPhysiNode *tableCountPhysiNode = (STableCountScanPhysiNode *)pNode;
+ pPhysiChildren = tableCountPhysiNode->scan.node.pChildren;
+ break;
+ }
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
SGroupSortPhysiNode *groupSortPhysiNode = (SGroupSortPhysiNode *)pNode;
pPhysiChildren = groupSortPhysiNode->node.pChildren;
@@ -1355,6 +1361,48 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
}
break;
}
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: {
+ STableCountScanPhysiNode *pLastRowNode = (STableCountScanPhysiNode *)pNode;
+ EXPLAIN_ROW_NEW(level, EXPLAIN_TABLE_COUNT_SCAN_FORMAT,
+ ('\0' != pLastRowNode->scan.tableName.tname[0] ? pLastRowNode->scan.tableName.tname : TSDB_INS_TABLE_TABLES));
+ EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
+ if (pResNode->pExecInfo) {
+ QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
+ EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+ }
+ EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, LIST_LENGTH(pLastRowNode->scan.pScanCols));
+ EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+ if (pLastRowNode->scan.pScanPseudoCols) {
+ EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pLastRowNode->scan.pScanPseudoCols->length);
+ EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+ }
+ EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pLastRowNode->scan.node.pOutputDataBlockDesc->totalRowSize);
+ EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+ EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
+ EXPLAIN_ROW_END();
+ QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
+
+ if (verbose) {
+ EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT);
+ EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT,
+ nodesGetOutputNumFromSlotList(pLastRowNode->scan.node.pOutputDataBlockDesc->pSlots));
+ EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
+ EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pLastRowNode->scan.node.pOutputDataBlockDesc->outputRowSize);
+ EXPLAIN_ROW_APPEND_LIMIT(pLastRowNode->scan.node.pLimit);
+ EXPLAIN_ROW_APPEND_SLIMIT(pLastRowNode->scan.node.pSlimit);
+ EXPLAIN_ROW_END();
+ QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+
+ if (pLastRowNode->scan.node.pConditions) {
+ EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
+ QRY_ERR_RET(nodesNodeToSQL(pLastRowNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE,
+ TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
+ EXPLAIN_ROW_END();
+ QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
+ }
+ }
+ break;
+ }
case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
SGroupSortPhysiNode *pSortNode = (SGroupSortPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_GROUP_SORT_FORMAT);
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index f17d7f6468..9d32bf830e 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -46,7 +46,6 @@ extern "C" {
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
-#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u)
#define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0)
#define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN)
#define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0)
@@ -109,6 +108,7 @@ typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr);
typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr);
typedef void (*__optr_close_fn_t)(void* param);
typedef int32_t (*__optr_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len);
+typedef int32_t (*__optr_reqBuf_fn_t)(struct SOperatorInfo* pOptr);
typedef struct STaskIdInfo {
uint64_t queryId; // this is also a request id
@@ -170,8 +170,9 @@ struct SExecTaskInfo {
STaskCostInfo cost;
int64_t owner; // if it is in execution
int32_t code;
+ int32_t qbufQuota; // total available buffer (in KB) during execution query
- int64_t version; // used for stream to record wal version
+ int64_t version; // used for stream to record wal version, why not move to sschemainfo
SStreamTaskInfo streamInfo;
SSchemaInfo schemaInfo;
STableListInfo* pTableInfoList; // this is a table list
@@ -198,6 +199,7 @@ typedef struct SOperatorFpSet {
__optr_fn_t getNextFn;
__optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP
__optr_close_fn_t closeFn;
+ __optr_reqBuf_fn_t reqBufFn; // total used buffer for blocking operator
__optr_encode_fn_t encodeResultRow;
__optr_decode_fn_t decodeResultRow;
__optr_explain_fn_t getExplainFn;
@@ -482,6 +484,26 @@ typedef struct {
SSnapContext* sContext;
} SStreamRawScanInfo;
+typedef struct STableCountScanSupp {
+ int16_t dbNameSlotId;
+ int16_t stbNameSlotId;
+ int16_t tbCountSlotId;
+ bool groupByDbName;
+ bool groupByStbName;
+ char dbNameFilter[TSDB_DB_NAME_LEN];
+ char stbNameFilter[TSDB_TABLE_NAME_LEN];
+} STableCountScanSupp;
+
+typedef struct STableCountScanOperatorInfo {
+ SReadHandle readHandle;
+ SSDataBlock* pRes;
+
+ STableCountScanSupp supp;
+
+ int32_t currGrpIdx;
+ SArray* stbUidList; // when group by db_name and/or stable_name
+} STableCountScanOperatorInfo;
+
typedef struct SOptrBasicInfo {
SResultRowInfo resultRowInfo;
SSDataBlock* pRes;
@@ -536,6 +558,7 @@ typedef struct SStreamIntervalOperatorInfo {
SArray* pChildren;
SStreamState* pState;
SWinKey delKey;
+ uint64_t numOfDatapack;
} SStreamIntervalOperatorInfo;
typedef struct SDataGroupInfo {
@@ -648,13 +671,14 @@ typedef struct SStreamFillOperatorInfo {
#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED)
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
- __optr_close_fn_t closeFn, __optr_explain_fn_t explain);
-int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);
+ __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain);
+int32_t optrDummyOpenFn(SOperatorInfo* pOperator);
int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num);
void setOperatorCompleted(SOperatorInfo* pOperator);
void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status,
void* pInfo, SExecTaskInfo* pTaskInfo);
void destroyOperatorInfo(SOperatorInfo* pOperator);
+int32_t optrDefaultBufFn(SOperatorInfo* pOperator);
void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
void cleanupBasicInfo(SOptrBasicInfo* pInfo);
@@ -717,6 +741,8 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
+
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
@@ -780,7 +806,7 @@ void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order,
int32_t checkForQueryBuf(size_t numOfTables);
bool isTaskKilled(SExecTaskInfo* pTaskInfo);
-void setTaskKilled(SExecTaskInfo* pTaskInfo);
+void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode);
void doDestroyTask(SExecTaskInfo* pTaskInfo);
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
@@ -789,8 +815,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
-int32_t getMaximumIdleDurationSec();
-
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
int32_t order);
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c
index c432f3c01c..672bb09b14 100644
--- a/source/libs/executor/src/cachescanoperator.c
+++ b/source/libs/executor/src/cachescanoperator.c
@@ -117,7 +117,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, NULL);
+ createOperatorFpSet(optrDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, optrDefaultBufFn, NULL);
pOperator->cost.openCost = 0;
return pOperator;
diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index 963a273290..76e27f12fc 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -70,7 +70,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
tsem_wait(&pExchangeInfo->ready);
if (isTaskKilled(pTaskInfo)) {
- longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+ longjmp(pTaskInfo->env, pTaskInfo->code);
}
for (int32_t i = 0; i < totalSources; ++i) {
@@ -198,7 +198,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
}
}
-static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) {
+static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) {
SExchangeInfo* pExchangeInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -307,7 +307,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock);
pOperator->fpSet =
- createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL);
+ createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL);
return pOperator;
_error:
@@ -570,13 +570,10 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) {
pOperator->status = OP_RES_TO_RETURN;
pOperator->cost.openCost = taosGetTimestampUs() - startTs;
-
- tsem_wait(&pExchangeInfo->ready);
if (isTaskKilled(pTaskInfo)) {
- longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+ longjmp(pTaskInfo->env, pTaskInfo->code);
}
- tsem_post(&pExchangeInfo->ready);
return TSDB_CODE_SUCCESS;
}
@@ -621,7 +618,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current);
tsem_wait(&pExchangeInfo->ready);
if (isTaskKilled(pTaskInfo)) {
- longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+ longjmp(pTaskInfo->env, pTaskInfo->code);
}
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 8a85885d98..1241bdf5a4 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -438,13 +438,12 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
goto end;
}
}
+ removeInvalidTable(uidList, tags);
int32_t rows = taosArrayGetSize(uidList);
if (rows == 0) {
goto end;
}
- // int64_t stt1 = taosGetTimestampUs();
- // qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt);
code = blockDataEnsureCapacity(pResBlock, rows);
if (code != TSDB_CODE_SUCCESS) {
@@ -452,7 +451,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
goto end;
}
- // int64_t st = taosGetTimestampUs();
for (int32_t i = 0; i < rows; i++) {
int64_t* uid = taosArrayGet(uidList, i);
for (int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++) {
@@ -467,7 +465,9 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
#endif
} else {
void* tag = taosHashGet(tags, uid, sizeof(int64_t));
- ASSERT(tag);
+ if (tag == NULL) {
+ continue;
+ }
STagVal tagVal = {0};
tagVal.cid = pColInfo->info.colId;
const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal);
@@ -923,14 +923,14 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
return -1;
}
- SArray* pTbList = getTableNameList(pList);
- int32_t numOfTables = taosArrayGetSize(pTbList);
- SHashObj *uHash = NULL;
+ SArray* pTbList = getTableNameList(pList);
+ int32_t numOfTables = taosArrayGetSize(pTbList);
+ SHashObj* uHash = NULL;
size_t listlen = taosArrayGetSize(list); // len > 0 means there already have uids
if (listlen > 0) {
uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
for (int i = 0; i < listlen; i++) {
- int64_t *uid = taosArrayGet(list, i);
+ int64_t* uid = taosArrayGet(list, i);
taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i));
}
}
@@ -1241,6 +1241,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
}
}
+ // set the output flag for each column in SColMatchInfo, according to the
*numOfOutputCols = 0;
int32_t num = LIST_LENGTH(pOutputNodeList->pSlots);
for (int32_t i = 0; i < num; ++i) {
@@ -1348,6 +1349,7 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) {
pExprNode->_function.functionId = pFuncNode->funcId;
pExprNode->_function.pFunctNode = pFuncNode;
+ pExprNode->_function.functionType = pFuncNode->funcType;
tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName));
@@ -1536,8 +1538,6 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
pCtx->start.key = INT64_MIN;
pCtx->end.key = INT64_MIN;
pCtx->numOfParams = pExpr->base.numOfParams;
- pCtx->isStream = false;
-
pCtx->param = pFunct->pParam;
pCtx->saveHandle.currentPage = -1;
}
@@ -1601,20 +1601,22 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) {
int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) {
pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols);
+
pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
- if (pCond->colList == NULL) {
+ pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)*pCond->numOfCols);
+ if (pCond->colList == NULL || pCond->pSlotList == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
+ taosMemoryFreeClear(pCond->colList);
+ taosMemoryFreeClear(pCond->pSlotList);
return terrno;
}
- // pCond->twindow = pTableScanNode->scanRange;
// TODO: get it from stable scan node
pCond->twindows = pTableScanNode->scanRange;
pCond->suid = pTableScanNode->scan.suid;
pCond->type = TIMEWINDOW_RANGE_CONTAINED;
pCond->startVersion = -1;
pCond->endVersion = -1;
- // pCond->type = pTableScanNode->scanFlag;
int32_t j = 0;
for (int32_t i = 0; i < pCond->numOfCols; ++i) {
@@ -1627,6 +1629,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
pCond->colList[j].type = pColNode->node.resType.type;
pCond->colList[j].bytes = pColNode->node.resType.bytes;
pCond->colList[j].colId = pColNode->colId;
+
+ pCond->pSlotList[j] = pNode->slotId;
j += 1;
}
@@ -1634,7 +1638,10 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
return TSDB_CODE_SUCCESS;
}
-void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFreeClear(pCond->colList); }
+void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) {
+ taosMemoryFreeClear(pCond->colList);
+ taosMemoryFreeClear(pCond->pSlotList);
+}
int32_t convertFillType(int32_t mode) {
int32_t type = TSDB_FILL_NONE;
@@ -1964,7 +1971,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond,
- struct SExecTaskInfo* pTaskInfo) {
+ SExecTaskInfo* pTaskInfo) {
int64_t st = taosGetTimestampUs();
const char* idStr = GET_TASKID(pTaskInfo);
@@ -2011,4 +2018,4 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag) {
char* pBuf = NULL;
qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
taosMemoryFree(pBuf);
-}
\ No newline at end of file
+}
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 10ceb9ccee..4ad81f609a 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -207,7 +207,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n
}
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
- pTaskInfo->cost.created = taosGetTimestampMs();
+ pTaskInfo->cost.created = taosGetTimestampUs();
pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE;
pTaskInfo->pRoot = createRawScanOperatorInfo(readers, pTaskInfo);
if (NULL == pTaskInfo->pRoot) {
@@ -503,7 +503,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
}
if (pTaskInfo->cost.start == 0) {
- pTaskInfo->cost.start = taosGetTimestampMs();
+ pTaskInfo->cost.start = taosGetTimestampUs();
}
if (isTaskKilled(pTaskInfo)) {
@@ -597,7 +597,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
}
if (pTaskInfo->cost.start == 0) {
- pTaskInfo->cost.start = taosGetTimestampMs();
+ pTaskInfo->cost.start = taosGetTimestampUs();
}
if (isTaskKilled(pTaskInfo)) {
@@ -688,7 +688,7 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) {
taosWUnLockLatch(&pTaskInfo->stopInfo.lock);
}
-int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
+int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
if (pTaskInfo == NULL) {
@@ -697,7 +697,7 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
qDebug("%s execTask async killed", GET_TASKID(pTaskInfo));
- setTaskKilled(pTaskInfo);
+ setTaskKilled(pTaskInfo, rspCode);
qStopTaskOperators(pTaskInfo);
@@ -706,15 +706,20 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) {
STaskCostInfo* pSummary = &pTaskInfo->cost;
+ int64_t idleTime = pSummary->start - pSummary->created;
SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder;
if (pSummary->pRecoder != NULL) {
qDebug(
- "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, "
+ "%s :cost summary: idle:%.2f ms, elapsed time:%.2f ms, extract tableList:%.2f ms, "
+ "createGroupIdMap:%.2f ms, total blocks:%d, "
"load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64,
- GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime,
- pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows,
- pRecorder->totalCheckedRows);
+ GET_TASKID(pTaskInfo), idleTime / 1000.0, pSummary->elapsedTime / 1000.0, pSummary->extractListTime,
+ pSummary->groupIdMapTime, pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks,
+ pRecorder->totalRows, pRecorder->totalCheckedRows);
+ } else {
+ qDebug("%s :cost summary: idle in queue:%.2f ms, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), idleTime / 1000.0,
+ pSummary->elapsedTime / 1000.0);
}
}
@@ -971,21 +976,27 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
pCond->order = TSDB_ORDER_ASC;
pCond->numOfCols = pMtInfo->schema->nCols;
pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
- if (pCond->colList == NULL) {
+ pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t) * pCond->numOfCols);
+ if (pCond->colList == NULL || pCond->pSlotList == NULL) {
+ taosMemoryFreeClear(pCond->colList);
+ taosMemoryFreeClear(pCond->pSlotList);
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return terrno;
}
- pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
+ pCond->twindows = TSWINDOW_INITIALIZER;
pCond->suid = pMtInfo->suid;
pCond->type = TIMEWINDOW_RANGE_CONTAINED;
pCond->startVersion = -1;
pCond->endVersion = sContext->snapVersion;
for (int32_t i = 0; i < pCond->numOfCols; ++i) {
- pCond->colList[i].type = pMtInfo->schema->pSchema[i].type;
- pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes;
- pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId;
+ SColumnInfo* pColInfo = &pCond->colList[i];
+ pColInfo->type = pMtInfo->schema->pSchema[i].type;
+ pColInfo->bytes = pMtInfo->schema->pSchema[i].bytes;
+ pColInfo->colId = pMtInfo->schema->pSchema[i].colId;
+
+ pCond->pSlotList[i] = i;
}
return TSDB_CODE_SUCCESS;
@@ -1078,7 +1089,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
int32_t num = tableListGetSize(pTaskInfo->pTableInfoList);
if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num,
- &pTableScanInfo->base.dataReader, NULL) < 0 ||
+ pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 ||
pTableScanInfo->base.dataReader == NULL) {
ASSERT(0);
}
@@ -1130,7 +1141,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
int32_t size = tableListGetSize(pTaskInfo->pTableInfoList);
ASSERT(size == 1);
- tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->dataReader, NULL);
+ tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL);
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index d0d8b42442..b408e32061 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -85,8 +85,6 @@ typedef struct SAggOperatorInfo {
SExprSupp scalarExprSup;
} SAggOperatorInfo;
-int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; }
-
static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock);
static void releaseQueryBuf(size_t numOfTables);
@@ -106,7 +104,7 @@ void setOperatorCompleted(SOperatorInfo* pOperator) {
pOperator->status = OP_EXEC_DONE;
ASSERT(pOperator->pTaskInfo != NULL);
- pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start * 1000) / 1000.0;
+ pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start) / 1000.0;
setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
}
@@ -120,19 +118,21 @@ void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, b
pOperator->pTaskInfo = pTaskInfo;
}
-int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) {
+int32_t optrDummyOpenFn(SOperatorInfo* pOperator) {
OPTR_SET_OPENED(pOperator);
pOperator->cost.openCost = 0;
return TSDB_CODE_SUCCESS;
}
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
- __optr_close_fn_t closeFn, __optr_explain_fn_t explain) {
+ __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn,
+ __optr_explain_fn_t explain) {
SOperatorFpSet fpSet = {
._openFn = openFn,
.getNextFn = nextFn,
.cleanupFn = cleanup,
.closeFn = closeFn,
+ .reqBufFn = reqBufFn,
.getExplainFn = explain,
};
@@ -610,21 +610,10 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
}
bool isTaskKilled(SExecTaskInfo* pTaskInfo) {
- // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived
- // abort current query execution.
- if (pTaskInfo->owner != 0 &&
- ((taosGetTimestampSec() - pTaskInfo->cost.start / 1000) > 10 * getMaximumIdleDurationSec())
- /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) {
- assert(pTaskInfo->cost.start != 0);
- // qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64
- // ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec());
- // return true;
- }
-
- return false;
+ return (0 != pTaskInfo->code) ? true : false;
}
-void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED; }
+void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; }
/////////////////////////////////////////////////////////////////////////////////////////////
STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) {
@@ -950,10 +939,10 @@ static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, u
}
static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs,
- const int32_t* rowCellOffset) {
+ const int32_t* rowEntryOffset) {
bool returnNotNull = false;
for (int32_t j = 0; j < numOfExprs; ++j) {
- struct SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowCellOffset);
+ SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowEntryOffset);
if (!isRowEntryInitialized(pResInfo)) {
continue;
}
@@ -1156,45 +1145,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
}
}
-// void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) {
-// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
-//
-// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) {
-// return;
-// }
-//
-// pQueryAttr->pos = 0;
-// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
-//
-// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
-// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle;
-//
-// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER;
-// while (tsdbNextDataBlock(pTsdbReadHandle)) {
-// if (isTaskKilled(pRuntimeEnv->qinfo)) {
-// T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
-// }
-//
-// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo);
-//
-// if (pQueryAttr->limit.offset > blockInfo.rows) {
-// pQueryAttr->limit.offset -= blockInfo.rows;
-// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey;
-// pTableQueryInfo->lastKey += step;
-//
-// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows,
-// pQuery->limit.offset);
-// } else { // find the appropriated start position in current block
-// updateOffsetVal(pRuntimeEnv, &blockInfo);
-// break;
-// }
-// }
-//
-// if (terrno != TSDB_CODE_SUCCESS) {
-// T_LONG_JMP(pRuntimeEnv->env, terrno);
-// }
-// }
-
// static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo,
// STableQueryInfo* pTableQueryInfo) {
// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
@@ -1381,7 +1331,8 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
int32_t type = pOperator->operatorType;
if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE || type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN ||
type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN ||
- type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN) {
+ type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN ||
+ type == QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN) {
*order = TSDB_ORDER_ASC;
*scanFlag = MAIN_SCAN;
return TSDB_CODE_SUCCESS;
@@ -1418,11 +1369,11 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
bool hasCountFunc = false;
+
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
- if ((strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "count") == 0) ||
- (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0) ||
- (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_partial") == 0) ||
- (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_merge") == 0)) {
+ const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
+ if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) ||
+ (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) {
hasCountFunc = true;
break;
}
@@ -1475,7 +1426,6 @@ static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppB
*ppBlock = NULL;
}
-
// this is a blocking operator
static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
if (OPTR_IS_OPENED(pOperator)) {
@@ -1626,6 +1576,15 @@ void destroyOperatorInfo(SOperatorInfo* pOperator) {
taosMemoryFreeClear(pOperator);
}
+// each operator should be set their own function to return total cost buffer
+int32_t optrDefaultBufFn(SOperatorInfo* pOperator) {
+ if (pOperator->blocking) {
+ ASSERT(0);
+ } else {
+ return 0;
+ }
+}
+
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz) {
*defaultPgsz = 4096;
while (*defaultPgsz < rowSize * 4) {
@@ -1651,7 +1610,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
pAggSup->currentPageId = -1;
pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t));
- pAggSup->pResultRowHashTable = tSimpleHashInit(10, hashFn);
+ pAggSup->pResultRowHashTable = tSimpleHashInit(100, hashFn);
if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@@ -1806,7 +1765,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo,
pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, optrDefaultBufFn, NULL);
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pTableScanInfo = downstream->info;
@@ -1867,7 +1826,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTaskInfo->schemaInfo.dbname = strdup(dbFName);
- pTaskInfo->cost.created = taosGetTimestampMs();
pTaskInfo->id.queryId = queryId;
pTaskInfo->execModel = model;
pTaskInfo->pTableInfoList = tableListCreate();
@@ -2073,6 +2031,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode;
pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo);
+ } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN == type) {
+ STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode;
+ pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
@@ -2371,6 +2332,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
goto _complete;
}
+ (*pTaskInfo)->cost.created = taosGetTimestampUs();
return TSDB_CODE_SUCCESS;
_complete:
diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c
index e26bfe9889..8d5af64777 100644
--- a/source/libs/executor/src/filloperator.c
+++ b/source/libs/executor/src/filloperator.c
@@ -381,7 +381,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->exprSupp.numOfExprs = pInfo->numOfExpr;
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@@ -1146,7 +1146,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
if (delTs > nextKey.ts) {
break;
}
- endTs = delTs;
+
SWinKey delKey = {.groupId = delGroupId, .ts = delTs};
if (delTs == nextKey.ts) {
code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
@@ -1159,7 +1159,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
streamStateFreeCur(pCur);
pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
}
- endTs = TMAX(ts, nextKey.ts - 1);
+ endTs = TMAX(delTs, nextKey.ts - 1);
if (code != TSDB_CODE_SUCCESS) {
break;
}
@@ -1478,7 +1478,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
pInfo->srcRowIndex = 0;
setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index 4601175561..5d34064b7e 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -470,7 +470,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL);
+ createOperatorFpSet(optrDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
@@ -850,7 +850,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
pOperator->exprSupp.numOfExprs = numOfCols;
pOperator->exprSupp.pExprInfo = pExprInfo;
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@@ -1142,7 +1142,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
pOperator->exprSupp.numOfExprs = numOfCols;
pOperator->exprSupp.pExprInfo = pExprInfo;
pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL);
+ createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL);
initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup);
code = appendDownstream(pOperator, &downstream, 1);
diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c
index e7cce39dfd..d460af971c 100644
--- a/source/libs/executor/src/joinoperator.c
+++ b/source/libs/executor/src/joinoperator.c
@@ -136,7 +136,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
pInfo->inputOrder = TSDB_ORDER_DESC;
}
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, pDownstream, numOfDownstream);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c
index da77facb21..65bb40b195 100644
--- a/source/libs/executor/src/projectoperator.c
+++ b/source/libs/executor/src/projectoperator.c
@@ -118,8 +118,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols);
setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo, pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL,
- destroyProjectOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doProjectOperation, NULL,
+ destroyProjectOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -415,7 +415,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr);
setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index b6353061fb..0d245f0815 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -30,7 +30,6 @@
#include "tcompare.h"
#include "thash.h"
#include "ttypes.h"
-#include "vnode.h"
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
@@ -224,10 +223,8 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA
}
static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
- bool allColumnsHaveAgg = true;
- SColumnDataAgg** pColAgg = NULL;
-
- int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg);
+ bool allColumnsHaveAgg = true;
+ int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pBlock->pBlockAgg, &allColumnsHaveAgg);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
@@ -236,6 +233,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
return false;
}
+#if 0
// if (allColumnsHaveAgg == true) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
@@ -256,6 +254,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
}
+#endif
return true;
}
@@ -285,7 +284,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
if (pLimit->offset > 0 && pLimitInfo->remainOffset > 0) {
if (pLimitInfo->remainOffset >= pBlock->info.rows) {
pLimitInfo->remainOffset -= pBlock->info.rows;
- pBlock->info.rows = 0;
+ blockDataEmpty(pBlock);
qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, id);
} else {
blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset);
@@ -385,12 +384,12 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1;
- SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
- if (pCols == NULL) {
+ SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
+ if (p == NULL) {
return terrno;
}
- relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
+ ASSERT(p == pBlock);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
// restore the previous value
@@ -487,10 +486,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid);
if (code != TSDB_CODE_SUCCESS) {
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
- qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid,
- tstrerror(terrno), idStr);
+ qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",
+ pBlock->info.id.uid, tstrerror(terrno), idStr);
} else {
- qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr);
+ qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno),
+ idStr);
}
metaReaderClear(&mr);
return terrno;
@@ -629,7 +629,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
while (tsdbNextDataBlock(pTableScanInfo->base.dataReader)) {
if (isTaskKilled(pTaskInfo)) {
- T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+ T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
// process this data block based on the probabilities
@@ -638,16 +638,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
continue;
}
- blockDataCleanup(pBlock);
- SDataBlockInfo* pBInfo = &pBlock->info;
-
- int32_t rows = 0;
- tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->id.uid, &pBInfo->window);
-
- blockDataEnsureCapacity(pBlock, rows); // todo remove it latter
- pBInfo->rows = rows;
-
- ASSERT(pBInfo->id.uid != 0);
+ ASSERT(pBlock->info.id.uid != 0);
pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
uint32_t status = 0;
@@ -778,7 +769,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
ASSERT(pInfo->base.dataReader == NULL);
int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num,
- (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo));
+ pInfo->pResBlock, (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
@@ -879,11 +870,11 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pInfo->base.scanFlag = MAIN_SCAN;
pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode);
pInfo->base.readHandle = *readHandle;
+ pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
+
pInfo->sample.sampleRatio = pTableScanNode->ratio;
pInfo->sample.seed = taosGetTimestampSec();
- pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
-
initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
@@ -908,8 +899,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
}
taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo,
- getTableScannerExecInfo);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo,
+ optrDefaultBufFn, getTableScannerExecInfo);
// for non-blocking operator, the open cost is always 0
pOperator->cost.openCost = 0;
@@ -934,7 +925,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo*
setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED,
pInfo, pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScanImpl, NULL, NULL, optrDefaultBufFn, NULL);
return pOperator;
}
@@ -994,10 +985,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo;
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
- blockDataCleanup(pBlock);
-
STsdbReader* pReader = NULL;
- int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader,
+ int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, (STsdbReader**)&pReader,
GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
@@ -1005,21 +994,10 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
return NULL;
}
- bool hasBlock = tsdbNextDataBlock(pReader);
- if (hasBlock) {
- SDataBlockInfo* pBInfo = &pBlock->info;
-
- int32_t rows = 0;
- tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window);
-
- SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL);
- blockDataEnsureCapacity(pBlock, rows);
- pBlock->info.rows = rows;
-
- relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
- doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, rows);
-
- pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid);
+ if (tsdbNextDataBlock(pReader)) {
+ /*SSDataBlock* p = */tsdbRetrieveDataBlock(pReader, NULL);
+ doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
+ pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
}
tsdbReaderClose(pReader);
@@ -1113,7 +1091,7 @@ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t
if (hasGroup) {
(*pRowIndex) += 1;
} else {
- while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] < endWin.ekey)) {
+ while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] <= endWin.ekey)) {
(*pRowIndex) += 1;
if ((*pRowIndex) == pDataBlockInfo->rows) {
break;
@@ -1436,8 +1414,8 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
NULL);
if (closedWin && pInfo->partitionSup.needCalc) {
gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId);
- appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, &gpId,
- NULL);
+ appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid,
+ &gpId, NULL);
}
}
}
@@ -2028,20 +2006,13 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
qDebug("tmqsnap doRawScan called");
if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
- SSDataBlock* pBlock = &pInfo->pRes;
-
if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) {
if (isTaskKilled(pTaskInfo)) {
- longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+ longjmp(pTaskInfo->env, pTaskInfo->code);
}
- int32_t rows = 0;
- tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.id.uid, &pBlock->info.window);
- pBlock->info.rows = rows;
-
- SArray* pCols = tsdbRetrieveDataBlock(pInfo->dataReader, NULL);
- pBlock->pDataBlock = pCols;
- if (pCols == NULL) {
+ SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL);
+ if (pBlock == NULL) {
longjmp(pTaskInfo->env, terrno);
}
@@ -2168,7 +2139,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT
setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, optrDefaultBufFn, NULL);
return pOperator;
_end:
@@ -2287,7 +2258,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
if (pHandle->initTableReader) {
pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
pTSInfo->base.dataReader = NULL;
- code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, &pTSInfo->base.dataReader, NULL);
+ code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, pTSInfo->pResBlock, &pTSInfo->base.dataReader, NULL);
if (code != 0) {
terrno = code;
destroyTableScanOperatorInfo(pTableScanOp);
@@ -2357,7 +2328,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
__optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan;
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL);
return pOperator;
@@ -2462,7 +2433,8 @@ static void destroyTagScanOperatorInfo(void* param) {
taosMemoryFreeClear(param);
}
-SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
+ SExecTaskInfo* pTaskInfo) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@@ -2493,7 +2465,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL);
return pOperator;
@@ -2511,17 +2483,14 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
int32_t readIdx = source->readerIdx;
SSDataBlock* pBlock = source->inputBlock;
- STableMergeScanInfo* pTableScanInfo = pOperator->info;
- SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx);
- blockDataCleanup(pBlock);
+ SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx);
int64_t st = taosGetTimestampUs();
-
- void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
+ void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
SReadHandle* pHandle = &pInfo->base.readHandle;
- int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->base.dataReader, GET_TASKID(pTaskInfo));
+ int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &pInfo->base.dataReader, GET_TASKID(pTaskInfo));
if (code != 0) {
T_LONG_JMP(pTaskInfo->env, code);
}
@@ -2529,22 +2498,15 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
STsdbReader* reader = pInfo->base.dataReader;
while (tsdbNextDataBlock(reader)) {
if (isTaskKilled(pTaskInfo)) {
- T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+ T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
// process this data block based on the probabilities
- bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
+ bool processThisBlock = processBlockWithProbability(&pInfo->sample);
if (!processThisBlock) {
continue;
}
- blockDataCleanup(pBlock);
-
- int32_t rows = 0;
- tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.id.uid, &pBlock->info.window);
- blockDataEnsureCapacity(pBlock, rows);
- pBlock->info.rows = rows;
-
if (pQueryCond->order == TSDB_ORDER_ASC) {
pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
} else {
@@ -2552,7 +2514,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
}
uint32_t status = 0;
- loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status);
+ loadDataBlock(pOperator, &pInfo->base, pBlock, &status);
// code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
@@ -2566,7 +2528,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
pOperator->resultInfo.totalRows += pBlock->info.rows;
- pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
+ pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
tsdbReaderClose(pInfo->base.dataReader);
pInfo->base.dataReader = NULL;
@@ -2646,6 +2608,8 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
param.readerIdx = i;
param.pOperator = pOperator;
param.inputBlock = createOneDataBlock(pInfo->pResBlock, false);
+ blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity);
+
taosArrayPush(pInfo->sortSourceParams, ¶m);
SQueryTableDataCond cond;
@@ -2902,8 +2866,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
pInfo, pTaskInfo);
pOperator->exprSupp.numOfExprs = numOfCols;
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo,
- getTableMergeScanExplainExecInfo);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo,
+ optrDefaultBufFn, getTableMergeScanExplainExecInfo);
pOperator->cost.openCost = 0;
return pOperator;
@@ -2913,3 +2877,351 @@ _error:
taosMemoryFree(pOperator);
return NULL;
}
+
+// ====================================================================================================================
+// TableCountScanOperator
+static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator);
+static void destoryTableCountScanOperator(void* param);
+static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
+ SSDataBlock* pRes, char* dbName, tb_uid_t stbUid);
+static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
+ SSDataBlock* pRes, char* dbName);
+static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+ STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName);
+static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+ STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName);
+static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+ STableCountScanSupp* pSupp, SSDataBlock* pRes);
+static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+ STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum,
+ size_t perfdbTableNum);
+static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes,
+ size_t infodbTableNum, size_t perfdbTableNum);
+static const char* GROUP_TAG_DB_NAME = "db_name";
+static const char* GROUP_TAG_STABLE_NAME = "stable_name";
+
+int32_t tblCountScanGetGroupTagsSlotId(const SNodeList* scanCols, STableCountScanSupp* supp) {
+ if (scanCols != NULL) {
+ SNode* pNode = NULL;
+ FOREACH(pNode, scanCols) {
+ if (nodeType(pNode) != QUERY_NODE_TARGET) {
+ return TSDB_CODE_QRY_SYS_ERROR;
+ }
+ STargetNode* targetNode = (STargetNode*)pNode;
+ if (nodeType(targetNode->pExpr) != QUERY_NODE_COLUMN) {
+ return TSDB_CODE_QRY_SYS_ERROR;
+ }
+ SColumnNode* colNode = (SColumnNode*)(targetNode->pExpr);
+ if (strcmp(colNode->colName, GROUP_TAG_DB_NAME) == 0) {
+ supp->dbNameSlotId = targetNode->slotId;
+ } else if (strcmp(colNode->colName, GROUP_TAG_STABLE_NAME) == 0) {
+ supp->stbNameSlotId = targetNode->slotId;
+ }
+ }
+ }
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t tblCountScanGetCountSlotId(const SNodeList* pseudoCols, STableCountScanSupp* supp) {
+ if (pseudoCols != NULL) {
+ SNode* pNode = NULL;
+ FOREACH(pNode, pseudoCols) {
+ if (nodeType(pNode) != QUERY_NODE_TARGET) {
+ return TSDB_CODE_QRY_SYS_ERROR;
+ }
+ STargetNode* targetNode = (STargetNode*)pNode;
+ if (nodeType(targetNode->pExpr) != QUERY_NODE_FUNCTION) {
+ return TSDB_CODE_QRY_SYS_ERROR;
+ }
+ SFunctionNode* funcNode = (SFunctionNode*)(targetNode->pExpr);
+ if (funcNode->funcType == FUNCTION_TYPE_TABLE_COUNT) {
+ supp->tbCountSlotId = targetNode->slotId;
+ }
+ }
+ }
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t tblCountScanGetInputs(SNodeList* groupTags, SName* tableName, STableCountScanSupp* supp) {
+ if (groupTags != NULL) {
+ SNode* pNode = NULL;
+ FOREACH(pNode, groupTags) {
+ if (nodeType(pNode) != QUERY_NODE_COLUMN) {
+ return TSDB_CODE_QRY_SYS_ERROR;
+ }
+ SColumnNode* colNode = (SColumnNode*)pNode;
+ if (strcmp(colNode->colName, GROUP_TAG_DB_NAME) == 0) {
+ supp->groupByDbName = true;
+ }
+ if (strcmp(colNode->colName, GROUP_TAG_STABLE_NAME) == 0) {
+ supp->groupByStbName = true;
+ }
+ }
+ } else {
+ strncpy(supp->dbNameFilter, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN);
+ strncpy(supp->stbNameFilter, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN);
+ }
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* scanCols, SNodeList* pseudoCols,
+ STableCountScanSupp* supp, SExecTaskInfo* taskInfo) {
+ int32_t code = 0;
+ code = tblCountScanGetInputs(groupTags, tableName, supp);
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s get table count scan supp. get inputs error", GET_TASKID(taskInfo));
+ return code;
+ }
+ supp->dbNameSlotId = -1;
+ supp->stbNameSlotId = -1;
+ supp->tbCountSlotId = -1;
+
+ code = tblCountScanGetGroupTagsSlotId(scanCols, supp);
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s get table count scan supp. get group tags slot id error", GET_TASKID(taskInfo));
+ return code;
+ }
+ code = tblCountScanGetCountSlotId(pseudoCols, supp);
+ if (code != TSDB_CODE_SUCCESS) {
+ qError("%s get table count scan supp. get count error", GET_TASKID(taskInfo));
+ return code;
+ }
+ return code;
+}
+
+SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableCountScanPhysiNode* pTblCountScanNode,
+ SExecTaskInfo* pTaskInfo) {
+ int32_t code = TSDB_CODE_SUCCESS;
+
+ SScanPhysiNode* pScanNode = &pTblCountScanNode->scan;
+ STableCountScanOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanOperatorInfo));
+ SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+
+ if (!pInfo || !pOperator) {
+ goto _error;
+ }
+
+ pInfo->readHandle = *readHandle;
+
+ SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc;
+ initResultSizeInfo(&pOperator->resultInfo, 1);
+ pInfo->pRes = createDataBlockFromDescNode(pDescNode);
+ blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+
+ getTableCountScanSupp(pTblCountScanNode->pGroupTags, &pTblCountScanNode->scan.tableName,
+ pTblCountScanNode->scan.pScanCols, pTblCountScanNode->scan.pScanPseudoCols, &pInfo->supp,
+ pTaskInfo);
+
+ setOperatorInfo(pOperator, "TableCountScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, false, OP_NOT_OPENED,
+ pInfo, pTaskInfo);
+ pOperator->fpSet =
+ createOperatorFpSet(optrDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, optrDefaultBufFn, NULL);
+ return pOperator;
+
+_error:
+ if (pInfo != NULL) {
+ destoryTableCountScanOperator(pInfo);
+ }
+ taosMemoryFreeClear(pOperator);
+ pTaskInfo->code = code;
+ return NULL;
+}
+
+void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* stbName, int64_t count,
+ SSDataBlock* pRes) {
+ if (pSupp->dbNameSlotId != -1) {
+ ASSERT(strlen(dbName));
+ SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId);
+ char varDbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
+ strncpy(varDataVal(varDbName), dbName, strlen(dbName));
+ varDataSetLen(varDbName, strlen(dbName));
+ colDataAppend(colInfoData, 0, varDbName, false);
+ }
+
+ if (pSupp->stbNameSlotId != -1) {
+ SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId);
+ if (strlen(stbName) != 0) {
+ char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
+ strncpy(varDataVal(varStbName), stbName, strlen(stbName));
+ varDataSetLen(varStbName, strlen(stbName));
+ colDataAppend(colInfoData, 0, varStbName, false);
+ } else {
+ colDataAppendNULL(colInfoData, 0);
+ }
+ }
+
+ if (pSupp->tbCountSlotId != -1) {
+ SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->tbCountSlotId);
+ colDataAppend(colInfoData, 0, (char*)&count, false);
+ }
+ pRes->info.rows = 1;
+}
+
+static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo) {
+ STableCountScanSupp* pSupp = &pInfo->supp;
+ SSDataBlock* pRes = pInfo->pRes;
+
+ size_t infodbTableNum;
+ getInfosDbMeta(NULL, &infodbTableNum);
+ size_t perfdbTableNum;
+ getPerfDbMeta(NULL, &perfdbTableNum);
+
+ if (pSupp->groupByDbName) {
+ buildSysDbGroupedTableCount(pOperator, pInfo, pSupp, pRes, infodbTableNum, perfdbTableNum);
+ return (pRes->info.rows > 0) ? pRes : NULL;
+ } else {
+ buildSysDbFilterTableCount(pOperator, pSupp, pRes, infodbTableNum, perfdbTableNum);
+ return (pRes->info.rows > 0) ? pRes : NULL;
+ }
+}
+
+static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes,
+ size_t infodbTableNum, size_t perfdbTableNum) {
+ if (strcmp(pSupp->dbNameFilter, TSDB_INFORMATION_SCHEMA_DB) == 0) {
+ fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
+ } else if (strcmp(pSupp->dbNameFilter, TSDB_PERFORMANCE_SCHEMA_DB) == 0) {
+ fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
+ } else if (strlen(pSupp->dbNameFilter) == 0) {
+ fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes);
+ }
+ setOperatorCompleted(pOperator);
+}
+
+static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+ STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum,
+ size_t perfdbTableNum) {
+ if (pInfo->currGrpIdx == 0) {
+ uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB));
+ pRes->info.id.groupId = groupId;
+ fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
+ } else if (pInfo->currGrpIdx == 1) {
+ uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB));
+ pRes->info.id.groupId = groupId;
+ fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
+ } else {
+ setOperatorCompleted(pOperator);
+ }
+ pInfo->currGrpIdx++;
+}
+
+static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) {
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ STableCountScanOperatorInfo* pInfo = pOperator->info;
+ STableCountScanSupp* pSupp = &pInfo->supp;
+ SSDataBlock* pRes = pInfo->pRes;
+ blockDataCleanup(pRes);
+
+ if (pOperator->status == OP_EXEC_DONE) {
+ return NULL;
+ }
+ if (pInfo->readHandle.mnd != NULL) {
+ return buildSysDbTableCount(pOperator, pInfo);
+ }
+
+ return buildVnodeDbTableCount(pOperator, pInfo, pSupp, pRes);
+}
+
+static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+ STableCountScanSupp* pSupp, SSDataBlock* pRes) {
+ const char* db = NULL;
+ int32_t vgId = 0;
+ char dbName[TSDB_DB_NAME_LEN] = {0};
+
+ // get dbname
+ vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+ SName sn = {0};
+ tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB);
+ tNameGetDbName(&sn, dbName);
+
+ if (pSupp->groupByDbName) {
+ buildVnodeGroupedTableCount(pOperator, pInfo, pSupp, pRes, vgId, dbName);
+ } else {
+ buildVnodeFilteredTbCount(pOperator, pInfo, pSupp, pRes, dbName);
+ }
+ return pRes->info.rows > 0 ? pRes : NULL;
+}
+
+static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+ STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) {
+ if (pSupp->groupByStbName) {
+ if (pInfo->stbUidList == NULL) {
+ pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t));
+ if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) {
+ qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr());
+ }
+ }
+ if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) {
+ tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx);
+ buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid);
+
+ pInfo->currGrpIdx++;
+ } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) {
+ buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName);
+
+ pInfo->currGrpIdx++;
+ } else {
+ setOperatorCompleted(pOperator);
+ }
+ } else {
+ uint64_t groupId = calcGroupId(dbName, strlen(dbName));
+ pRes->info.id.groupId = groupId;
+ int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta);
+ fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes);
+ setOperatorCompleted(pOperator);
+ }
+}
+
+static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+ STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) {
+ if (strlen(pSupp->dbNameFilter) != 0) {
+ if (strlen(pSupp->stbNameFilter) != 0) {
+ tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter);
+ SMetaStbStats stats = {0};
+ metaGetStbStats(pInfo->readHandle.meta, uid, &stats);
+ int64_t ctbNum = stats.ctbNum;
+ fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes);
+ } else {
+ int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta);
+ fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
+ }
+ } else {
+ int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta);
+ fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
+ }
+ setOperatorCompleted(pOperator);
+}
+
+static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
+ SSDataBlock* pRes, char* dbName) {
+ char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
+ snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
+ uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
+ pRes->info.id.groupId = groupId;
+ int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta);
+ fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes);
+}
+
+static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
+ SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) {
+ char stbName[TSDB_TABLE_NAME_LEN] = {0};
+ metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName);
+
+ char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
+ snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName);
+ uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
+ pRes->info.id.groupId = groupId;
+
+ SMetaStbStats stats = {0};
+ metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats);
+ int64_t ctbNum = stats.ctbNum;
+
+ fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes);
+}
+
+static void destoryTableCountScanOperator(void* param) {
+ STableCountScanOperatorInfo* pTableCountScanInfo = param;
+ blockDataDestroy(pTableCountScanInfo->pRes);
+
+ taosArrayDestroy(pTableCountScanInfo->stbUidList);
+ taosMemoryFreeClear(param);
+}
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index ec754f31b0..005b794f0b 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -75,7 +75,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
// TODO dynamic set the available sort buffer
pOperator->fpSet =
- createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, getExplainExecInfo);
+ createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, optrDefaultBufFn, getExplainExecInfo);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -521,8 +521,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys);
setOperatorInfo(pOperator, "GroupSortOperator", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, false, OP_NOT_OPENED, pInfo, pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo,
- getGroupSortExplainExecInfo);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo,
+ optrDefaultBufFn, getGroupSortExplainExecInfo);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -559,7 +559,7 @@ typedef struct SMultiwayMergeOperatorInfo {
STupleHandle* prefetchedTuple;
} SMultiwayMergeOperatorInfo;
-int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
+int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) {
SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@@ -577,9 +577,15 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort);
for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
+ SOperatorInfo* pDownstream = pOperator->pDownstream[i];
+ if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) {
+ pDownstream->fpSet._openFn(pDownstream);
+ }
+
SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
- ps->param = pOperator->pDownstream[i];
+ ps->param = pDownstream;
ps->onlyRef = true;
+
tsortAddSource(pInfo->pSortHandle, ps);
}
@@ -714,7 +720,6 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) {
}
qDebug("start to merge final sorted rows, %s", GET_TASKID(pTaskInfo));
-
SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->matchInfo.pList, pOperator);
if (pBlock != NULL) {
pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -781,7 +786,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
SSDataBlock* pInputBlock = createDataBlockFromDescNode(pChildNode->pOutputDataBlockDesc);
- initResultSizeInfo(&pOperator->resultInfo, 4096);
+ initResultSizeInfo(&pOperator->resultInfo, 1024);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
pInfo->groupSort = pMergePhyNode->groupSort;
@@ -792,8 +797,8 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result.
setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL,
- destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo);
+ pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL,
+ destroyMultiwayMergeOperatorInfo, optrDefaultBufFn, getMultiwayMergeExplainExecInfo);
code = appendDownstream(pOperator, downStreams, numStreams);
if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c
index 3d35326749..32905ca897 100644
--- a/source/libs/executor/src/sysscanoperator.c
+++ b/source/libs/executor/src/sysscanoperator.c
@@ -1437,7 +1437,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, optrDefaultBufFn, NULL);
return pOperator;
_error:
@@ -1874,78 +1874,80 @@ static void destroyBlockDistScanOperatorInfo(void* param) {
}
static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) {
- memset(pCond, 0, sizeof(SQueryTableDataCond));
+ memset(pCond, 0, sizeof(SQueryTableDataCond));
- pCond->order = TSDB_ORDER_ASC;
- pCond->numOfCols = 1;
- pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo));
- if (pCond->colList == NULL) {
- terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
- return terrno;
- }
+ pCond->order = TSDB_ORDER_ASC;
+ pCond->numOfCols = 1;
+ pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo));
+ pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t));
+ if (pCond->colList == NULL || pCond->pSlotList == NULL) {
+ terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
+ return terrno;
+ }
- pCond->colList->colId = 1;
- pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP;
- pCond->colList->bytes = sizeof(TSKEY);
+ pCond->colList->colId = 1;
+ pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP;
+ pCond->colList->bytes = sizeof(TSKEY);
- pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
- pCond->suid = uid;
- pCond->type = TIMEWINDOW_RANGE_CONTAINED;
- pCond->startVersion = -1;
- pCond->endVersion = -1;
+ pCond->pSlotList[0] = 0;
- return TSDB_CODE_SUCCESS;
+ pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
+ pCond->suid = uid;
+ pCond->type = TIMEWINDOW_RANGE_CONTAINED;
+ pCond->startVersion = -1;
+ pCond->endVersion = -1;
+
+ return TSDB_CODE_SUCCESS;
}
SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode,
SExecTaskInfo* pTaskInfo) {
- SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo));
- SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
- if (pInfo == NULL || pOperator == NULL) {
- pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
- goto _error;
- }
+ SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo));
+ SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+ if (pInfo == NULL || pOperator == NULL) {
+ pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _error;
+ }
- {
- SQueryTableDataCond cond = {0};
+ pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc);
+ blockDataEnsureCapacity(pInfo->pResBlock, 1);
- int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond);
- if (code != TSDB_CODE_SUCCESS) {
- goto _error;
- }
-
- STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
- size_t num = tableListGetSize(pTableListInfo);
- void* pList = tableListGetInfo(pTableListInfo, 0);
-
- code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str);
- cleanupQueryTableDataCond(&cond);
- if (code != 0) {
- goto _error;
- }
- }
-
- pInfo->readHandle = *readHandle;
- pInfo->uid = pBlockScanNode->suid;
-
- pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc);
- blockDataEnsureCapacity(pInfo->pResBlock, 1);
-
- int32_t numOfCols = 0;
- SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
- int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
+ {
+ SQueryTableDataCond cond = {0};
+ int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond);
if (code != TSDB_CODE_SUCCESS) {
- goto _error;
+ goto _error;
}
- setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false,
- OP_NOT_OPENED, pInfo, pTaskInfo);
- pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL);
- return pOperator;
+ STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
+ size_t num = tableListGetSize(pTableListInfo);
+ void* pList = tableListGetInfo(pTableListInfo, 0);
- _error:
- taosMemoryFreeClear(pInfo);
- taosMemoryFreeClear(pOperator);
- return NULL;
+ code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str);
+ cleanupQueryTableDataCond(&cond);
+ if (code != 0) {
+ goto _error;
+ }
+ }
+
+ pInfo->readHandle = *readHandle;
+ pInfo->uid = pBlockScanNode->suid;
+
+ int32_t numOfCols = 0;
+ SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
+ int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
+ if (code != TSDB_CODE_SUCCESS) {
+ goto _error;
+ }
+
+ setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false,
+ OP_NOT_OPENED, pInfo, pTaskInfo);
+ pOperator->fpSet =
+ createOperatorFpSet(optrDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, optrDefaultBufFn, NULL);
+ return pOperator;
+
+_error:
+ taosMemoryFreeClear(pInfo);
+ taosMemoryFreeClear(pOperator);
+ return NULL;
}
\ No newline at end of file
diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c
index a66a5e7089..d30c1fbfa1 100644
--- a/source/libs/executor/src/tfill.c
+++ b/source/libs/executor/src/tfill.c
@@ -45,8 +45,18 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd
if (pCol->notFillCol) {
bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfo, rowIndex);
if (!filled) {
- SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal;
- SGroupKeys* pKey = taosArrayGet(p, i);
+ SRowVal* p = NULL;
+ if (FILL_IS_ASC_FILL(pFillInfo)) {
+ if (pFillInfo->prev.key != 0) {
+ p = &pFillInfo->prev; // prev has been set value
+ } else { // otherwise, use the value in the next row
+ p = &pFillInfo->next;
+ }
+ } else {
+ p = &pFillInfo->next;
+ }
+
+ SGroupKeys* pKey = taosArrayGet(p->pRowVal, i);
doSetVal(pDstColInfo, rowIndex, pKey);
}
} else {
@@ -246,7 +256,10 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) {
static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull);
-static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray* pRow) {
+static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal) {
+ SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId);
+ pRowVal->key = ((int64_t*)pTsCol->pData)[rowIndex];
+
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType;
if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) {
@@ -257,7 +270,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray
bool isNull = colDataIsNull_s(pSrcCol, rowIndex);
char* p = colDataGetData(pSrcCol, rowIndex);
- saveColData(pRow, i, p, isNull);
+ saveColData(pRowVal->pRowVal, i, p, isNull);
} else {
ASSERT(0);
}
@@ -281,7 +294,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
// set the next value for interpolation
if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) {
- copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next.pRowVal);
+ copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, &pFillInfo->next);
}
if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) &&
@@ -303,7 +316,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) {
int32_t nextRowIndex = pFillInfo->index + 1;
- copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, pFillInfo->next.pRowVal);
+ copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next);
}
// copy rows to dst buffer
@@ -319,6 +332,9 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
if (!colDataIsNull_s(pSrc, pFillInfo->index)) {
colDataAppend(pDst, index, src, false);
saveColData(pFillInfo->prev.pRowVal, i, src, false);
+ if (pFillInfo->srcTsSlotId == dstSlotId) {
+ pFillInfo->prev.key = *(int64_t*)src;
+ }
} else { // the value is null
if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
colDataAppend(pDst, index, (const char*)&pFillInfo->currentKey, false);
diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c
index 90f5dde7c3..2374d80bbf 100644
--- a/source/libs/executor/src/timesliceoperator.c
+++ b/source/libs/executor/src/timesliceoperator.c
@@ -544,7 +544,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index f2c2e24a41..2d0893b97b 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1049,14 +1049,14 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
return TSDB_CODE_SUCCESS;
}
- SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+ SOperatorInfo* downstream = pOperator->pDownstream[0];
+
SIntervalAggOperatorInfo* pInfo = pOperator->info;
SExprSupp* pSup = &pOperator->exprSupp;
int32_t scanFlag = MAIN_SCAN;
-
- int64_t st = taosGetTimestampUs();
- SOperatorInfo* downstream = pOperator->pDownstream[0];
+ int64_t st = taosGetTimestampUs();
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
@@ -1268,15 +1268,11 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
}
SSDataBlock* pBlock = pInfo->binfo.pRes;
-
- ASSERT(pInfo->execModel == OPTR_EXEC_MODEL_BATCH);
-
pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
return NULL;
}
- blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
while (1) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL);
@@ -1649,23 +1645,34 @@ static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
// the primary timestamp column
bool needed = false;
- pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
- pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
- { // ts column
- SColumn c = {0};
- c.colId = 1;
- c.slotId = pInfo->primaryTsIndex;
- c.type = TSDB_DATA_TYPE_TIMESTAMP;
- c.bytes = sizeof(int64_t);
- taosArrayPush(pInfo->pInterpCols, &c);
+ for(int32_t i = 0; i < numOfCols; ++i) {
+ SExprInfo* pExpr = pCtx[i].pExpr;
+ if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
+ needed = true;
+ break;
+ }
+ }
- SGroupKeys key = {0};
- key.bytes = c.bytes;
- key.type = c.type;
- key.isNull = true; // to denote no value is assigned yet
- key.pData = taosMemoryCalloc(1, c.bytes);
- taosArrayPush(pInfo->pPrevValues, &key);
+ if (needed) {
+ pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
+ pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
+
+ { // ts column
+ SColumn c = {0};
+ c.colId = 1;
+ c.slotId = pInfo->primaryTsIndex;
+ c.type = TSDB_DATA_TYPE_TIMESTAMP;
+ c.bytes = sizeof(int64_t);
+ taosArrayPush(pInfo->pInterpCols, &c);
+
+ SGroupKeys key;
+ key.bytes = c.bytes;
+ key.type = c.type;
+ key.isNull = true; // to denote no value is assigned yet
+ key.pData = taosMemoryCalloc(1, c.bytes);
+ taosArrayPush(pInfo->pPrevValues, &key);
+ }
}
for (int32_t i = 0; i < numOfCols; ++i) {
@@ -1676,7 +1683,6 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
SColumn c = *pParam->pCol;
taosArrayPush(pInfo->pInterpCols, &c);
- needed = true;
SGroupKeys key = {0};
key.bytes = c.bytes;
@@ -1708,7 +1714,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
for (int32_t i = 0; i < numOfExpr; i++) {
- pCtx[i].isStream = true;
+// pCtx[i].isStream = true;
}
}
@@ -1727,7 +1733,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId;
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
- initResultSizeInfo(&pOperator->resultInfo, 4096);
+ initResultSizeInfo(&pOperator->resultInfo, 512);
+ blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
@@ -1755,7 +1762,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
pInfo->interval = interval;
- pInfo->execModel = pTaskInfo->execModel;
pInfo->twAggSup = as;
pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock;
@@ -1773,11 +1779,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
goto _error;
}
- if (isStream) {
- ASSERT(num > 0);
- initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
- }
-
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, pInfo);
if (pInfo->timeWindowInterpo) {
@@ -1792,7 +1793,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
pInfo, pTaskInfo);
pOperator->fpSet =
- createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL);
+ createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -2010,7 +2011,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet =
- createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL);
+ createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -2083,7 +2084,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL);
+ createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, optrDefaultBufFn, NULL);
pOperator->pTaskInfo = pTaskInfo;
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -2516,9 +2517,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
pOperator->status = OP_RES_TO_RETURN;
- qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi");
+ qDebug("===stream===return data:%s. recv datablock num:%" PRIu64 , IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack);
+ pInfo->numOfDatapack = 0;
break;
}
+ pInfo->numOfDatapack++;
printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv");
ASSERT(pBlock->info.type != STREAM_INVERT);
@@ -2734,6 +2737,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
pInfo->delKey.ts = INT64_MAX;
pInfo->delKey.groupId = 0;
+ pInfo->numOfDatapack = 0;
pOperator->operatorType = pPhyNode->type;
pOperator->blocking = true;
@@ -2741,7 +2745,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pOperator->info = pInfo;
pOperator->fpSet =
- createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL);
+ createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL);
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
}
@@ -3553,7 +3557,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL);
+ createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL);
if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
@@ -3697,8 +3701,8 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {
pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
- pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL,
- destroyStreamSessionAggOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL,
+ destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL);
}
setOperatorInfo(pOperator, name, pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo);
@@ -4058,7 +4062,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL);
+ createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -4306,12 +4310,11 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
iaInfo->win = pTaskInfo->window;
iaInfo->inputOrder = TSDB_ORDER_ASC;
iaInfo->interval = interval;
- iaInfo->execModel = pTaskInfo->execModel;
iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId;
iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock;
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
- initResultSizeInfo(&pOperator->resultInfo, 4096);
+ initResultSizeInfo(&pOperator->resultInfo, 512);
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
@@ -4336,7 +4339,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
false, OP_NOT_OPENED, miaInfo, pTaskInfo);
pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL);
+ createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -4612,7 +4615,6 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
pIntervalInfo->win = pTaskInfo->window;
pIntervalInfo->inputOrder = TSDB_ORDER_ASC;
pIntervalInfo->interval = interval;
- pIntervalInfo->execModel = pTaskInfo->execModel;
pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock;
pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
@@ -4642,7 +4644,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false,
OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo);
pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL);
+ createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@@ -4700,8 +4702,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
+ qDebug("===stream===return data:single interval. recv datablock num:%" PRIu64, pInfo->numOfDatapack);
+ pInfo->numOfDatapack = 0;
break;
}
+ pInfo->numOfDatapack++;
printDataBlock(pBlock, "single interval recv");
if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
@@ -4851,11 +4856,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->pChildren = NULL;
pInfo->delKey.ts = INT64_MAX;
pInfo->delKey.groupId = 0;
+ pInfo->numOfDatapack = 0;
setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
- pOperator->fpSet =
- createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL);
+ pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL,
+ destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL);
initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
code = appendDownstream(pOperator, &downstream, 1);
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index d5a2f77175..37688c0b53 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -34,14 +34,12 @@ struct SSortHandle {
int32_t pageSize;
int32_t numOfPages;
SDiskbasedBuf* pBuf;
-
- SArray* pSortInfo;
- SArray* pOrderedSource;
-
- int32_t loops;
- uint64_t sortElapsed;
- int64_t startTs;
- uint64_t totalElapsed;
+ SArray* pSortInfo;
+ SArray* pOrderedSource;
+ int32_t loops;
+ uint64_t sortElapsed;
+ int64_t startTs;
+ uint64_t totalElapsed;
int32_t sourceId;
SSDataBlock* pDataBlock;
@@ -99,9 +97,9 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page
}
static int32_t sortComparCleanup(SMsortComparParam* cmpParam) {
+ // NOTICE: pSource may be, if it is SORT_MULTISOURCE_MERGE
for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
- SSortSource* pSource =
- cmpParam->pSources[i]; // NOTICE: pSource may be SGenericSource *, if it is SORT_MULTISOURCE_MERGE
+ SSortSource* pSource = cmpParam->pSources[i];
blockDataDestroy(pSource->src.pBlock);
taosMemoryFreeClear(pSource);
}
@@ -231,15 +229,15 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId, pPageIdList);
}
-static void setCurrentSourceIsDone(SSortSource* pSource, SSortHandle* pHandle) {
+static void setCurrentSourceDone(SSortSource* pSource, SSortHandle* pHandle) {
pSource->src.rowIndex = -1;
++pHandle->numOfCompletedSources;
}
-static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex,
+static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32_t startIndex, int32_t endIndex,
SSortHandle* pHandle) {
- cmpParam->pSources = taosArrayGet(pSources, startIndex);
- cmpParam->numOfSources = (endIndex - startIndex + 1);
+ pParam->pSources = taosArrayGet(pSources, startIndex);
+ pParam->numOfSources = (endIndex - startIndex + 1);
int32_t code = 0;
@@ -247,7 +245,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
if (pHandle->pBuf == NULL) {
if (!osTempSpaceAvailable()) {
code = TSDB_CODE_NO_AVAIL_DISK;
- qError("Sort compare init failed since %s", terrstr(code));
+ qError("Sort compare init failed since %s, %s", terrstr(code), pHandle->idStr);
return code;
}
@@ -260,12 +258,12 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
}
if (pHandle->type == SORT_SINGLESOURCE_SORT) {
- for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
- SSortSource* pSource = cmpParam->pSources[i];
+ for (int32_t i = 0; i < pParam->numOfSources; ++i) {
+ SSortSource* pSource = pParam->pSources[i];
// set current source is done
if (taosArrayGetSize(pSource->pageIdList) == 0) {
- setCurrentSourceIsDone(pSource, pHandle);
+ setCurrentSourceDone(pSource, pHandle);
continue;
}
@@ -280,15 +278,21 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
releaseBufPage(pHandle->pBuf, pPage);
}
} else {
- for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
- SSortSource* pSource = cmpParam->pSources[i];
+ qDebug("start init for the multiway merge sort, %s", pHandle->idStr);
+ int64_t st = taosGetTimestampUs();
+
+ for (int32_t i = 0; i < pParam->numOfSources; ++i) {
+ SSortSource* pSource = pParam->pSources[i];
pSource->src.pBlock = pHandle->fetchfp(pSource->param);
// set current source is done
if (pSource->src.pBlock == NULL) {
- setCurrentSourceIsDone(pSource, pHandle);
+ setCurrentSourceDone(pSource, pHandle);
}
}
+
+ int64_t et = taosGetTimestampUs();
+ qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr);
}
return code;
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index fe010786eb..a2a827e7e5 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -2080,6 +2080,11 @@ static int32_t translateTagsPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, in
return TSDB_CODE_SUCCESS;
}
+static int32_t translateTableCountPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+ pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
+ return TSDB_CODE_SUCCESS;
+}
+
// clang-format off
const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
@@ -3241,6 +3246,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
+ {
+ .name = "_table_count",
+ .type = FUNCTION_TYPE_TABLE_COUNT,
+ .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC,
+ .translateFunc = translateTableCountPseudoColumn,
+ .getEnvFunc = NULL,
+ .initFunc = NULL,
+ .sprocessFunc = NULL,
+ .finalizeFunc = NULL
+ },
};
// clang-format on
diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c
index f06bafafe3..1c74d22a82 100644
--- a/source/libs/function/src/detail/tavgfunction.c
+++ b/source/libs/function/src/detail/tavgfunction.c
@@ -270,7 +270,7 @@ static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* p
#if __AVX2__
// find the start position that are aligned to 32bytes address in memory
- int32_t width = (bitWidth>>3u) / sizeof(int64_t);
+ int32_t width = (bitWidth >> 3u) / sizeof(int64_t);
int32_t remainder = numOfRows % width;
int32_t rounds = numOfRows / width;
@@ -286,20 +286,11 @@ static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* p
}
// let sum up the final results
- if (type == TSDB_DATA_TYPE_BIGINT) {
- const int64_t* q = (const int64_t*)∑
- pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
+ const int64_t* q = (const int64_t*)∑
+ pRes->sum.isum += q[0] + q[1] + q[2] + q[3];
- for (int32_t j = 0; j < remainder; ++j) {
- pRes->sum.isum += plist[j + rounds * width];
- }
- } else {
- const uint64_t* q = (const uint64_t*)∑
- pRes->sum.usum += q[0] + q[1] + q[2] + q[3];
-
- for (int32_t j = 0; j < remainder; ++j) {
- pRes->sum.usum += (uint64_t)plist[j + rounds * width];
- }
+ for (int32_t j = 0; j < remainder; ++j) {
+ pRes->sum.isum += plist[j + rounds * width];
}
#endif
@@ -588,14 +579,14 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
const int64_t* plist = (const int64_t*) pCol->pData;
// 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop
- if (simdAvailable) {
+ if (simdAvailable && type == TSDB_DATA_TYPE_BIGINT) {
i64VectorSumAVX2(plist, numOfRows, pAvgRes);
} else {
for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
if (type == TSDB_DATA_TYPE_BIGINT) {
pAvgRes->sum.isum += plist[i];
} else {
- pAvgRes->sum.isum += (uint64_t)plist[i];
+ pAvgRes->sum.usum += (uint64_t)plist[i];
}
}
}
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index d87df2d0f3..9d249e0831 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -598,9 +598,10 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
return 0;
}
static bool udfdRpcRfp(int32_t code, tmsg_t msgType) {
- if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
- code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) {
- if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) {
+ if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER ||
+ code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING ||
+ code == TSDB_CODE_APP_IS_STOPPING) {
+ if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) {
return false;
}
return true;
diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c
index 50e410b339..352e78c2fc 100644
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@@ -221,6 +221,8 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiTableScan";
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN:
return "PhysiTableSeqScan";
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN:
+ return "PhysiTableMergeScan";
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN:
return "PhysiSreamScan";
case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:
@@ -229,8 +231,8 @@ const char* nodesNodeName(ENodeType type) {
return "PhysiBlockDistScan";
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN:
return "PhysiLastRowScan";
- case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN:
- return "PhysiTableMergeScan";
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
+ return "PhysiTableCountScan";
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return "PhysiProject";
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
@@ -4646,6 +4648,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
return physiScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN:
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
return physiLastRowScanNodeToJson(pObj, pJson);
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN:
@@ -4800,6 +4803,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToLogicPlan(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
return jsonToPhysiScanNode(pJson, pObj);
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN:
return jsonToPhysiLastRowScanNode(pJson, pObj);
diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c
index 1e8ff8da1a..bf6cd33af7 100644
--- a/source/libs/nodes/src/nodesMsgFuncs.c
+++ b/source/libs/nodes/src/nodesMsgFuncs.c
@@ -3640,6 +3640,7 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = physiScanNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN:
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
code = physiLastRowScanNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
@@ -3778,6 +3779,7 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
code = msgToPhysiScanNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN:
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
code = msgToPhysiLastRowScanNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index 2f0f6b3432..f363cd0f10 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -494,6 +494,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SBlockDistScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN:
return makeNode(type, sizeof(SLastRowScanPhysiNode));
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
+ return makeNode(type, sizeof(STableCountScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN:
@@ -1120,6 +1122,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:
case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN:
+ case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN:
destroyScanPhysiNode((SScanPhysiNode*)pNode);
break;
case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: {
diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h
index c53d3f9320..ce5a63f5d0 100644
--- a/source/libs/parser/inc/parUtil.h
+++ b/source/libs/parser/inc/parUtil.h
@@ -86,7 +86,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta);
STableMeta* tableMetaDup(const STableMeta* pTableMeta);
int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
-int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
+int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache);
diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c
index 3481eddba4..f90a42add3 100644
--- a/source/libs/parser/src/parAstParser.c
+++ b/source/libs/parser/src/parAstParser.c
@@ -140,7 +140,7 @@ static int32_t collectMetaKeyFromInsTagsImpl(SCollectMetaKeyCxt* pCxt, SName* pN
static int32_t collectMetaKeyFromInsTags(SCollectMetaKeyCxt* pCxt) {
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pStmt;
SName name = {0};
- int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name);
+ int32_t code = getVnodeSysTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name);
if (TSDB_CODE_SUCCESS == code) {
code = collectMetaKeyFromInsTagsImpl(pCxt, &name);
}
@@ -165,7 +165,8 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c
if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) {
code = reserveDnodeRequiredInCache(pCxt->pMetaCache);
}
- if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) &&
+ if (TSDB_CODE_SUCCESS == code &&
+ (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) &&
QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) {
code = collectMetaKeyFromInsTags(pCxt);
}
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index d2d97f7b90..bca139664d 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -47,6 +47,7 @@ typedef struct STranslateContext {
SParseMetaCache* pMetaCache;
bool createStream;
bool stableQuery;
+ bool showRewrite;
} STranslateContext;
typedef struct SFullDatabaseName {
@@ -750,8 +751,8 @@ static bool isPrimaryKeyImpl(SNode* pExpr) {
return (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pExpr)->colId);
} else if (QUERY_NODE_FUNCTION == nodeType(pExpr)) {
SFunctionNode* pFunc = (SFunctionNode*)pExpr;
- if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_FIRST == pFunc->funcType ||
- FUNCTION_TYPE_LAST == pFunc->funcType) {
+ if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pFunc->funcType ||
+ FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType) {
return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0));
} else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType) {
return true;
@@ -2209,22 +2210,28 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) {
}
static bool sysTableFromVnode(const char* pTable) {
- return (0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) ||
- (0 == strcmp(pTable, TSDB_INS_TABLE_TABLE_DISTRIBUTED) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)));
+ return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)));
}
static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); }
-static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName,
- SArray** pVgroupList) {
+static int32_t getVnodeSysTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName,
+ SArray** pVgroupList) {
if (0 == pTargetName->type) {
return getDBVgInfoImpl(pCxt, pName, pVgroupList);
}
+ if (0 == strcmp(pTargetName->dbname, TSDB_INFORMATION_SCHEMA_DB) ||
+ 0 == strcmp(pTargetName->dbname, TSDB_PERFORMANCE_SCHEMA_DB)) {
+ pTargetName->type = 0;
+ return TSDB_CODE_SUCCESS;
+ }
+
if (TSDB_DB_NAME_T == pTargetName->type) {
int32_t code = getDBVgInfoImpl(pCxt, pTargetName, pVgroupList);
- if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code ||
- TSDB_CODE_MND_DB_IN_DROPPING == code) {
+ if (!pCxt->showRewrite && (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code ||
+ TSDB_CODE_MND_DB_IN_DROPPING == code)) {
+ // system table query should not report errors
code = TSDB_CODE_SUCCESS;
}
return code;
@@ -2241,50 +2248,44 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge
}
} else if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code ||
TSDB_CODE_MND_DB_IN_DROPPING == code) {
+ // system table query should not report errors
code = TSDB_CODE_SUCCESS;
}
return code;
}
-static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) {
+static int32_t getVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgs, bool* pHasUserDbCond) {
if (!isSelectStmt(pCxt->pCurrStmt)) {
return TSDB_CODE_SUCCESS;
}
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SName targetName = {0};
- int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName);
+ int32_t code = getVnodeSysTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName);
if (TSDB_CODE_SUCCESS == code) {
- code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList);
+ code = getVnodeSysTableVgroupListImpl(pCxt, &targetName, pName, pVgs);
}
+ *pHasUserDbCond = (0 != targetName.type && taosArrayGetSize(*pVgs) > 0);
return code;
}
static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
- int32_t code = TSDB_CODE_SUCCESS;
- SArray* vgroupList = NULL;
- if (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS)) {
- code = getTagsTableVgroupList(pCxt, pName, &vgroupList);
- } else if ('\0' != pRealTable->qualDbName[0]) {
- if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) {
- code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList);
- }
- } else {
- code = getDBVgInfoImpl(pCxt, pName, &vgroupList);
- }
+ bool hasUserDbCond = false;
+ SArray* pVgs = NULL;
+ int32_t code = getVnodeSysTableVgroupList(pCxt, pName, &pVgs, &hasUserDbCond);
if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) &&
- isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(vgroupList)) {
+ isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(pVgs)) {
((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true;
}
- if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) {
- code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList);
+ if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) {
+ code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &pVgs);
}
if (TSDB_CODE_SUCCESS == code) {
- code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList);
+ code = toVgroupsInfo(pVgs, &pRealTable->pVgroupList);
}
- taosArrayDestroy(vgroupList);
+ taosArrayDestroy(pVgs);
return code;
}
@@ -2309,30 +2310,39 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea
}
}
+static int32_t setSuperTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
+ SArray* vgroupList = NULL;
+ int32_t code = getDBVgInfoImpl(pCxt, pName, &vgroupList);
+ if (TSDB_CODE_SUCCESS == code) {
+ code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList);
+ }
+ taosArrayDestroy(vgroupList);
+ return code;
+}
+
+static int32_t setNormalTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
+ pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo));
+ if (NULL == pRealTable->pVgroupList) {
+ return TSDB_CODE_OUT_OF_MEMORY;
+ }
+ pRealTable->pVgroupList->numOfVgroups = 1;
+ return getTableHashVgroupImpl(pCxt, pName, pRealTable->pVgroupList->vgroups);
+}
+
static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
if (pCxt->pParseCxt->topicQuery) {
return TSDB_CODE_SUCCESS;
}
- int32_t code = TSDB_CODE_SUCCESS;
if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) {
- SArray* vgroupList = NULL;
- code = getDBVgInfoImpl(pCxt, pName, &vgroupList);
- if (TSDB_CODE_SUCCESS == code) {
- code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList);
- }
- taosArrayDestroy(vgroupList);
- } else if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) {
- code = setSysTableVgroupList(pCxt, pName, pRealTable);
- } else {
- pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo));
- if (NULL == pRealTable->pVgroupList) {
- return TSDB_CODE_OUT_OF_MEMORY;
- }
- pRealTable->pVgroupList->numOfVgroups = 1;
- code = getTableHashVgroupImpl(pCxt, pName, pRealTable->pVgroupList->vgroups);
+ return setSuperTableVgroupList(pCxt, pName, pRealTable);
}
- return code;
+
+ if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) {
+ return setSysTableVgroupList(pCxt, pName, pRealTable);
+ }
+
+ return setNormalTableVgroupList(pCxt, pName, pRealTable);
}
static uint8_t getStmtPrecision(SNode* pStmt) {
@@ -2366,7 +2376,6 @@ static bool isSingleTable(SRealTableNode* pRealTable) {
int8_t tableType = pRealTable->pMeta->tableType;
if (TSDB_SYSTEM_TABLE == tableType) {
return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) &&
- 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLE_DISTRIBUTED) &&
0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS);
}
return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType);
@@ -6296,6 +6305,7 @@ static int32_t rewriteShow(STranslateContext* pCxt, SQuery* pQuery) {
code = createShowCondition((SShowStmt*)pQuery->pRoot, pStmt);
}
if (TSDB_CODE_SUCCESS == code) {
+ pCxt->showRewrite = true;
pQuery->showRewrite = true;
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = (SNode*)pStmt;
@@ -6349,6 +6359,7 @@ static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) {
code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags);
}
if (TSDB_CODE_SUCCESS == code) {
+ pCxt->showRewrite = true;
pQuery->showRewrite = true;
pSelect->tagScan = true;
nodesDestroyNode(pQuery->pRoot);
@@ -6379,6 +6390,7 @@ static int32_t rewriteShowDnodeVariables(STranslateContext* pCxt, SQuery* pQuery
}
}
if (TSDB_CODE_SUCCESS == code) {
+ pCxt->showRewrite = true;
pQuery->showRewrite = true;
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = (SNode*)pSelect;
@@ -6398,6 +6410,7 @@ static int32_t rewriteShowVnodes(STranslateContext* pCxt, SQuery* pQuery) {
}
}
if (TSDB_CODE_SUCCESS == code) {
+ pCxt->showRewrite = true;
pQuery->showRewrite = true;
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = (SNode*)pStmt;
@@ -6439,6 +6452,7 @@ static int32_t rewriteShowTableDist(STranslateContext* pCxt, SQuery* pQuery) {
code = nodesListMakeStrictAppend(&pStmt->pProjectionList, createBlockDistFunc());
}
if (TSDB_CODE_SUCCESS == code) {
+ pCxt->showRewrite = true;
pQuery->showRewrite = true;
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = (SNode*)pStmt;
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 7a56b0e0fa..d9fc83c20f 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -474,7 +474,7 @@ static int32_t getInsTagsTableTargetNameFromCond(int32_t acctId, SLogicCondition
return TSDB_CODE_SUCCESS;
}
-int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) {
+int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) {
if (NULL == pWhere) {
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp
index de7487434c..cc51beb842 100644
--- a/source/libs/parser/test/mockCatalog.cpp
+++ b/source/libs/parser/test/mockCatalog.cpp
@@ -65,9 +65,10 @@ void generateInformationSchema(MockCatalogService* mcs) {
.addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN)
.addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN)
.done();
- mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 2)
- .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN)
+ mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 3)
.addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN)
+ .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN)
+ .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN)
.done();
mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLE_DISTRIBUTED, TSDB_SYSTEM_TABLE, 2)
.addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN)
@@ -140,7 +141,7 @@ void generatePerformanceSchema(MockCatalogService* mcs) {
void generateTestTables(MockCatalogService* mcs, const std::string& db) {
mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6)
.setPrecision(TSDB_TIME_PRECISION_MILLI)
- .setVgid(1)
+ .setVgid(2)
.addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
.addColumn("c1", TSDB_DATA_TYPE_INT)
.addColumn("c2", TSDB_DATA_TYPE_BINARY, 20)
@@ -182,9 +183,9 @@ void generateTestStables(MockCatalogService* mcs, const std::string& db) {
.addTag("tag2", TSDB_DATA_TYPE_BINARY, 20)
.addTag("tag3", TSDB_DATA_TYPE_TIMESTAMP);
builder.done();
- mcs->createSubTable(db, "st1", "st1s1", 1);
- mcs->createSubTable(db, "st1", "st1s2", 2);
- mcs->createSubTable(db, "st1", "st1s3", 1);
+ mcs->createSubTable(db, "st1", "st1s1", 2);
+ mcs->createSubTable(db, "st1", "st1s2", 3);
+ mcs->createSubTable(db, "st1", "st1s3", 2);
}
{
ITableBuilder& builder = mcs->createTableBuilder(db, "st2", TSDB_SUPER_TABLE, 3, 1)
@@ -194,8 +195,8 @@ void generateTestStables(MockCatalogService* mcs, const std::string& db) {
.addColumn("c2", TSDB_DATA_TYPE_BINARY, 20)
.addTag("jtag", TSDB_DATA_TYPE_JSON);
builder.done();
- mcs->createSubTable(db, "st2", "st2s1", 1);
- mcs->createSubTable(db, "st2", "st2s2", 2);
+ mcs->createSubTable(db, "st2", "st2s1", 2);
+ mcs->createSubTable(db, "st2", "st2s2", 3);
}
}
diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp
index be2e4b90b9..bed8f82d3a 100644
--- a/source/libs/parser/test/mockCatalogService.cpp
+++ b/source/libs/parser/test/mockCatalogService.cpp
@@ -20,15 +20,19 @@
#include