Merge remote-tracking branch 'origin/3.0' into enh/insert_multi_table_optimize
This commit is contained in:
dapan1121
commit
051ac7d874
|
|
@ -2,7 +2,7 @@
|
|||
# taos-tools
|
||||
ExternalProject_Add(taos-tools
|
||||
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
|
||||
GIT_TAG cf30c86
|
||||
GIT_TAG 823fae5
|
||||
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
|
||||
BINARY_DIR ""
|
||||
#BUILD_IN_SOURCE TRUE
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
---
|
||||
title: Write from Kafka
|
||||
---
|
||||
|
||||
import Tabs from "@theme/Tabs";
|
||||
import TabItem from "@theme/TabItem";
|
||||
import PyKafka from "./_py_kafka.mdx";
|
||||
|
||||
## About Kafka
|
||||
|
||||
Apache Kafka is an open-source distributed event streaming platform, used by thousands of companies for high-performance data pipelines, streaming analytics, data integration, and mission-critical applications. For the key concepts of kafka, please refer to [kafka documentation](https://kafka.apache.org/documentation/#gettingStarted).
|
||||
|
||||
### kafka topic
|
||||
|
||||
Messages in Kafka are organized by topics. A topic may have one or more partitions. We can manage kafka topics through `kafka-topics`.
|
||||
|
||||
create a topic named `kafka-events`:
|
||||
|
||||
```
|
||||
bin/kafka-topics.sh --create --topic kafka-events --bootstrap-server localhost:9092
|
||||
```
|
||||
|
||||
Alter `kafka-events` topic to set partitions to 3:
|
||||
|
||||
```
|
||||
bin/kafka-topics.sh --alter --topic kafka-events --partitions 3 --bootstrap-server=localhost:9092
|
||||
```
|
||||
|
||||
Show all topics and partitions in Kafka:
|
||||
|
||||
```
|
||||
bin/kafka-topics.sh --bootstrap-server=localhost:9092 --describe
|
||||
```
|
||||
|
||||
## Insert into TDengine
|
||||
|
||||
We can write data into TDengine via SQL or Schemaless. For more information, please refer to [Insert Using SQL](/develop/insert-data/sql-writing/) or [High Performance Writing](/develop/insert-data/high-volume/) or [Schemaless Writing](/reference/schemaless/).
|
||||
|
||||
## Examples
|
||||
|
||||
<Tabs defaultValue="Python" groupId="lang">
|
||||
<TabItem label="Python" value="Python">
|
||||
<PyKafka />
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
### python Kafka 客户端
|
||||
|
||||
For python kafka client, please refer to [kafka client](https://cwiki.apache.org/confluence/display/KAFKA/Clients#Clients-Python). In this document, we use [kafka-python](http://github.com/dpkp/kafka-python).
|
||||
|
||||
### consume from Kafka
|
||||
|
||||
The simple way to consume messages from Kafka is to read messages one by one. The demo is as follows:
|
||||
|
||||
```
|
||||
from kafka import KafkaConsumer
|
||||
consumer = KafkaConsumer('my_favorite_topic')
|
||||
for msg in consumer:
|
||||
print (msg)
|
||||
```
|
||||
|
||||
For higher performance, we can consume message from kafka in batch. The demo is as follows:
|
||||
|
||||
```
|
||||
from kafka import KafkaConsumer
|
||||
consumer = KafkaConsumer('my_favorite_topic')
|
||||
while True:
|
||||
msgs = consumer.poll(timeout_ms=500, max_records=1000)
|
||||
if msgs:
|
||||
print (msgs)
|
||||
```
|
||||
|
||||
### multi-threading
|
||||
|
||||
For more higher performance we can process data from kafka in multi-thread. We can use python's ThreadPoolExecutor to achieve multithreading. The demo is as follows:
|
||||
|
||||
```
|
||||
from concurrent.futures import ThreadPoolExecutor, Future
|
||||
pool = ThreadPoolExecutor(max_workers=10)
|
||||
pool.submit(...)
|
||||
```
|
||||
|
||||
### multi-process
|
||||
|
||||
For more higher performance, sometimes we use multiprocessing. In this case, the number of Kafka Consumers should not be greater than the number of Kafka Topic Partitions. The demo is as follows:
|
||||
|
||||
```
|
||||
from multiprocessing import Process
|
||||
|
||||
ps = []
|
||||
for i in range(5):
|
||||
p = Process(target=Consumer().consume())
|
||||
p.start()
|
||||
ps.append(p)
|
||||
|
||||
for p in ps:
|
||||
p.join()
|
||||
```
|
||||
|
||||
In addition to python's built-in multithreading and multiprocessing library, we can also use the third-party library gunicorn.
|
||||
|
||||
### Examples
|
||||
|
||||
```py
|
||||
{{#include docs/examples/python/kafka_example.py}}
|
||||
```
|
||||
|
|
@ -76,7 +76,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource
|
|||
You can also download zip files from [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) or [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) and install manually. The commands are as follows:
|
||||
|
||||
```bash
|
||||
GF_VERSION=3.2.2
|
||||
GF_VERSION=3.2.7
|
||||
# from GitHub
|
||||
wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip
|
||||
# from Grafana
|
||||
|
|
|
|||
|
|
@ -38,15 +38,9 @@ Download the latest TDengine-server from the [Downloads](http://tdengine.com/en/
|
|||
|
||||
## Data Connection Setup
|
||||
|
||||
### Download TDengine plug-in to grafana plug-in directory
|
||||
### Install Grafana Plugin and Configure Data Source
|
||||
|
||||
```bash
|
||||
1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip
|
||||
2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/
|
||||
3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine
|
||||
4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini
|
||||
5. sudo systemctl restart grafana-server.service
|
||||
```
|
||||
Please refer to [Install Grafana Plugin and Configure Data Source](/third-party/grafana/#install-grafana-plugin-and-configure-data-source)
|
||||
|
||||
### Modify /etc/telegraf/telegraf.conf
|
||||
|
||||
|
|
|
|||
|
|
@ -41,15 +41,9 @@ Download the latest TDengine-server from the [Downloads](http://tdengine.com/en/
|
|||
|
||||
## Data Connection Setup
|
||||
|
||||
### Copy the TDengine plugin to the grafana plugin directory
|
||||
### Install Grafana Plugin and Configure Data Source
|
||||
|
||||
```bash
|
||||
1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip
|
||||
2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/
|
||||
3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine
|
||||
4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini
|
||||
5. sudo systemctl restart grafana-server.service
|
||||
```
|
||||
Please refer to [Install Grafana Plugin and Configure Data Source](/third-party/grafana/#install-grafana-plugin-and-configure-data-source)
|
||||
|
||||
### Configure collectd
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,192 @@
|
|||
#! encoding = utf-8
|
||||
import json
|
||||
import time
|
||||
from json import JSONDecodeError
|
||||
from typing import Callable
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor, Future
|
||||
|
||||
import taos
|
||||
from kafka import KafkaConsumer
|
||||
from kafka.consumer.fetcher import ConsumerRecord
|
||||
|
||||
|
||||
class Consumer(object):
|
||||
DEFAULT_CONFIGS = {
|
||||
'kafka_brokers': 'localhost:9092',
|
||||
'kafka_topic': 'python_kafka',
|
||||
'kafka_group_id': 'taos',
|
||||
'taos_host': 'localhost',
|
||||
'taos_user': 'root',
|
||||
'taos_password': 'taosdata',
|
||||
'taos_database': 'power',
|
||||
'taos_port': 6030,
|
||||
'timezone': None,
|
||||
'clean_after_testing': False,
|
||||
'bath_consume': True,
|
||||
'batch_size': 1000,
|
||||
'async_model': True,
|
||||
'workers': 10
|
||||
}
|
||||
|
||||
LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose',
|
||||
'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale',
|
||||
'California.SantaClara', 'California.Cupertino']
|
||||
|
||||
CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1'
|
||||
USE_DATABASE_SQL = 'use {}'
|
||||
DROP_TABLE_SQL = 'drop table if exists meters'
|
||||
DROP_DATABASE_SQL = 'drop database if exists {}'
|
||||
CREATE_STABLE_SQL = 'create stable meters (ts timestamp, current float, voltage int, phase float) ' \
|
||||
'tags (location binary(64), groupId int)'
|
||||
CREATE_TABLE_SQL = 'create table if not exists {} using meters tags (\'{}\', {})'
|
||||
INSERT_SQL_HEADER = "insert into "
|
||||
INSERT_PART_SQL = 'power.{} values (\'{}\', {}, {}, {})'
|
||||
|
||||
def __init__(self, **configs):
|
||||
self.config: dict = self.DEFAULT_CONFIGS
|
||||
self.config.update(configs)
|
||||
self.consumer = KafkaConsumer(
|
||||
self.config.get('kafka_topic'), # topic
|
||||
bootstrap_servers=self.config.get('kafka_brokers'),
|
||||
group_id=self.config.get('kafka_group_id'),
|
||||
)
|
||||
self.taos = taos.connect(
|
||||
host=self.config.get('taos_host'),
|
||||
user=self.config.get('taos_user'),
|
||||
password=self.config.get('taos_password'),
|
||||
port=self.config.get('taos_port'),
|
||||
timezone=self.config.get('timezone'),
|
||||
)
|
||||
if self.config.get('async_model'):
|
||||
self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers'))
|
||||
self.tasks: list[Future] = []
|
||||
# tags and table mapping # key: {location}_{groupId} value:
|
||||
self.tag_table_mapping = {}
|
||||
i = 0
|
||||
for location in self.LOCATIONS:
|
||||
for j in range(1, 11):
|
||||
table_name = 'd{}'.format(i)
|
||||
self._cache_table(location=location, group_id=j, table_name=table_name)
|
||||
i += 1
|
||||
|
||||
def init_env(self):
|
||||
# create database and table
|
||||
self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database')))
|
||||
self.taos.execute(self.CREATE_DATABASE_SQL.format(self.config.get('taos_database')))
|
||||
self.taos.execute(self.USE_DATABASE_SQL.format(self.config.get('taos_database')))
|
||||
self.taos.execute(self.DROP_TABLE_SQL)
|
||||
self.taos.execute(self.CREATE_STABLE_SQL)
|
||||
for tags, table_name in self.tag_table_mapping.items():
|
||||
location, group_id = _get_location_and_group(tags)
|
||||
self.taos.execute(self.CREATE_TABLE_SQL.format(table_name, location, group_id))
|
||||
|
||||
def consume(self):
|
||||
logging.warning('## start consumer topic-[%s]', self.config.get('kafka_topic'))
|
||||
try:
|
||||
if self.config.get('bath_consume'):
|
||||
self._run_batch(self._to_taos_batch)
|
||||
else:
|
||||
self._run(self._to_taos)
|
||||
except KeyboardInterrupt:
|
||||
logging.warning("## caught keyboard interrupt, stopping")
|
||||
finally:
|
||||
self.stop()
|
||||
|
||||
def stop(self):
|
||||
# close consumer
|
||||
if self.consumer is not None:
|
||||
self.consumer.commit()
|
||||
self.consumer.close()
|
||||
|
||||
# multi thread
|
||||
if self.config.get('async_model'):
|
||||
for task in self.tasks:
|
||||
while not task.done():
|
||||
pass
|
||||
if self.pool is not None:
|
||||
self.pool.shutdown()
|
||||
|
||||
# clean data
|
||||
if self.config.get('clean_after_testing'):
|
||||
self.taos.execute(self.DROP_TABLE_SQL)
|
||||
self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database')))
|
||||
# close taos
|
||||
if self.taos is not None:
|
||||
self.taos.close()
|
||||
|
||||
def _run(self, f: Callable[[ConsumerRecord], bool]):
|
||||
for message in self.consumer:
|
||||
if self.config.get('async_model'):
|
||||
self.pool.submit(f(message))
|
||||
else:
|
||||
f(message)
|
||||
|
||||
def _run_batch(self, f: Callable[[list[list[ConsumerRecord]]], None]):
|
||||
while True:
|
||||
messages = self.consumer.poll(timeout_ms=500, max_records=self.config.get('batch_size'))
|
||||
if messages:
|
||||
if self.config.get('async_model'):
|
||||
self.pool.submit(f, messages.values())
|
||||
else:
|
||||
f(list(messages.values()))
|
||||
if not messages:
|
||||
time.sleep(0.1)
|
||||
|
||||
def _to_taos(self, message: ConsumerRecord) -> bool:
|
||||
sql = self.INSERT_SQL_HEADER + self._build_sql(message.value)
|
||||
if len(sql) == 0: # decode error, skip
|
||||
return True
|
||||
logging.info('## insert sql %s', sql)
|
||||
return self.taos.execute(sql=sql) == 1
|
||||
|
||||
def _to_taos_batch(self, messages: list[list[ConsumerRecord]]):
|
||||
sql = self._build_sql_batch(messages=messages)
|
||||
if len(sql) == 0: # decode error, skip
|
||||
return
|
||||
self.taos.execute(sql=sql)
|
||||
|
||||
def _build_sql(self, msg_value: str) -> str:
|
||||
try:
|
||||
data = json.loads(msg_value)
|
||||
except JSONDecodeError as e:
|
||||
logging.error('## decode message [%s] error ', msg_value, e)
|
||||
return ''
|
||||
location = data.get('location')
|
||||
group_id = data.get('groupId')
|
||||
ts = data.get('ts')
|
||||
current = data.get('current')
|
||||
voltage = data.get('voltage')
|
||||
phase = data.get('phase')
|
||||
|
||||
table_name = self._get_table_name(location=location, group_id=group_id)
|
||||
return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase)
|
||||
|
||||
def _build_sql_batch(self, messages: list[list[ConsumerRecord]]) -> str:
|
||||
sql_list = []
|
||||
for partition_messages in messages:
|
||||
for message in partition_messages:
|
||||
sql_list.append(self._build_sql(message.value))
|
||||
|
||||
return self.INSERT_SQL_HEADER + ' '.join(sql_list)
|
||||
|
||||
def _cache_table(self, location: str, group_id: int, table_name: str):
|
||||
self.tag_table_mapping[_tag_table_mapping_key(location=location, group_id=group_id)] = table_name
|
||||
|
||||
def _get_table_name(self, location: str, group_id: int) -> str:
|
||||
return self.tag_table_mapping.get(_tag_table_mapping_key(location=location, group_id=group_id))
|
||||
|
||||
|
||||
def _tag_table_mapping_key(location: str, group_id: int):
|
||||
return '{}_{}'.format(location, group_id)
|
||||
|
||||
|
||||
def _get_location_and_group(key: str) -> (str, int):
|
||||
fields = key.split('_')
|
||||
return fields[0], fields[1]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
consumer = Consumer(async_model=True)
|
||||
consumer.init_env()
|
||||
consumer.consume()
|
||||
|
|
@ -69,7 +69,7 @@ TDengine 的主要功能如下：
|
|||
|
||||
- **[分析能力](https://www.taosdata.com/tdengine/easy_data_analytics)**：通过超级表、存储计算分离、分区分片、预计算和其它技术，TDengine 能够高效地浏览、格式化和访问数据。
|
||||
|
||||
- **[核心开源](https://www.taosdata.com/tdengine/open_source_time-series_database)**：TDengine 的核心代码包括集群功能全部在开源协议下公开。全球超过 140k 个运行实例，GitHub Star 19k，且拥有一个活跃的开发者社区。
|
||||
- **[核心开源](https://www.taosdata.com/tdengine/open_source_time-series_database)**：TDengine 的核心代码包括集群功能全部在开源协议下公开。全球超过 140k 个运行实例，GitHub Star 20k，且拥有一个活跃的开发者社区。
|
||||
|
||||
采用 TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。表现在几个方面：
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
---
|
||||
title: 从 Kafka 写入
|
||||
---
|
||||
|
||||
import Tabs from "@theme/Tabs";
|
||||
import TabItem from "@theme/TabItem";
|
||||
import PyKafka from "./_py_kafka.mdx";
|
||||
|
||||
## Kafka 介绍
|
||||
|
||||
Apache Kafka 是开源的分布式消息分发平台，被广泛应用于高性能数据管道、流式数据分析、数据集成和事件驱动类型的应用程序。Kafka 包含 Producer、Consumer 和 Topic，其中 Producer 是向 Kafka 发送消息的进程，Consumer 是从 Kafka 消费消息的进程。Kafka 相关概念可以参考[官方文档](https://kafka.apache.org/documentation/#gettingStarted)。
|
||||
|
||||
|
||||
### kafka topic
|
||||
|
||||
Kafka 的消息按 topic 组织，每个 topic 会有一到多个 partition。可以通过 kafka 的 `kafka-topics` 管理 topic。
|
||||
|
||||
创建名为 `kafka-events` 的topic:
|
||||
|
||||
```
|
||||
bin/kafka-topics.sh --create --topic kafka-events --bootstrap-server localhost:9092
|
||||
```
|
||||
|
||||
修改 `kafka-events` 的 partition 数量为 3:
|
||||
|
||||
```
|
||||
bin/kafka-topics.sh --alter --topic kafka-events --partitions 3 --bootstrap-server=localhost:9092
|
||||
```
|
||||
|
||||
展示所有的 topic 和 partition:
|
||||
|
||||
```
|
||||
bin/kafka-topics.sh --bootstrap-server=localhost:9092 --describe
|
||||
```
|
||||
|
||||
## 写入 TDengine
|
||||
|
||||
TDengine 支持 Sql 方式和 Schemaless 方式的数据写入，Sql 方式数据写入可以参考 [TDengine SQL 写入](/develop/insert-data/sql-writing/) 和 [TDengine 高效写入](/develop/insert-data/high-volume/)。Schemaless 方式数据写入可以参考 [TDengine Schemaless 写入](/reference/schemaless/) 文档。
|
||||
|
||||
## 示例代码
|
||||
|
||||
<Tabs defaultValue="Python" groupId="lang">
|
||||
<TabItem label="Python" value="Python">
|
||||
<PyKafka />
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
### python Kafka 客户端
|
||||
|
||||
Kafka 的 python 客户端可以参考文档 [kafka client](https://cwiki.apache.org/confluence/display/KAFKA/Clients#Clients-Python)。推荐使用 [confluent-kafka-python](https://github.com/confluentinc/confluent-kafka-python) 和 [kafka-python](http://github.com/dpkp/kafka-python)。以下示例以 [kafka-python](http://github.com/dpkp/kafka-python) 为例。
|
||||
|
||||
### 从 Kafka 消费数据
|
||||
|
||||
Kafka 客户端采用 pull 的方式从 Kafka 消费数据，可以采用单条消费的方式或批量消费的方式读取数据。使用 [kafka-python](http://github.com/dpkp/kafka-python) 客户端单条消费数据的示例如下：
|
||||
|
||||
```
|
||||
from kafka import KafkaConsumer
|
||||
consumer = KafkaConsumer('my_favorite_topic')
|
||||
for msg in consumer:
|
||||
print (msg)
|
||||
```
|
||||
|
||||
单条消费的方式在数据流量大的情况下往往存在性能瓶颈，导致 Kafka 消息积压，更推荐使用批量消费的方式消费数据。使用 [kafka-python](http://github.com/dpkp/kafka-python) 客户端批量消费数据的示例如下：
|
||||
|
||||
```
|
||||
from kafka import KafkaConsumer
|
||||
consumer = KafkaConsumer('my_favorite_topic')
|
||||
while True:
|
||||
msgs = consumer.poll(timeout_ms=500, max_records=1000)
|
||||
if msgs:
|
||||
print (msgs)
|
||||
```
|
||||
|
||||
### Python 多线程
|
||||
|
||||
为了提高数据写入效率，通常采用多线程的方式写入数据，可以使用 python 线程池 ThreadPoolExecutor 实现多线程。示例代码如下：
|
||||
|
||||
```
|
||||
from concurrent.futures import ThreadPoolExecutor, Future
|
||||
pool = ThreadPoolExecutor(max_workers=10)
|
||||
pool.submit(...)
|
||||
```
|
||||
|
||||
### Python 多进程
|
||||
|
||||
单个python进程不能充分发挥多核 CPU 的性能，有时候我们会选择多进程的方式。在多进程的情况下，需要注意，Kafka Consumer 的数量应该小于等于 Kafka Topic Partition 数量。Python 多进程示例代码如下：
|
||||
|
||||
```
|
||||
from multiprocessing import Process
|
||||
|
||||
ps = []
|
||||
for i in range(5):
|
||||
p = Process(target=Consumer().consume())
|
||||
p.start()
|
||||
ps.append(p)
|
||||
|
||||
for p in ps:
|
||||
p.join()
|
||||
```
|
||||
|
||||
除了 Python 内置的多线程和多进程方式，还可以通过第三方库 gunicorn 实现并发。
|
||||
|
||||
### 完整示例
|
||||
|
||||
```py
|
||||
{{#include docs/examples/python/kafka_example.py}}
|
||||
```
|
||||
|
|
@ -39,15 +39,9 @@ IT 运维监测数据通常都是对时间特性比较敏感的数据，例如
|
|||
|
||||
## 数据链路设置
|
||||
|
||||
### 下载 TDengine 插件到 Grafana 插件目录
|
||||
### 安装 Grafana Plugin 并配置数据源
|
||||
|
||||
```bash
|
||||
1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip
|
||||
2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/
|
||||
3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine
|
||||
4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini
|
||||
5. sudo systemctl restart grafana-server.service
|
||||
```
|
||||
请参考[安装 Grafana Plugin 并配置数据源](/third-party/grafana/#%E5%AE%89%E8%A3%85-grafana-plugin-%E5%B9%B6%E9%85%8D%E7%BD%AE%E6%95%B0%E6%8D%AE%E6%BA%90)。
|
||||
|
||||
### 修改 /etc/telegraf/telegraf.conf
|
||||
|
||||
|
|
|
|||
|
|
@ -41,15 +41,9 @@ IT 运维监测数据通常都是对时间特性比较敏感的数据，例如
|
|||
|
||||
## 数据链路设置
|
||||
|
||||
### 复制 TDengine 插件到 grafana 插件目录
|
||||
### 安装 Grafana Plugin 并配置数据源
|
||||
|
||||
```bash
|
||||
1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.3/tdengine-datasource-3.1.3.zip
|
||||
2. sudo unzip tdengine-datasource-3.1.3.zip -d /var/lib/grafana/plugins/
|
||||
3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine
|
||||
4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini
|
||||
5. sudo systemctl restart grafana-server.service
|
||||
```
|
||||
请参考[安装 Grafana Plugin 并配置数据源](/third-party/grafana/#%E5%AE%89%E8%A3%85-grafana-plugin-%E5%B9%B6%E9%85%8D%E7%BD%AE%E6%95%B0%E6%8D%AE%E6%BA%90)。
|
||||
|
||||
### 配置 collectd
|
||||
|
||||
|
|
|
|||
|
|
@ -201,3 +201,45 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端
|
|||
OOM 是操作系统的保护机制，当操作系统内存(包括 SWAP )不足时，会杀掉某些进程，从而保证操作系统的稳定运行。通常内存不足主要是如下两个原因导致，一是剩余内存小于 vm.min_free_kbytes ；二是程序请求的内存大于剩余内存。还有一种情况是内存充足但程序占用了特殊的内存地址，也会触发 OOM 。
|
||||
|
||||
TDengine 会预先为每个 VNode 分配好内存，每个 Database 的 VNode 个数受 建库时的vgroups参数影响，每个 VNode 占用的内存大小受 buffer参数 影响。要防止 OOM，需要在项目建设之初合理规划内存，并合理设置 SWAP ，除此之外查询过量的数据也有可能导致内存暴涨，这取决于具体的查询语句。TDengine 企业版对内存管理做了优化，采用了新的内存分配器，对稳定性有更高要求的用户可以考虑选择企业版。
|
||||
|
||||
### 19. 在macOS上遇到Too many open files怎么办？
|
||||
|
||||
taosd日志文件报错Too many open file，是由于taosd打开文件数超过系统设置的上限所致。
|
||||
解决方案如下：
|
||||
1. 新建文件 /Library/LaunchDaemons/limit.maxfiles.plist，写入以下内容(以下示例将limit和maxfiles改为10万，可按需修改)：
|
||||
```
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
|
||||
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>limit.maxfiles</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>launchctl</string>
|
||||
<string>limit</string>
|
||||
<string>maxfiles</string>
|
||||
<string>100000</string>
|
||||
<string>100000</string>
|
||||
</array>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>ServiceIPC</key>
|
||||
<false/>
|
||||
</dict>
|
||||
</plist>
|
||||
```
|
||||
2. 修改文件权限
|
||||
```
|
||||
sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
|
||||
sudo chmod 644 /Library/LaunchDaemons/limit.maxfiles.plist
|
||||
```
|
||||
3. 加载 plist 文件 (或重启系统后生效。launchd在启动时会自动加载该目录的 plist)
|
||||
```
|
||||
sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist
|
||||
```
|
||||
4.确认更改后的限制
|
||||
```
|
||||
launchctl limit maxfiles
|
||||
```
|
||||
|
|
|
|||
|
|
@ -307,6 +307,7 @@ DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res);
|
|||
DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw);
|
||||
DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw);
|
||||
DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname);
|
||||
DLL_EXPORT int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields);
|
||||
DLL_EXPORT void tmq_free_raw(tmq_raw_data raw);
|
||||
// Returning null means error. Returned result need to be freed by tmq_free_json_meta
|
||||
DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res);
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ extern "C" {
|
|||
#define TSDB_INS_TABLE_TOPICS "ins_topics"
|
||||
#define TSDB_INS_TABLE_STREAMS "ins_streams"
|
||||
#define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks"
|
||||
#define TSDB_INS_TABLE_USER_PRIVILEGES "ins_user_privileges"
|
||||
|
||||
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
|
||||
#define TSDB_PERFS_TABLE_SMAS "perf_smas"
|
||||
|
|
|
|||
|
|
@ -129,6 +129,7 @@ extern char tsUdfdLdLibPath[];
|
|||
extern char tsSmlChildTableName[];
|
||||
extern char tsSmlTagName[];
|
||||
extern bool tsSmlDataFormat;
|
||||
extern int32_t tsSmlBatchSize;
|
||||
|
||||
// wal
|
||||
extern int64_t tsWalFsyncDataSizeLimit;
|
||||
|
|
@ -136,6 +137,7 @@ extern int64_t tsWalFsyncDataSizeLimit;
|
|||
// internal
|
||||
extern int32_t tsTransPullupInterval;
|
||||
extern int32_t tsMqRebalanceInterval;
|
||||
extern int32_t tsStreamCheckpointTickInterval;
|
||||
extern int32_t tsTtlUnit;
|
||||
extern int32_t tsTtlPushInterval;
|
||||
extern int32_t tsGrantHBInterval;
|
||||
|
|
|
|||
|
|
@ -120,6 +120,7 @@ typedef enum _mgmt_table {
|
|||
TSDB_MGMT_TABLE_VNODES,
|
||||
TSDB_MGMT_TABLE_APPS,
|
||||
TSDB_MGMT_TABLE_STREAM_TASKS,
|
||||
TSDB_MGMT_TABLE_PRIVILEGES,
|
||||
TSDB_MGMT_TABLE_MAX,
|
||||
} EShowType;
|
||||
|
||||
|
|
@ -151,6 +152,8 @@ typedef enum _mgmt_table {
|
|||
#define TSDB_ALTER_USER_REMOVE_ALL_DB 0x8
|
||||
#define TSDB_ALTER_USER_ENABLE 0x9
|
||||
#define TSDB_ALTER_USER_SYSINFO 0xA
|
||||
#define TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC 0xB
|
||||
#define TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC 0xC
|
||||
|
||||
#define TSDB_ALTER_USER_PRIVILEGES 0x2
|
||||
|
||||
|
|
@ -620,7 +623,7 @@ typedef struct {
|
|||
int8_t enable;
|
||||
char user[TSDB_USER_LEN];
|
||||
char pass[TSDB_USET_PASSWORD_LEN];
|
||||
char dbname[TSDB_DB_FNAME_LEN];
|
||||
char objname[TSDB_DB_FNAME_LEN]; // db or topic
|
||||
} SAlterUserReq;
|
||||
|
||||
int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
|
||||
|
|
@ -1059,6 +1062,7 @@ typedef struct {
|
|||
int32_t vgId;
|
||||
int8_t syncState;
|
||||
int8_t syncRestore;
|
||||
int8_t syncCanRead;
|
||||
int64_t cacheUsage;
|
||||
int64_t numOfTables;
|
||||
int64_t numOfTimeSeries;
|
||||
|
|
@ -1144,6 +1148,13 @@ typedef struct {
|
|||
int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
|
||||
int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int64_t tick;
|
||||
} SMStreamTickReq;
|
||||
|
||||
int32_t tSerializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
|
||||
int32_t tDeserializeSMStreamTickMsg(void* buf, int32_t bufLen, SMStreamTickReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int32_t id;
|
||||
uint16_t port; // node sync Port
|
||||
|
|
@ -1744,6 +1755,8 @@ typedef struct {
|
|||
int64_t watermark;
|
||||
int32_t numOfTags;
|
||||
SArray* pTags; // array of SField
|
||||
// 3.0.20
|
||||
int64_t checkpointFreq; // ms
|
||||
} SCMCreateStreamReq;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -1943,6 +1956,12 @@ typedef struct {
|
|||
SHashObj* rebSubHash; // SHashObj<key, SMqRebSubscribe>
|
||||
} SMqDoRebalanceMsg;
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int64_t checkpointId;
|
||||
char streamName[TSDB_STREAM_FNAME_LEN];
|
||||
} SMStreamDoCheckpointMsg;
|
||||
|
||||
typedef struct {
|
||||
int64_t status;
|
||||
} SMVSubscribeRsp;
|
||||
|
|
|
|||
|
|
@ -172,6 +172,8 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_MND_SERVER_VERSION, "server-version", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_VND_MSG)
|
||||
|
|
@ -241,8 +243,11 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "stream-recover-finish", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "stream-task-check", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT, "stream-checkpoint", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESTORE_CHECKPOINT, "stream-restore-checkpoint", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_MON_MSG)
|
||||
|
|
@ -282,6 +287,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, "vnode-stream-recover1", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, "vnode-stream-recover2", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_CHECK_POINT_SOURCE, "vnode-stream-checkpoint-source", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
|
||||
|
||||
TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)
|
||||
|
|
|
|||
|
|
@ -58,282 +58,284 @@
|
|||
#define TK_TO 40
|
||||
#define TK_REVOKE 41
|
||||
#define TK_FROM 42
|
||||
#define TK_NK_COMMA 43
|
||||
#define TK_READ 44
|
||||
#define TK_WRITE 45
|
||||
#define TK_NK_DOT 46
|
||||
#define TK_DNODE 47
|
||||
#define TK_PORT 48
|
||||
#define TK_DNODES 49
|
||||
#define TK_NK_IPTOKEN 50
|
||||
#define TK_FORCE 51
|
||||
#define TK_LOCAL 52
|
||||
#define TK_QNODE 53
|
||||
#define TK_BNODE 54
|
||||
#define TK_SNODE 55
|
||||
#define TK_MNODE 56
|
||||
#define TK_DATABASE 57
|
||||
#define TK_USE 58
|
||||
#define TK_FLUSH 59
|
||||
#define TK_TRIM 60
|
||||
#define TK_IF 61
|
||||
#define TK_NOT 62
|
||||
#define TK_EXISTS 63
|
||||
#define TK_BUFFER 64
|
||||
#define TK_CACHEMODEL 65
|
||||
#define TK_CACHESIZE 66
|
||||
#define TK_COMP 67
|
||||
#define TK_DURATION 68
|
||||
#define TK_NK_VARIABLE 69
|
||||
#define TK_MAXROWS 70
|
||||
#define TK_MINROWS 71
|
||||
#define TK_KEEP 72
|
||||
#define TK_PAGES 73
|
||||
#define TK_PAGESIZE 74
|
||||
#define TK_TSDB_PAGESIZE 75
|
||||
#define TK_PRECISION 76
|
||||
#define TK_REPLICA 77
|
||||
#define TK_STRICT 78
|
||||
#define TK_VGROUPS 79
|
||||
#define TK_SINGLE_STABLE 80
|
||||
#define TK_RETENTIONS 81
|
||||
#define TK_SCHEMALESS 82
|
||||
#define TK_WAL_LEVEL 83
|
||||
#define TK_WAL_FSYNC_PERIOD 84
|
||||
#define TK_WAL_RETENTION_PERIOD 85
|
||||
#define TK_WAL_RETENTION_SIZE 86
|
||||
#define TK_WAL_ROLL_PERIOD 87
|
||||
#define TK_WAL_SEGMENT_SIZE 88
|
||||
#define TK_STT_TRIGGER 89
|
||||
#define TK_TABLE_PREFIX 90
|
||||
#define TK_TABLE_SUFFIX 91
|
||||
#define TK_NK_COLON 92
|
||||
#define TK_MAX_SPEED 93
|
||||
#define TK_TABLE 94
|
||||
#define TK_NK_LP 95
|
||||
#define TK_NK_RP 96
|
||||
#define TK_STABLE 97
|
||||
#define TK_ADD 98
|
||||
#define TK_COLUMN 99
|
||||
#define TK_MODIFY 100
|
||||
#define TK_RENAME 101
|
||||
#define TK_TAG 102
|
||||
#define TK_SET 103
|
||||
#define TK_NK_EQ 104
|
||||
#define TK_USING 105
|
||||
#define TK_TAGS 106
|
||||
#define TK_COMMENT 107
|
||||
#define TK_BOOL 108
|
||||
#define TK_TINYINT 109
|
||||
#define TK_SMALLINT 110
|
||||
#define TK_INT 111
|
||||
#define TK_INTEGER 112
|
||||
#define TK_BIGINT 113
|
||||
#define TK_FLOAT 114
|
||||
#define TK_DOUBLE 115
|
||||
#define TK_BINARY 116
|
||||
#define TK_TIMESTAMP 117
|
||||
#define TK_NCHAR 118
|
||||
#define TK_UNSIGNED 119
|
||||
#define TK_JSON 120
|
||||
#define TK_VARCHAR 121
|
||||
#define TK_MEDIUMBLOB 122
|
||||
#define TK_BLOB 123
|
||||
#define TK_VARBINARY 124
|
||||
#define TK_DECIMAL 125
|
||||
#define TK_MAX_DELAY 126
|
||||
#define TK_WATERMARK 127
|
||||
#define TK_ROLLUP 128
|
||||
#define TK_TTL 129
|
||||
#define TK_SMA 130
|
||||
#define TK_FIRST 131
|
||||
#define TK_LAST 132
|
||||
#define TK_SHOW 133
|
||||
#define TK_DATABASES 134
|
||||
#define TK_TABLES 135
|
||||
#define TK_STABLES 136
|
||||
#define TK_MNODES 137
|
||||
#define TK_QNODES 138
|
||||
#define TK_FUNCTIONS 139
|
||||
#define TK_INDEXES 140
|
||||
#define TK_ACCOUNTS 141
|
||||
#define TK_APPS 142
|
||||
#define TK_CONNECTIONS 143
|
||||
#define TK_LICENCES 144
|
||||
#define TK_GRANTS 145
|
||||
#define TK_QUERIES 146
|
||||
#define TK_SCORES 147
|
||||
#define TK_TOPICS 148
|
||||
#define TK_VARIABLES 149
|
||||
#define TK_CLUSTER 150
|
||||
#define TK_BNODES 151
|
||||
#define TK_SNODES 152
|
||||
#define TK_TRANSACTIONS 153
|
||||
#define TK_DISTRIBUTED 154
|
||||
#define TK_CONSUMERS 155
|
||||
#define TK_SUBSCRIPTIONS 156
|
||||
#define TK_VNODES 157
|
||||
#define TK_LIKE 158
|
||||
#define TK_TBNAME 159
|
||||
#define TK_QTAGS 160
|
||||
#define TK_AS 161
|
||||
#define TK_INDEX 162
|
||||
#define TK_FUNCTION 163
|
||||
#define TK_INTERVAL 164
|
||||
#define TK_TOPIC 165
|
||||
#define TK_WITH 166
|
||||
#define TK_META 167
|
||||
#define TK_CONSUMER 168
|
||||
#define TK_GROUP 169
|
||||
#define TK_DESC 170
|
||||
#define TK_DESCRIBE 171
|
||||
#define TK_RESET 172
|
||||
#define TK_QUERY 173
|
||||
#define TK_CACHE 174
|
||||
#define TK_EXPLAIN 175
|
||||
#define TK_ANALYZE 176
|
||||
#define TK_VERBOSE 177
|
||||
#define TK_NK_BOOL 178
|
||||
#define TK_RATIO 179
|
||||
#define TK_NK_FLOAT 180
|
||||
#define TK_OUTPUTTYPE 181
|
||||
#define TK_AGGREGATE 182
|
||||
#define TK_BUFSIZE 183
|
||||
#define TK_STREAM 184
|
||||
#define TK_INTO 185
|
||||
#define TK_TRIGGER 186
|
||||
#define TK_AT_ONCE 187
|
||||
#define TK_WINDOW_CLOSE 188
|
||||
#define TK_IGNORE 189
|
||||
#define TK_EXPIRED 190
|
||||
#define TK_FILL_HISTORY 191
|
||||
#define TK_SUBTABLE 192
|
||||
#define TK_KILL 193
|
||||
#define TK_CONNECTION 194
|
||||
#define TK_TRANSACTION 195
|
||||
#define TK_BALANCE 196
|
||||
#define TK_VGROUP 197
|
||||
#define TK_MERGE 198
|
||||
#define TK_REDISTRIBUTE 199
|
||||
#define TK_SPLIT 200
|
||||
#define TK_DELETE 201
|
||||
#define TK_INSERT 202
|
||||
#define TK_NULL 203
|
||||
#define TK_NK_QUESTION 204
|
||||
#define TK_NK_ARROW 205
|
||||
#define TK_ROWTS 206
|
||||
#define TK_QSTART 207
|
||||
#define TK_QEND 208
|
||||
#define TK_QDURATION 209
|
||||
#define TK_WSTART 210
|
||||
#define TK_WEND 211
|
||||
#define TK_WDURATION 212
|
||||
#define TK_IROWTS 213
|
||||
#define TK_CAST 214
|
||||
#define TK_NOW 215
|
||||
#define TK_TODAY 216
|
||||
#define TK_TIMEZONE 217
|
||||
#define TK_CLIENT_VERSION 218
|
||||
#define TK_SERVER_VERSION 219
|
||||
#define TK_SERVER_STATUS 220
|
||||
#define TK_CURRENT_USER 221
|
||||
#define TK_COUNT 222
|
||||
#define TK_LAST_ROW 223
|
||||
#define TK_CASE 224
|
||||
#define TK_END 225
|
||||
#define TK_WHEN 226
|
||||
#define TK_THEN 227
|
||||
#define TK_ELSE 228
|
||||
#define TK_BETWEEN 229
|
||||
#define TK_IS 230
|
||||
#define TK_NK_LT 231
|
||||
#define TK_NK_GT 232
|
||||
#define TK_NK_LE 233
|
||||
#define TK_NK_GE 234
|
||||
#define TK_NK_NE 235
|
||||
#define TK_MATCH 236
|
||||
#define TK_NMATCH 237
|
||||
#define TK_CONTAINS 238
|
||||
#define TK_IN 239
|
||||
#define TK_JOIN 240
|
||||
#define TK_INNER 241
|
||||
#define TK_SELECT 242
|
||||
#define TK_DISTINCT 243
|
||||
#define TK_WHERE 244
|
||||
#define TK_PARTITION 245
|
||||
#define TK_BY 246
|
||||
#define TK_SESSION 247
|
||||
#define TK_STATE_WINDOW 248
|
||||
#define TK_SLIDING 249
|
||||
#define TK_FILL 250
|
||||
#define TK_VALUE 251
|
||||
#define TK_NONE 252
|
||||
#define TK_PREV 253
|
||||
#define TK_LINEAR 254
|
||||
#define TK_NEXT 255
|
||||
#define TK_HAVING 256
|
||||
#define TK_RANGE 257
|
||||
#define TK_EVERY 258
|
||||
#define TK_ORDER 259
|
||||
#define TK_SLIMIT 260
|
||||
#define TK_SOFFSET 261
|
||||
#define TK_LIMIT 262
|
||||
#define TK_OFFSET 263
|
||||
#define TK_ASC 264
|
||||
#define TK_NULLS 265
|
||||
#define TK_ABORT 266
|
||||
#define TK_AFTER 267
|
||||
#define TK_ATTACH 268
|
||||
#define TK_BEFORE 269
|
||||
#define TK_BEGIN 270
|
||||
#define TK_BITAND 271
|
||||
#define TK_BITNOT 272
|
||||
#define TK_BITOR 273
|
||||
#define TK_BLOCKS 274
|
||||
#define TK_CHANGE 275
|
||||
#define TK_COMMA 276
|
||||
#define TK_COMPACT 277
|
||||
#define TK_CONCAT 278
|
||||
#define TK_CONFLICT 279
|
||||
#define TK_COPY 280
|
||||
#define TK_DEFERRED 281
|
||||
#define TK_DELIMITERS 282
|
||||
#define TK_DETACH 283
|
||||
#define TK_DIVIDE 284
|
||||
#define TK_DOT 285
|
||||
#define TK_EACH 286
|
||||
#define TK_FAIL 287
|
||||
#define TK_FILE 288
|
||||
#define TK_FOR 289
|
||||
#define TK_GLOB 290
|
||||
#define TK_ID 291
|
||||
#define TK_IMMEDIATE 292
|
||||
#define TK_IMPORT 293
|
||||
#define TK_INITIALLY 294
|
||||
#define TK_INSTEAD 295
|
||||
#define TK_ISNULL 296
|
||||
#define TK_KEY 297
|
||||
#define TK_MODULES 298
|
||||
#define TK_NK_BITNOT 299
|
||||
#define TK_NK_SEMI 300
|
||||
#define TK_NOTNULL 301
|
||||
#define TK_OF 302
|
||||
#define TK_PLUS 303
|
||||
#define TK_PRIVILEGE 304
|
||||
#define TK_RAISE 305
|
||||
#define TK_REPLACE 306
|
||||
#define TK_RESTRICT 307
|
||||
#define TK_ROW 308
|
||||
#define TK_SEMI 309
|
||||
#define TK_STAR 310
|
||||
#define TK_STATEMENT 311
|
||||
#define TK_STRING 312
|
||||
#define TK_TIMES 313
|
||||
#define TK_UPDATE 314
|
||||
#define TK_VALUES 315
|
||||
#define TK_VARIABLE 316
|
||||
#define TK_VIEW 317
|
||||
#define TK_WAL 318
|
||||
#define TK_SUBSCRIBE 43
|
||||
#define TK_NK_COMMA 44
|
||||
#define TK_READ 45
|
||||
#define TK_WRITE 46
|
||||
#define TK_NK_DOT 47
|
||||
#define TK_DNODE 48
|
||||
#define TK_PORT 49
|
||||
#define TK_DNODES 50
|
||||
#define TK_NK_IPTOKEN 51
|
||||
#define TK_FORCE 52
|
||||
#define TK_LOCAL 53
|
||||
#define TK_QNODE 54
|
||||
#define TK_BNODE 55
|
||||
#define TK_SNODE 56
|
||||
#define TK_MNODE 57
|
||||
#define TK_DATABASE 58
|
||||
#define TK_USE 59
|
||||
#define TK_FLUSH 60
|
||||
#define TK_TRIM 61
|
||||
#define TK_IF 62
|
||||
#define TK_NOT 63
|
||||
#define TK_EXISTS 64
|
||||
#define TK_BUFFER 65
|
||||
#define TK_CACHEMODEL 66
|
||||
#define TK_CACHESIZE 67
|
||||
#define TK_COMP 68
|
||||
#define TK_DURATION 69
|
||||
#define TK_NK_VARIABLE 70
|
||||
#define TK_MAXROWS 71
|
||||
#define TK_MINROWS 72
|
||||
#define TK_KEEP 73
|
||||
#define TK_PAGES 74
|
||||
#define TK_PAGESIZE 75
|
||||
#define TK_TSDB_PAGESIZE 76
|
||||
#define TK_PRECISION 77
|
||||
#define TK_REPLICA 78
|
||||
#define TK_STRICT 79
|
||||
#define TK_VGROUPS 80
|
||||
#define TK_SINGLE_STABLE 81
|
||||
#define TK_RETENTIONS 82
|
||||
#define TK_SCHEMALESS 83
|
||||
#define TK_WAL_LEVEL 84
|
||||
#define TK_WAL_FSYNC_PERIOD 85
|
||||
#define TK_WAL_RETENTION_PERIOD 86
|
||||
#define TK_WAL_RETENTION_SIZE 87
|
||||
#define TK_WAL_ROLL_PERIOD 88
|
||||
#define TK_WAL_SEGMENT_SIZE 89
|
||||
#define TK_STT_TRIGGER 90
|
||||
#define TK_TABLE_PREFIX 91
|
||||
#define TK_TABLE_SUFFIX 92
|
||||
#define TK_NK_COLON 93
|
||||
#define TK_MAX_SPEED 94
|
||||
#define TK_TABLE 95
|
||||
#define TK_NK_LP 96
|
||||
#define TK_NK_RP 97
|
||||
#define TK_STABLE 98
|
||||
#define TK_ADD 99
|
||||
#define TK_COLUMN 100
|
||||
#define TK_MODIFY 101
|
||||
#define TK_RENAME 102
|
||||
#define TK_TAG 103
|
||||
#define TK_SET 104
|
||||
#define TK_NK_EQ 105
|
||||
#define TK_USING 106
|
||||
#define TK_TAGS 107
|
||||
#define TK_COMMENT 108
|
||||
#define TK_BOOL 109
|
||||
#define TK_TINYINT 110
|
||||
#define TK_SMALLINT 111
|
||||
#define TK_INT 112
|
||||
#define TK_INTEGER 113
|
||||
#define TK_BIGINT 114
|
||||
#define TK_FLOAT 115
|
||||
#define TK_DOUBLE 116
|
||||
#define TK_BINARY 117
|
||||
#define TK_TIMESTAMP 118
|
||||
#define TK_NCHAR 119
|
||||
#define TK_UNSIGNED 120
|
||||
#define TK_JSON 121
|
||||
#define TK_VARCHAR 122
|
||||
#define TK_MEDIUMBLOB 123
|
||||
#define TK_BLOB 124
|
||||
#define TK_VARBINARY 125
|
||||
#define TK_DECIMAL 126
|
||||
#define TK_MAX_DELAY 127
|
||||
#define TK_WATERMARK 128
|
||||
#define TK_ROLLUP 129
|
||||
#define TK_TTL 130
|
||||
#define TK_SMA 131
|
||||
#define TK_FIRST 132
|
||||
#define TK_LAST 133
|
||||
#define TK_SHOW 134
|
||||
#define TK_PRIVILEGES 135
|
||||
#define TK_DATABASES 136
|
||||
#define TK_TABLES 137
|
||||
#define TK_STABLES 138
|
||||
#define TK_MNODES 139
|
||||
#define TK_QNODES 140
|
||||
#define TK_FUNCTIONS 141
|
||||
#define TK_INDEXES 142
|
||||
#define TK_ACCOUNTS 143
|
||||
#define TK_APPS 144
|
||||
#define TK_CONNECTIONS 145
|
||||
#define TK_LICENCES 146
|
||||
#define TK_GRANTS 147
|
||||
#define TK_QUERIES 148
|
||||
#define TK_SCORES 149
|
||||
#define TK_TOPICS 150
|
||||
#define TK_VARIABLES 151
|
||||
#define TK_CLUSTER 152
|
||||
#define TK_BNODES 153
|
||||
#define TK_SNODES 154
|
||||
#define TK_TRANSACTIONS 155
|
||||
#define TK_DISTRIBUTED 156
|
||||
#define TK_CONSUMERS 157
|
||||
#define TK_SUBSCRIPTIONS 158
|
||||
#define TK_VNODES 159
|
||||
#define TK_LIKE 160
|
||||
#define TK_TBNAME 161
|
||||
#define TK_QTAGS 162
|
||||
#define TK_AS 163
|
||||
#define TK_INDEX 164
|
||||
#define TK_FUNCTION 165
|
||||
#define TK_INTERVAL 166
|
||||
#define TK_TOPIC 167
|
||||
#define TK_WITH 168
|
||||
#define TK_META 169
|
||||
#define TK_CONSUMER 170
|
||||
#define TK_GROUP 171
|
||||
#define TK_DESC 172
|
||||
#define TK_DESCRIBE 173
|
||||
#define TK_RESET 174
|
||||
#define TK_QUERY 175
|
||||
#define TK_CACHE 176
|
||||
#define TK_EXPLAIN 177
|
||||
#define TK_ANALYZE 178
|
||||
#define TK_VERBOSE 179
|
||||
#define TK_NK_BOOL 180
|
||||
#define TK_RATIO 181
|
||||
#define TK_NK_FLOAT 182
|
||||
#define TK_OUTPUTTYPE 183
|
||||
#define TK_AGGREGATE 184
|
||||
#define TK_BUFSIZE 185
|
||||
#define TK_STREAM 186
|
||||
#define TK_INTO 187
|
||||
#define TK_TRIGGER 188
|
||||
#define TK_AT_ONCE 189
|
||||
#define TK_WINDOW_CLOSE 190
|
||||
#define TK_IGNORE 191
|
||||
#define TK_EXPIRED 192
|
||||
#define TK_FILL_HISTORY 193
|
||||
#define TK_SUBTABLE 194
|
||||
#define TK_KILL 195
|
||||
#define TK_CONNECTION 196
|
||||
#define TK_TRANSACTION 197
|
||||
#define TK_BALANCE 198
|
||||
#define TK_VGROUP 199
|
||||
#define TK_MERGE 200
|
||||
#define TK_REDISTRIBUTE 201
|
||||
#define TK_SPLIT 202
|
||||
#define TK_DELETE 203
|
||||
#define TK_INSERT 204
|
||||
#define TK_NULL 205
|
||||
#define TK_NK_QUESTION 206
|
||||
#define TK_NK_ARROW 207
|
||||
#define TK_ROWTS 208
|
||||
#define TK_QSTART 209
|
||||
#define TK_QEND 210
|
||||
#define TK_QDURATION 211
|
||||
#define TK_WSTART 212
|
||||
#define TK_WEND 213
|
||||
#define TK_WDURATION 214
|
||||
#define TK_IROWTS 215
|
||||
#define TK_CAST 216
|
||||
#define TK_NOW 217
|
||||
#define TK_TODAY 218
|
||||
#define TK_TIMEZONE 219
|
||||
#define TK_CLIENT_VERSION 220
|
||||
#define TK_SERVER_VERSION 221
|
||||
#define TK_SERVER_STATUS 222
|
||||
#define TK_CURRENT_USER 223
|
||||
#define TK_COUNT 224
|
||||
#define TK_LAST_ROW 225
|
||||
#define TK_CASE 226
|
||||
#define TK_END 227
|
||||
#define TK_WHEN 228
|
||||
#define TK_THEN 229
|
||||
#define TK_ELSE 230
|
||||
#define TK_BETWEEN 231
|
||||
#define TK_IS 232
|
||||
#define TK_NK_LT 233
|
||||
#define TK_NK_GT 234
|
||||
#define TK_NK_LE 235
|
||||
#define TK_NK_GE 236
|
||||
#define TK_NK_NE 237
|
||||
#define TK_MATCH 238
|
||||
#define TK_NMATCH 239
|
||||
#define TK_CONTAINS 240
|
||||
#define TK_IN 241
|
||||
#define TK_JOIN 242
|
||||
#define TK_INNER 243
|
||||
#define TK_SELECT 244
|
||||
#define TK_DISTINCT 245
|
||||
#define TK_WHERE 246
|
||||
#define TK_PARTITION 247
|
||||
#define TK_BY 248
|
||||
#define TK_SESSION 249
|
||||
#define TK_STATE_WINDOW 250
|
||||
#define TK_SLIDING 251
|
||||
#define TK_FILL 252
|
||||
#define TK_VALUE 253
|
||||
#define TK_NONE 254
|
||||
#define TK_PREV 255
|
||||
#define TK_LINEAR 256
|
||||
#define TK_NEXT 257
|
||||
#define TK_HAVING 258
|
||||
#define TK_RANGE 259
|
||||
#define TK_EVERY 260
|
||||
#define TK_ORDER 261
|
||||
#define TK_SLIMIT 262
|
||||
#define TK_SOFFSET 263
|
||||
#define TK_LIMIT 264
|
||||
#define TK_OFFSET 265
|
||||
#define TK_ASC 266
|
||||
#define TK_NULLS 267
|
||||
#define TK_ABORT 268
|
||||
#define TK_AFTER 269
|
||||
#define TK_ATTACH 270
|
||||
#define TK_BEFORE 271
|
||||
#define TK_BEGIN 272
|
||||
#define TK_BITAND 273
|
||||
#define TK_BITNOT 274
|
||||
#define TK_BITOR 275
|
||||
#define TK_BLOCKS 276
|
||||
#define TK_CHANGE 277
|
||||
#define TK_COMMA 278
|
||||
#define TK_COMPACT 279
|
||||
#define TK_CONCAT 280
|
||||
#define TK_CONFLICT 281
|
||||
#define TK_COPY 282
|
||||
#define TK_DEFERRED 283
|
||||
#define TK_DELIMITERS 284
|
||||
#define TK_DETACH 285
|
||||
#define TK_DIVIDE 286
|
||||
#define TK_DOT 287
|
||||
#define TK_EACH 288
|
||||
#define TK_FAIL 289
|
||||
#define TK_FILE 290
|
||||
#define TK_FOR 291
|
||||
#define TK_GLOB 292
|
||||
#define TK_ID 293
|
||||
#define TK_IMMEDIATE 294
|
||||
#define TK_IMPORT 295
|
||||
#define TK_INITIALLY 296
|
||||
#define TK_INSTEAD 297
|
||||
#define TK_ISNULL 298
|
||||
#define TK_KEY 299
|
||||
#define TK_MODULES 300
|
||||
#define TK_NK_BITNOT 301
|
||||
#define TK_NK_SEMI 302
|
||||
#define TK_NOTNULL 303
|
||||
#define TK_OF 304
|
||||
#define TK_PLUS 305
|
||||
#define TK_PRIVILEGE 306
|
||||
#define TK_RAISE 307
|
||||
#define TK_REPLACE 308
|
||||
#define TK_RESTRICT 309
|
||||
#define TK_ROW 310
|
||||
#define TK_SEMI 311
|
||||
#define TK_STAR 312
|
||||
#define TK_STATEMENT 313
|
||||
#define TK_STRING 314
|
||||
#define TK_TIMES 315
|
||||
#define TK_UPDATE 316
|
||||
#define TK_VALUES 317
|
||||
#define TK_VARIABLE 318
|
||||
#define TK_VIEW 319
|
||||
#define TK_WAL 320
|
||||
|
||||
#define TK_NK_SPACE 600
|
||||
#define TK_NK_COMMENT 601
|
||||
|
|
|
|||
|
|
@ -152,7 +152,7 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo);
|
|||
* @param tinfo qhandle
|
||||
* @return
|
||||
*/
|
||||
int32_t qAsyncKillTask(qTaskInfo_t tinfo);
|
||||
int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode);
|
||||
|
||||
/**
|
||||
* destroy query info structure
|
||||
|
|
@ -160,13 +160,6 @@ int32_t qAsyncKillTask(qTaskInfo_t tinfo);
|
|||
*/
|
||||
void qDestroyTask(qTaskInfo_t tinfo);
|
||||
|
||||
/**
|
||||
* Get the queried table uid
|
||||
* @param qHandle
|
||||
* @return
|
||||
*/
|
||||
int64_t qGetQueriedTableUid(qTaskInfo_t tinfo);
|
||||
|
||||
/**
|
||||
* Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ typedef struct SFuncExecFuncs {
|
|||
#define MAX_INTERVAL_TIME_WINDOW 10000000 // maximum allowed time windows in final results
|
||||
|
||||
#define TOP_BOTTOM_QUERY_LIMIT 100
|
||||
#define FUNCTIONS_NAME_MAX_LENGTH 16
|
||||
#define FUNCTIONS_NAME_MAX_LENGTH 32
|
||||
|
||||
typedef struct SResultRowEntryInfo {
|
||||
bool initialized : 1; // output buffer has been initialized
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ extern "C" {
|
|||
#define PRIVILEGE_TYPE_ALL PRIVILEGE_TYPE_MASK(0)
|
||||
#define PRIVILEGE_TYPE_READ PRIVILEGE_TYPE_MASK(1)
|
||||
#define PRIVILEGE_TYPE_WRITE PRIVILEGE_TYPE_MASK(2)
|
||||
#define PRIVILEGE_TYPE_SUBSCRIBE PRIVILEGE_TYPE_MASK(3)
|
||||
|
||||
#define PRIVILEGE_TYPE_TEST_MASK(val, mask) (((val) & (mask)) != 0)
|
||||
|
||||
|
|
@ -423,7 +424,7 @@ typedef struct SDropFunctionStmt {
|
|||
typedef struct SGrantStmt {
|
||||
ENodeType type;
|
||||
char userName[TSDB_USER_LEN];
|
||||
char dbName[TSDB_DB_NAME_LEN];
|
||||
char objName[TSDB_DB_NAME_LEN]; // db or topic
|
||||
int64_t privileges;
|
||||
} SGrantStmt;
|
||||
|
||||
|
|
|
|||
|
|
@ -187,6 +187,7 @@ typedef enum ENodeType {
|
|||
QUERY_NODE_SHOW_TRANSACTIONS_STMT,
|
||||
QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT,
|
||||
QUERY_NODE_SHOW_VNODES_STMT,
|
||||
QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
|
||||
QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
|
||||
QUERY_NODE_SHOW_CREATE_TABLE_STMT,
|
||||
QUERY_NODE_SHOW_CREATE_STABLE_STMT,
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
|
|||
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
|
||||
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
|
||||
|
||||
#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR)
|
||||
#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED)
|
||||
#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR)
|
||||
#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later
|
||||
|
||||
|
|
@ -268,7 +268,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
|
|||
((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
|
||||
(_code) == TSDB_CODE_NODE_NOT_DEPLOYED || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
|
||||
SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
|
||||
(_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK)
|
||||
(_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK || \
|
||||
(_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
|
||||
|
||||
#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
|
||||
((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
|
||||
|
|
|
|||
|
|
@ -275,31 +275,6 @@ typedef struct {
|
|||
SEpSet epSet;
|
||||
} SStreamChildEpInfo;
|
||||
|
||||
typedef struct {
|
||||
int32_t srcNodeId;
|
||||
int32_t srcChildId;
|
||||
int64_t stateSaveVer;
|
||||
int64_t stateProcessedVer;
|
||||
} SStreamCheckpointInfo;
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int64_t checkTs;
|
||||
int32_t checkpointId; // incremental
|
||||
int32_t taskId;
|
||||
SArray* checkpointVer; // SArray<SStreamCheckpointInfo>
|
||||
} SStreamMultiVgCheckpointInfo;
|
||||
|
||||
typedef struct {
|
||||
int32_t taskId;
|
||||
int32_t checkpointId; // incremental
|
||||
} SStreamCheckpointKey;
|
||||
|
||||
typedef struct {
|
||||
int32_t taskId;
|
||||
SArray* checkpointVer;
|
||||
} SStreamRecoveringState;
|
||||
|
||||
typedef struct SStreamTask {
|
||||
int64_t streamId;
|
||||
int32_t taskId;
|
||||
|
|
@ -364,6 +339,10 @@ typedef struct SStreamTask {
|
|||
int64_t checkReqId;
|
||||
SArray* checkReqIds; // shuffle
|
||||
int32_t refCnt;
|
||||
|
||||
int64_t checkpointingId;
|
||||
int32_t checkpointAlignCnt;
|
||||
|
||||
} SStreamTask;
|
||||
|
||||
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
|
||||
|
|
@ -509,6 +488,60 @@ typedef struct {
|
|||
int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq);
|
||||
int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int64_t checkpointId;
|
||||
int32_t taskId;
|
||||
int32_t nodeId;
|
||||
int64_t expireTime;
|
||||
} SStreamCheckpointSourceReq;
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int64_t checkpointId;
|
||||
int32_t taskId;
|
||||
int32_t nodeId;
|
||||
int64_t expireTime;
|
||||
} SStreamCheckpointSourceRsp;
|
||||
|
||||
int32_t tEncodeSStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq);
|
||||
int32_t tDecodeSStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSourceReq* pReq);
|
||||
|
||||
int32_t tEncodeSStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckpointSourceRsp* pRsp);
|
||||
int32_t tDecodeSStreamCheckpointSourceRsp(SDecoder* pDecoder, SStreamCheckpointSourceRsp* pRsp);
|
||||
|
||||
typedef struct {
|
||||
SMsgHead msgHead;
|
||||
int64_t streamId;
|
||||
int64_t checkpointId;
|
||||
int32_t downstreamTaskId;
|
||||
int32_t downstreamNodeId;
|
||||
int32_t upstreamTaskId;
|
||||
int32_t upstreamNodeId;
|
||||
int32_t childId;
|
||||
int64_t expireTime;
|
||||
int8_t taskLevel;
|
||||
} SStreamCheckpointReq;
|
||||
|
||||
typedef struct {
|
||||
SMsgHead msgHead;
|
||||
int64_t streamId;
|
||||
int64_t checkpointId;
|
||||
int32_t downstreamTaskId;
|
||||
int32_t downstreamNodeId;
|
||||
int32_t upstreamTaskId;
|
||||
int32_t upstreamNodeId;
|
||||
int32_t childId;
|
||||
int64_t expireTime;
|
||||
int8_t taskLevel;
|
||||
} SStreamCheckpointRsp;
|
||||
|
||||
int32_t tEncodeSStreamCheckpointReq(SEncoder* pEncoder, const SStreamCheckpointReq* pReq);
|
||||
int32_t tDecodeSStreamCheckpointReq(SDecoder* pDecoder, SStreamCheckpointReq* pReq);
|
||||
|
||||
int32_t tEncodeSStreamCheckpointRsp(SEncoder* pEncoder, const SStreamCheckpointRsp* pRsp);
|
||||
int32_t tDecodeSStreamCheckpointRsp(SDecoder* pDecoder, SStreamCheckpointRsp* pRsp);
|
||||
|
||||
typedef struct {
|
||||
int64_t streamId;
|
||||
int32_t downstreamTaskId;
|
||||
|
|
@ -598,18 +631,22 @@ void streamMetaClose(SStreamMeta* streamMeta);
|
|||
|
||||
int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
|
||||
int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen);
|
||||
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
|
||||
SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId);
|
||||
|
||||
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
|
||||
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
|
||||
void streamMetaRemoveTask1(SStreamMeta* pMeta, int32_t taskId);
|
||||
void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
|
||||
|
||||
int32_t streamMetaBegin(SStreamMeta* pMeta);
|
||||
int32_t streamMetaCommit(SStreamMeta* pMeta);
|
||||
int32_t streamMetaRollBack(SStreamMeta* pMeta);
|
||||
int32_t streamLoadTasks(SStreamMeta* pMeta);
|
||||
|
||||
// checkpoint
|
||||
int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
|
||||
int32_t streamProcessCheckpointReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointReq* pReq);
|
||||
int32_t streamProcessCheckpointRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointRsp* pRsp);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ extern "C" {
|
|||
#define SNAPSHOT_MAX_CLOCK_SKEW_MS 1000 * 10
|
||||
#define SNAPSHOT_WAIT_MS 1000 * 30
|
||||
|
||||
#define SYNC_MAX_RETRY_BACKOFF 5
|
||||
#define SYNC_LOG_REPL_RETRY_WAIT_MS 100
|
||||
#define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
|
||||
#define SYNC_HEART_TIMEOUT_MS 1000 * 8
|
||||
|
||||
|
|
@ -49,7 +51,7 @@ extern "C" {
|
|||
#define SYNC_MAX_BATCH_SIZE 1
|
||||
#define SYNC_INDEX_BEGIN 0
|
||||
#define SYNC_INDEX_INVALID -1
|
||||
#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF
|
||||
#define SYNC_TERM_INVALID -1 // 0xFFFFFFFFFFFFFFFF
|
||||
|
||||
typedef enum {
|
||||
SYNC_STRATEGY_NO_SNAPSHOT = 0,
|
||||
|
|
@ -60,13 +62,14 @@ typedef enum {
|
|||
typedef uint64_t SyncNodeId;
|
||||
typedef int32_t SyncGroupId;
|
||||
typedef int64_t SyncIndex;
|
||||
typedef uint64_t SyncTerm;
|
||||
typedef int64_t SyncTerm;
|
||||
|
||||
typedef struct SSyncNode SSyncNode;
|
||||
typedef struct SWal SWal;
|
||||
typedef struct SSyncRaftEntry SSyncRaftEntry;
|
||||
|
||||
typedef enum {
|
||||
TAOS_SYNC_STATE_OFFLINE = 0,
|
||||
TAOS_SYNC_STATE_FOLLOWER = 100,
|
||||
TAOS_SYNC_STATE_CANDIDATE = 101,
|
||||
TAOS_SYNC_STATE_LEADER = 102,
|
||||
|
|
@ -135,13 +138,13 @@ typedef struct SSnapshotMeta {
|
|||
typedef struct SSyncFSM {
|
||||
void* data;
|
||||
|
||||
void (*FpCommitCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
|
||||
void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
|
||||
void (*FpRollBackCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
|
||||
void (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
|
||||
void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
|
||||
void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
|
||||
|
||||
void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm);
|
||||
void (*FpReConfigCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SReConfigCbMeta* pMeta);
|
||||
void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
|
||||
void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SReConfigCbMeta* pMeta);
|
||||
void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta);
|
||||
bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm);
|
||||
int32_t (*FpApplyQueueItems)(const struct SSyncFSM* pFsm);
|
||||
|
||||
|
|
@ -210,15 +213,20 @@ typedef struct SSyncInfo {
|
|||
int32_t (*syncEqCtrlMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg);
|
||||
} SSyncInfo;
|
||||
|
||||
// if state == leader
|
||||
// if restored, display "leader"
|
||||
// if !restored && canRead, display "leader*"
|
||||
// if !restored && !canRead, display "leader**"
|
||||
typedef struct SSyncState {
|
||||
ESyncState state;
|
||||
bool restored;
|
||||
bool canRead;
|
||||
} SSyncState;
|
||||
|
||||
int32_t syncInit();
|
||||
void syncCleanUp();
|
||||
int64_t syncOpen(SSyncInfo* pSyncInfo);
|
||||
void syncStart(int64_t rid);
|
||||
int32_t syncStart(int64_t rid);
|
||||
void syncStop(int64_t rid);
|
||||
void syncPreStop(int64_t rid);
|
||||
int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak);
|
||||
|
|
|
|||
|
|
@ -170,7 +170,7 @@ int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo
|
|||
|
||||
// Assign version automatically and return to caller,
|
||||
// -1 will be returned for failed writes
|
||||
int64_t walAppendLog(SWal *, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
|
||||
int64_t walAppendLog(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
|
||||
|
||||
void walFsync(SWal *, bool force);
|
||||
|
||||
|
|
|
|||
|
|
@ -169,6 +169,9 @@ void taosSetMaskSIGPIPE();
|
|||
uint32_t taosInetAddr(const char *ipAddr);
|
||||
const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len);
|
||||
|
||||
uint64_t taosHton64(uint64_t val);
|
||||
uint64_t taosNtoh64(uint64_t val);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ extern "C" {
|
|||
#ifdef WINDOWS
|
||||
|
||||
#define CLOCK_REALTIME 0
|
||||
#define CLOCK_MONOTONIC 0
|
||||
|
||||
#define MILLISECOND_PER_SECOND (1000i64)
|
||||
#else
|
||||
|
|
@ -82,6 +83,13 @@ static FORCE_INLINE int64_t taosGetTimestampNs() {
|
|||
return (int64_t)systemTime.tv_sec * 1000000000LL + (int64_t)systemTime.tv_nsec;
|
||||
}
|
||||
|
||||
//@return timestamp of monotonic clock in millisecond
|
||||
static FORCE_INLINE int64_t taosGetMonoTimestampMs() {
|
||||
struct timespec systemTime = {0};
|
||||
taosClockGetTime(CLOCK_MONOTONIC, &systemTime);
|
||||
return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_nsec / 1000000;
|
||||
}
|
||||
|
||||
char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm);
|
||||
struct tm *taosLocalTime(const time_t *timep, struct tm *result);
|
||||
struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst);
|
||||
|
|
|
|||
|
|
@ -40,21 +40,37 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error
|
||||
|
||||
// rpc
|
||||
// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x
|
||||
// #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x
|
||||
#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003)
|
||||
#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004)
|
||||
// #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x
|
||||
// #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x
|
||||
// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x
|
||||
// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) //2.x
|
||||
// #define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) //2.x
|
||||
// #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) //2.x
|
||||
#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B)
|
||||
// #define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) //2.x
|
||||
// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) //2.x
|
||||
// #define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) //2.x
|
||||
// #define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) //2.x
|
||||
// #define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) //2.x
|
||||
// #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //2.x
|
||||
// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //2.x
|
||||
#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013)
|
||||
#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014)
|
||||
#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015)
|
||||
// #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //2.x
|
||||
#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017)
|
||||
#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018)
|
||||
#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019)
|
||||
|
||||
//common & util
|
||||
#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013)
|
||||
#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014)
|
||||
|
||||
#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100)
|
||||
#define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101)
|
||||
#define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0102)
|
||||
// #define TSDB_CODE_COM_INVALID_CFG_MSG TAOS_DEF_ERROR_CODE(0, 0x0103) // 2.x
|
||||
#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104)
|
||||
#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105)
|
||||
#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106)
|
||||
|
|
@ -69,7 +85,7 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113)
|
||||
#define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114)
|
||||
#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115)
|
||||
#define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0116)
|
||||
#define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0116) //
|
||||
#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117)
|
||||
#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118)
|
||||
#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119)
|
||||
|
|
@ -81,7 +97,7 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F)
|
||||
|
||||
#define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0120)
|
||||
#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121)
|
||||
#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) //
|
||||
#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0122)
|
||||
#define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0123)
|
||||
#define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0124)
|
||||
|
|
@ -94,6 +110,9 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012B)
|
||||
#define TSDB_CODE_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x012C)
|
||||
|
||||
#define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) //
|
||||
#define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) //
|
||||
|
||||
//client
|
||||
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200)
|
||||
#define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201)
|
||||
|
|
@ -324,6 +343,7 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_VND_COL_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0526)
|
||||
#define TSDB_CODE_VND_COL_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x0527)
|
||||
#define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528)
|
||||
#define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529)
|
||||
|
||||
// tsdb
|
||||
#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600)
|
||||
|
|
|
|||
|
|
@ -220,15 +220,10 @@ void taosArrayClear(SArray* pArray);
|
|||
*/
|
||||
void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
|
||||
|
||||
/**
|
||||
* clear the array (remove all element)
|
||||
* @param pArray
|
||||
* @param fp
|
||||
*/
|
||||
void taosArrayClearP(SArray* pArray, FDelete fp);
|
||||
|
||||
void* taosArrayDestroy(SArray* pArray);
|
||||
|
||||
void taosArrayDestroyP(SArray* pArray, FDelete fp);
|
||||
|
||||
void taosArrayDestroyEx(SArray* pArray, FDelete fp);
|
||||
|
||||
/**
|
||||
|
|
@ -238,12 +233,6 @@ void taosArrayDestroyEx(SArray* pArray, FDelete fp);
|
|||
*/
|
||||
void taosArraySort(SArray* pArray, __compar_fn_t comparFn);
|
||||
|
||||
/**
|
||||
* sort string array
|
||||
* @param pArray
|
||||
*/
|
||||
void taosArraySortString(SArray* pArray, __compar_fn_t comparFn);
|
||||
|
||||
/**
|
||||
* search the array
|
||||
* @param pArray
|
||||
|
|
|
|||
|
|
@ -281,6 +281,7 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_DNODE_ROLE_VNODE 2
|
||||
|
||||
#define TSDB_MAX_REPLICA 5
|
||||
#define TSDB_SYNC_LOG_BUFFER_SIZE 4096
|
||||
|
||||
#define TSDB_TBNAME_COLUMN_INDEX (-1)
|
||||
#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta
|
||||
|
|
|
|||
|
|
@ -1645,7 +1645,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) {
|
|||
static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
|
||||
char* p = (char*)pResultInfo->pData;
|
||||
|
||||
// version + length + numOfRows + numOfCol + groupId + flag_segment + column_info
|
||||
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length |
|
||||
int32_t len = getVersion1BlockMetaSize(p, numOfCols);
|
||||
int32_t* colLength = (int32_t*)(p + len);
|
||||
len += sizeof(int32_t) * numOfCols;
|
||||
|
|
|
|||
|
|
@ -1211,6 +1211,208 @@ static void destroyVgHash(void* data) {
|
|||
taosMemoryFreeClear(vgData->data);
|
||||
}
|
||||
|
||||
int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields){
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
STableMeta* pTableMeta = NULL;
|
||||
SQuery* pQuery = NULL;
|
||||
SSubmitReq* subReq = NULL;
|
||||
|
||||
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
|
||||
if (!pRequest) {
|
||||
uError("WriteRaw:createRequest error request is null");
|
||||
code = terrno;
|
||||
goto end;
|
||||
}
|
||||
|
||||
pRequest->syncQuery = true;
|
||||
if (!pRequest->pDb) {
|
||||
uError("WriteRaw:not use db");
|
||||
code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
|
||||
goto end;
|
||||
}
|
||||
|
||||
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
|
||||
tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname));
|
||||
tstrncpy(pName.tname, tbname, sizeof(pName.tname));
|
||||
|
||||
struct SCatalog* pCatalog = NULL;
|
||||
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("WriteRaw: get gatlog error");
|
||||
goto end;
|
||||
}
|
||||
|
||||
SRequestConnInfo conn = {0};
|
||||
conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter;
|
||||
conn.requestId = pRequest->requestId;
|
||||
conn.requestObjRefId = pRequest->self;
|
||||
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
|
||||
|
||||
SVgroupInfo vgData = {0};
|
||||
code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vgData);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbname);
|
||||
goto end;
|
||||
}
|
||||
|
||||
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname);
|
||||
goto end;
|
||||
}
|
||||
uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid);
|
||||
uint64_t uid = pTableMeta->uid;
|
||||
int32_t numOfCols = pTableMeta->tableInfo.numOfColumns;
|
||||
|
||||
uint16_t fLen = 0;
|
||||
int32_t rowSize = 0;
|
||||
int16_t nVar = 0;
|
||||
for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
|
||||
SSchema* schema = pTableMeta->schema + i;
|
||||
fLen += TYPE_BYTES[schema->type];
|
||||
rowSize += schema->bytes;
|
||||
if (IS_VAR_DATA_TYPE(schema->type)) {
|
||||
nVar++;
|
||||
}
|
||||
}
|
||||
|
||||
fLen -= sizeof(TSKEY);
|
||||
|
||||
int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) +
|
||||
(int32_t)TD_BITMAP_BYTES(numOfCols - 1);
|
||||
int32_t schemaLen = 0;
|
||||
int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize;
|
||||
|
||||
int32_t totalLen = sizeof(SSubmitReq) + submitLen;
|
||||
subReq = taosMemoryCalloc(1, totalLen);
|
||||
SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq));
|
||||
void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk));
|
||||
STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen);
|
||||
|
||||
SRowBuilder rb = {0};
|
||||
tdSRowInit(&rb, pTableMeta->sversion);
|
||||
tdSRowSetTpInfo(&rb, numOfCols, fLen);
|
||||
int32_t dataLen = 0;
|
||||
|
||||
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length |
|
||||
char* pStart = pData + getVersion1BlockMetaSize(pData, numFields);
|
||||
int32_t* colLength = (int32_t*)pStart;
|
||||
pStart += sizeof(int32_t) * numFields;
|
||||
|
||||
SResultColumn* pCol = taosMemoryCalloc(numFields, sizeof(SResultColumn));
|
||||
|
||||
for (int32_t i = 0; i < numFields; ++i) {
|
||||
if (IS_VAR_DATA_TYPE(fields[i].type)) {
|
||||
pCol[i].offset = (int32_t*)pStart;
|
||||
pStart += rows * sizeof(int32_t);
|
||||
} else {
|
||||
pCol[i].nullbitmap = pStart;
|
||||
pStart += BitmapLen(rows);
|
||||
}
|
||||
|
||||
pCol[i].pData = pStart;
|
||||
pStart += colLength[i];
|
||||
}
|
||||
|
||||
SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
|
||||
for (int i = 0; i < numFields; i++) {
|
||||
TAOS_FIELD* schema = &fields[i];
|
||||
taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t));
|
||||
}
|
||||
|
||||
for (int32_t j = 0; j < rows; j++) {
|
||||
tdSRowResetBuf(&rb, rowData);
|
||||
int32_t offset = 0;
|
||||
for (int32_t k = 0; k < numOfCols; k++) {
|
||||
const SSchema* pColumn = &pTableMeta->schema[k];
|
||||
int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name));
|
||||
if (!index) { // add none
|
||||
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k);
|
||||
}else{
|
||||
if (IS_VAR_DATA_TYPE(pColumn->type)) {
|
||||
if (pCol[*index].offset[j] != -1) {
|
||||
char* data = pCol[*index].pData + pCol[*index].offset[j];
|
||||
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k);
|
||||
} else {
|
||||
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k);
|
||||
}
|
||||
} else {
|
||||
if (!colDataIsNull_f(pCol[*index].nullbitmap, j)) {
|
||||
char* data = pCol[*index].pData + pColumn->bytes * j;
|
||||
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k);
|
||||
} else {
|
||||
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
|
||||
offset += TYPE_BYTES[pColumn->type];
|
||||
}
|
||||
}
|
||||
tdSRowEnd(&rb);
|
||||
int32_t rowLen = TD_ROW_LEN(rowData);
|
||||
rowData = POINTER_SHIFT(rowData, rowLen);
|
||||
dataLen += rowLen;
|
||||
}
|
||||
|
||||
taosHashCleanup(schemaHash);
|
||||
taosMemoryFree(pCol);
|
||||
|
||||
blk->uid = htobe64(uid);
|
||||
blk->suid = htobe64(suid);
|
||||
blk->sversion = htonl(pTableMeta->sversion);
|
||||
blk->schemaLen = htonl(schemaLen);
|
||||
blk->numOfRows = htonl(rows);
|
||||
blk->dataLen = htonl(dataLen);
|
||||
subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen;
|
||||
subReq->numOfBlocks = 1;
|
||||
|
||||
pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
|
||||
if (NULL == pQuery) {
|
||||
uError("create SQuery error");
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto end;
|
||||
}
|
||||
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
|
||||
pQuery->haveResultSet = false;
|
||||
pQuery->msgType = TDMT_VND_SUBMIT;
|
||||
pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT);
|
||||
if (NULL == pQuery->pRoot) {
|
||||
uError("create pQuery->pRoot error");
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto end;
|
||||
}
|
||||
SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot);
|
||||
nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES);
|
||||
|
||||
SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
|
||||
if (NULL == dst) {
|
||||
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
|
||||
goto end;
|
||||
}
|
||||
dst->vg = vgData;
|
||||
dst->numOfTables = subReq->numOfBlocks;
|
||||
dst->size = subReq->length;
|
||||
dst->pData = (char*)subReq;
|
||||
subReq->header.vgId = htonl(dst->vg.vgId);
|
||||
subReq->version = htonl(1);
|
||||
subReq->header.contLen = htonl(subReq->length);
|
||||
subReq->length = htonl(subReq->length);
|
||||
subReq->numOfBlocks = htonl(subReq->numOfBlocks);
|
||||
subReq = NULL; // no need free
|
||||
taosArrayPush(nodeStmt->pDataBlocks, &dst);
|
||||
|
||||
launchQueryImpl(pRequest, pQuery, true, NULL);
|
||||
code = pRequest->code;
|
||||
|
||||
end:
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
qDestroyQuery(pQuery);
|
||||
taosMemoryFree(subReq);
|
||||
return code;
|
||||
}
|
||||
|
||||
int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
STableMeta* pTableMeta = NULL;
|
||||
|
|
@ -1293,6 +1495,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
|
|||
tdSRowSetTpInfo(&rb, numOfCols, fLen);
|
||||
int32_t dataLen = 0;
|
||||
|
||||
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length |
|
||||
char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols);
|
||||
int32_t* colLength = (int32_t*)pStart;
|
||||
pStart += sizeof(int32_t) * numOfCols;
|
||||
|
|
@ -1577,7 +1780,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
|
|||
const SSchema* pColumn = &pTableMeta->schema[k];
|
||||
int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name));
|
||||
if (!index) {
|
||||
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k);
|
||||
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k);
|
||||
} else {
|
||||
char* colData = rspObj.resInfo.row[*index];
|
||||
if (!colData) {
|
||||
|
|
@ -1670,6 +1873,7 @@ end:
|
|||
return code;
|
||||
}
|
||||
|
||||
|
||||
static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SHashObj* pVgHash = NULL;
|
||||
|
|
@ -1882,7 +2086,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
|
|||
const SSchema* pColumn = &pTableMeta->schema[k];
|
||||
int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name));
|
||||
if (!index) {
|
||||
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k);
|
||||
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k);
|
||||
} else {
|
||||
char* colData = rspObj.resInfo.row[*index];
|
||||
if (!colData) {
|
||||
|
|
@ -1963,7 +2167,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
|
|||
launchQueryImpl(pRequest, pQuery, true, NULL);
|
||||
code = pRequest->code;
|
||||
|
||||
end:
|
||||
end:
|
||||
tDeleteSTaosxRsp(&rspObj.rsp);
|
||||
rspObj.resInfo.pRspMsg = NULL;
|
||||
doFreeReqResultInfo(&rspObj.resInfo);
|
||||
|
|
|
|||
|
|
@ -79,7 +79,6 @@
|
|||
#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" "
|
||||
|
||||
#define MAX_RETRY_TIMES 5
|
||||
#define LINE_BATCH 2000
|
||||
//=================================================================================================
|
||||
typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
|
||||
|
||||
|
|
@ -140,6 +139,8 @@ typedef struct {
|
|||
int32_t numOfSTables;
|
||||
int32_t numOfCTables;
|
||||
int32_t numOfCreateSTables;
|
||||
int32_t numOfAlterColSTables;
|
||||
int32_t numOfAlterTagSTables;
|
||||
|
||||
int64_t parseTime;
|
||||
int64_t schemaTime;
|
||||
|
|
@ -467,6 +468,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
goto end;
|
||||
}
|
||||
info->cost.numOfCreateSTables++;
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
|
||||
code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname);
|
||||
goto end;
|
||||
}
|
||||
} else if (code == TSDB_CODE_SUCCESS) {
|
||||
hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true,
|
||||
HASH_NO_LOCK);
|
||||
|
|
@ -505,8 +513,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
|
||||
goto end;
|
||||
}
|
||||
}
|
||||
|
||||
info->cost.numOfAlterTagSTables++;
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
|
@ -516,6 +524,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto end;
|
||||
}
|
||||
}
|
||||
|
||||
taosHashClear(hashTmp);
|
||||
for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
|
||||
|
|
@ -552,12 +561,20 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
|
||||
goto end;
|
||||
}
|
||||
}
|
||||
|
||||
info->cost.numOfAlterColSTables++;
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto end;
|
||||
}
|
||||
code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname);
|
||||
goto end;
|
||||
}
|
||||
}
|
||||
|
||||
needCheckMeta = true;
|
||||
taosHashCleanup(hashTmp);
|
||||
hashTmp = NULL;
|
||||
|
|
@ -565,13 +582,6 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code));
|
||||
goto end;
|
||||
}
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
|
||||
code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname);
|
||||
goto end;
|
||||
}
|
||||
|
||||
if (needCheckMeta) {
|
||||
code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags,
|
||||
|
|
@ -596,7 +606,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
|
|||
end:
|
||||
taosHashCleanup(hashTmp);
|
||||
taosMemoryFreeClear(pTableMeta);
|
||||
catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
|
||||
// catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -867,7 +877,10 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra
|
|||
}
|
||||
uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts);
|
||||
|
||||
if (ts == -1) return TSDB_CODE_INVALID_TIMESTAMP;
|
||||
if (ts <= 0) {
|
||||
uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts);
|
||||
return TSDB_CODE_INVALID_TIMESTAMP;
|
||||
}
|
||||
|
||||
// add ts to
|
||||
SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1);
|
||||
|
|
@ -2066,7 +2079,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *
|
|||
|
||||
static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) {
|
||||
SSmlLineInfo elements = {0};
|
||||
uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s", info->id, (info->isRawLine ? "rawdata" : sql));
|
||||
uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : sql));
|
||||
|
||||
int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf);
|
||||
if (ret != TSDB_CODE_SUCCESS) {
|
||||
|
|
@ -2359,11 +2372,12 @@ static int32_t smlInsertData(SSmlHandle *info) {
|
|||
|
||||
static void smlPrintStatisticInfo(SSmlHandle *info) {
|
||||
uError("SML:0x%" PRIx64
|
||||
" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d \
|
||||
" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
|
||||
parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
|
||||
"",
|
||||
info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables,
|
||||
info->cost.numOfCreateSTables, info->cost.schemaTime - info->cost.parseTime,
|
||||
info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables,
|
||||
info->cost.schemaTime - info->cost.parseTime,
|
||||
info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
|
||||
info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
|
||||
}
|
||||
|
|
@ -2562,7 +2576,7 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char
|
|||
goto end;
|
||||
}
|
||||
|
||||
batchs = ceil(((double)numLines) / LINE_BATCH);
|
||||
batchs = ceil(((double)numLines) / tsSmlBatchSize);
|
||||
params.total = batchs;
|
||||
for (int i = 0; i < batchs; ++i) {
|
||||
SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT, 0);
|
||||
|
|
@ -2581,7 +2595,7 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char
|
|||
info->isRawLine = (rawLine == NULL);
|
||||
info->ttl = ttl;
|
||||
|
||||
int32_t perBatch = LINE_BATCH;
|
||||
int32_t perBatch = tsSmlBatchSize;
|
||||
|
||||
if (numLines > perBatch) {
|
||||
numLines -= perBatch;
|
||||
|
|
|
|||
|
|
@ -206,16 +206,15 @@ static const SSysDbTableSchema vgroupsSchema[] = {
|
|||
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "v1_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "v1_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
|
||||
{.name = "v1_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "v2_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
|
||||
{.name = "v2_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "v3_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
|
||||
{.name = "v3_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
|
||||
{.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
|
||||
{.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
|
||||
};
|
||||
|
||||
|
|
@ -274,6 +273,12 @@ static const SSysDbTableSchema vnodesSchema[] = {
|
|||
{.name = "dnode_ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
};
|
||||
|
||||
static const SSysDbTableSchema userUserPrivilegesSchema[] = {
|
||||
{.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
{.name = "object_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
};
|
||||
|
||||
static const SSysTableMeta infosMeta[] = {
|
||||
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
|
||||
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
|
||||
|
|
@ -298,6 +303,7 @@ static const SSysTableMeta infosMeta[] = {
|
|||
{TSDB_INS_TABLE_STREAMS, streamSchema, tListLen(streamSchema), false},
|
||||
{TSDB_INS_TABLE_STREAM_TASKS, streamTaskSchema, tListLen(streamTaskSchema), false},
|
||||
{TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
|
||||
{TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), false},
|
||||
};
|
||||
|
||||
static const SSysDbTableSchema connectionsSchema[] = {
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table
|
|||
// If set to empty system will generate table name using MD5 hash.
|
||||
// true means that the name and order of cols in each line are the same(only for influx protocol)
|
||||
bool tsSmlDataFormat = false;
|
||||
int32_t tsSmlBatchSize = 10000;
|
||||
|
||||
// query
|
||||
int32_t tsQueryPolicy = 1;
|
||||
|
|
@ -166,6 +167,7 @@ int64_t tsWalFsyncDataSizeLimit = (100 * 1024 * 1024L);
|
|||
// internal
|
||||
int32_t tsTransPullupInterval = 2;
|
||||
int32_t tsMqRebalanceInterval = 2;
|
||||
int32_t tsStreamCheckpointTickInterval = 1;
|
||||
int32_t tsTtlUnit = 86400;
|
||||
int32_t tsTtlPushInterval = 86400;
|
||||
int32_t tsGrantHBInterval = 60;
|
||||
|
|
@ -306,6 +308,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
|
|||
if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
|
||||
if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
|
||||
if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "rpcRetryLimit", tsRpcRetryLimit, 1, 100000, 0) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "rpcRetryInterval", tsRpcRetryInterval, 1, 100000, 0) != 0) return -1;
|
||||
|
|
@ -648,6 +651,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
|
||||
tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
|
||||
|
||||
tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32;
|
||||
tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32;
|
||||
|
||||
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
|
||||
|
|
@ -1021,6 +1025,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
|
|||
tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
|
||||
} else if (strcasecmp("smlDataFormat", name) == 0) {
|
||||
tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
|
||||
} else if (strcasecmp("smlBatchSize", name) == 0) {
|
||||
tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32;
|
||||
} else if (strcasecmp("shellActivityTimer", name) == 0) {
|
||||
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
|
||||
} else if (strcasecmp("supportVnodes", name) == 0) {
|
||||
|
|
|
|||
|
|
@ -992,15 +992,20 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
|
|||
if (tEncodeI32(&encoder, vlen) < 0) return -1;
|
||||
for (int32_t i = 0; i < vlen; ++i) {
|
||||
SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i);
|
||||
int64_t reserved = 0;
|
||||
if (tEncodeI32(&encoder, pload->vgId) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pload->syncState) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pload->syncRestore) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pload->syncCanRead) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pload->cacheUsage) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pload->compStorage) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pload->pointsWritten) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, reserved) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, reserved) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, reserved) < 0) return -1;
|
||||
}
|
||||
|
||||
// mnode loads
|
||||
|
|
@ -1065,15 +1070,20 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
|
|||
|
||||
for (int32_t i = 0; i < vlen; ++i) {
|
||||
SVnodeLoad vload = {0};
|
||||
int64_t reserved = 0;
|
||||
if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &vload.syncState) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &vload.syncRestore) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &vload.syncCanRead) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &vload.cacheUsage) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
|
||||
if (taosArrayPush(pReq->pVloads, &vload) == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
|
|
@ -1288,7 +1298,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
|
|||
if (tEncodeI8(&encoder, pReq->enable) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->dbname) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->objname) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
|
|
@ -1307,7 +1317,7 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
|
|||
if (tDecodeI8(&decoder, &pReq->enable) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->dbname) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->objname) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
|
|
@ -3738,6 +3748,31 @@ int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pReq->tick) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSMStreamTickMsg(void *buf, int32_t bufLen, SMStreamTickReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeI64(&decoder, &pReq->tick) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) {
|
||||
if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1;
|
||||
if (tEncodeU16(pEncoder, pReplica->port) < 0) return -1;
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
|
|||
SServerStatusRsp statusRsp = {0};
|
||||
SMonMloadInfo minfo = {0};
|
||||
(*pMgmt->getMnodeLoadsFp)(&minfo);
|
||||
if (minfo.isMnode && minfo.load.syncState == TAOS_SYNC_STATE_ERROR) {
|
||||
if (minfo.isMnode && (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) {
|
||||
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED;
|
||||
snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState));
|
||||
return;
|
||||
|
|
@ -160,7 +160,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
|
|||
(*pMgmt->getVnodeLoadsFp)(&vinfo);
|
||||
for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) {
|
||||
SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i);
|
||||
if (pLoad->syncState == TAOS_SYNC_STATE_ERROR) {
|
||||
if (pLoad->syncState == TAOS_SYNC_STATE_ERROR || pLoad->syncState == TAOS_SYNC_STATE_OFFLINE) {
|
||||
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED;
|
||||
snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pLoad->vgId,
|
||||
syncStr(pLoad->syncState));
|
||||
|
|
|
|||
|
|
@ -144,6 +144,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
|
|||
break;
|
||||
default:
|
||||
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
|
||||
dGError("msg:%p, not processed in mgmt queue", pMsg);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -196,7 +196,7 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER;
|
||||
|
||||
code = 0;
|
||||
|
||||
|
|
|
|||
|
|
@ -162,11 +162,13 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
|
|||
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
|
||||
if (pMsg == NULL) return -1;
|
||||
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
|
||||
pRpc->pCont = NULL;
|
||||
|
||||
dTrace("msg:%p, is created and will put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
|
||||
int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg);
|
||||
if (code != 0) {
|
||||
dTrace("msg:%p, is freed", pMsg);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
return code;
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
|
|||
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
|
||||
if (pMsg == NULL) return -1;
|
||||
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
|
||||
pRpc->pCont = NULL;
|
||||
|
||||
switch (qtype) {
|
||||
case QUERY_QUEUE:
|
||||
|
|
@ -74,6 +75,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
|
|||
return 0;
|
||||
default:
|
||||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
return -1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
|
|||
pHead->contLen = htonl(pHead->contLen);
|
||||
pHead->vgId = SNODE_HANDLE;
|
||||
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
|
||||
pRpc->pCont = NULL;
|
||||
|
||||
switch (qtype) {
|
||||
case STREAM_QUEUE:
|
||||
|
|
|
|||
|
|
@ -464,7 +464,7 @@ SArray *vmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
code = 0;
|
||||
|
||||
|
|
|
|||
|
|
@ -246,12 +246,12 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
|
|||
pHead->contLen = htonl(pHead->contLen);
|
||||
pHead->vgId = htonl(pHead->vgId);
|
||||
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
|
||||
pRpc->pCont = NULL;
|
||||
|
||||
int32_t code = vmPutMsgToQueue(pMgmt, pMsg, qtype);
|
||||
if (code != 0) {
|
||||
dTrace("msg:%p, is freed", pMsg);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
pRpc->pCont = NULL;
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -51,13 +51,15 @@ static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) {
|
|||
}
|
||||
|
||||
int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
|
||||
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)];
|
||||
if (msgFp == NULL) {
|
||||
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
|
||||
dGError("msg:%p, not processed since no handler", pMsg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name);
|
||||
pMsg->info.wrapper = pWrapper;
|
||||
return (*msgFp)(pWrapper->pMgmt, pMsg);
|
||||
|
|
@ -99,18 +101,23 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
|
|||
dmProcessServerStartupStatus(pDnode, pRpc);
|
||||
return;
|
||||
} else {
|
||||
terrno = TSDB_CODE_APP_NOT_READY;
|
||||
if (pDnode->status == DND_STAT_INIT) {
|
||||
terrno = TSDB_CODE_APP_IS_STARTING;
|
||||
} else {
|
||||
terrno = TSDB_CODE_APP_IS_STOPPING;
|
||||
}
|
||||
goto _OVER;
|
||||
}
|
||||
}
|
||||
|
||||
if (IsReq(pRpc) && pRpc->pCont == NULL) {
|
||||
if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) {
|
||||
dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType));
|
||||
terrno = TSDB_CODE_INVALID_MSG_LEN;
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
if (pHandle->defaultNtype == NODE_END) {
|
||||
dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType));
|
||||
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
|
||||
goto _OVER;
|
||||
}
|
||||
|
|
@ -234,7 +241,8 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcRe
|
|||
|
||||
static bool rpcRfp(int32_t code, tmsg_t msgType) {
|
||||
if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
|
||||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) {
|
||||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK ||
|
||||
code == TSDB_CODE_VND_STOPPED) {
|
||||
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH ||
|
||||
msgType == TDMT_SCH_MERGE_FETCH) {
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -71,6 +71,9 @@ typedef enum {
|
|||
MND_OPER_READ_DB,
|
||||
MND_OPER_READ_OR_WRITE_DB,
|
||||
MND_OPER_SHOW_VARIBALES,
|
||||
MND_OPER_SUBSCRIBE,
|
||||
MND_OPER_CREATE_TOPIC,
|
||||
MND_OPER_DROP_TOPIC,
|
||||
} EOperType;
|
||||
|
||||
typedef enum {
|
||||
|
|
@ -273,6 +276,7 @@ typedef struct {
|
|||
int32_t authVersion;
|
||||
SHashObj* readDbs;
|
||||
SHashObj* writeDbs;
|
||||
SHashObj* topics;
|
||||
SRWLatch lock;
|
||||
} SUserObj;
|
||||
|
||||
|
|
@ -328,6 +332,7 @@ typedef struct {
|
|||
int32_t dnodeId;
|
||||
ESyncState syncState;
|
||||
bool syncRestore;
|
||||
bool syncCanRead;
|
||||
} SVnodeGid;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -635,10 +640,14 @@ typedef struct {
|
|||
SArray* tasks; // SArray<SArray<SStreamTask>>
|
||||
SSchemaWrapper outputSchema;
|
||||
SSchemaWrapper tagSchema;
|
||||
|
||||
// 3.0.20
|
||||
int64_t checkpointFreq; // ms
|
||||
int64_t currentTick; // do not serialize
|
||||
} SStreamObj;
|
||||
|
||||
int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj);
|
||||
int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj);
|
||||
int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj, int32_t sver);
|
||||
void tFreeStreamObj(SStreamObj* pObj);
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -648,15 +657,6 @@ typedef struct {
|
|||
SArray* childInfo; // SArray<SStreamChildEpInfo>
|
||||
} SStreamCheckpointObj;
|
||||
|
||||
#if 0
|
||||
typedef struct {
|
||||
int64_t uid;
|
||||
int64_t streamId;
|
||||
int8_t status;
|
||||
int8_t stage;
|
||||
} SStreamRecoverObj;
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ void mndCleanupPrivilege(SMnode *pMnode);
|
|||
int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType);
|
||||
int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb);
|
||||
int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname);
|
||||
int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic);
|
||||
int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName);
|
||||
int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);
|
||||
int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
|
||||
int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp);
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ extern "C" {
|
|||
int32_t mndInitTopic(SMnode *pMnode);
|
||||
void mndCleanupTopic(SMnode *pMnode);
|
||||
|
||||
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName);
|
||||
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName);
|
||||
void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic);
|
||||
|
||||
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic);
|
||||
|
|
|
|||
|
|
@ -36,8 +36,6 @@ int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup);
|
|||
SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId);
|
||||
int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup);
|
||||
int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups);
|
||||
int32_t mndAddVnodeToVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray);
|
||||
int32_t mndRemoveVnodeFromVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid);
|
||||
int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid);
|
||||
int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup);
|
||||
int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType);
|
||||
|
|
|
|||
|
|
@ -147,6 +147,8 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SAcctObj *pAcct = NULL;
|
||||
SSdbRow *pRow = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -156,10 +158,10 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SAcctObj));
|
||||
pRow = sdbAllocRow(sizeof(SAcctObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SAcctObj *pAcct = sdbGetRowObj(pRow);
|
||||
pAcct = sdbGetRowObj(pRow);
|
||||
if (pAcct == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -186,7 +188,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("acct:%s, failed to decode from raw:%p since %s", pAcct->acct, pRaw, terrstr());
|
||||
mError("acct:%s, failed to decode from raw:%p since %s", pAcct == NULL ? "null" : pAcct->acct, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -157,6 +157,8 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SClusterObj *pCluster = NULL;
|
||||
SSdbRow *pRow = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -166,10 +168,10 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SClusterObj));
|
||||
pRow = sdbAllocRow(sizeof(SClusterObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SClusterObj *pCluster = sdbGetRowObj(pRow);
|
||||
pCluster = sdbGetRowObj(pRow);
|
||||
if (pCluster == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -184,7 +186,8 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster->id, pRaw, terrstr());
|
||||
mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster == NULL ? 0 : pCluster->id, pRaw,
|
||||
terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -538,7 +538,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
|
|||
|
||||
int32_t code = -1;
|
||||
SArray *newSub = subscribe.topicNames;
|
||||
taosArraySortString(newSub, taosArrayCompareString);
|
||||
taosArraySort(newSub, taosArrayCompareString);
|
||||
taosArrayRemoveDuplicateP(newSub, taosArrayCompareString, taosMemoryFree);
|
||||
|
||||
int32_t newTopicNum = taosArrayGetSize(newSub);
|
||||
|
|
@ -712,6 +712,8 @@ CM_ENCODE_OVER:
|
|||
|
||||
SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SMqConsumerObj *pConsumer = NULL;
|
||||
void *buf = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
|
|
@ -722,10 +724,10 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
|
|||
goto CM_DECODE_OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SMqConsumerObj));
|
||||
pRow = sdbAllocRow(sizeof(SMqConsumerObj));
|
||||
if (pRow == NULL) goto CM_DECODE_OVER;
|
||||
|
||||
SMqConsumerObj *pConsumer = sdbGetRowObj(pRow);
|
||||
pConsumer = sdbGetRowObj(pRow);
|
||||
if (pConsumer == NULL) goto CM_DECODE_OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -745,7 +747,8 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
|
|||
CM_DECODE_OVER:
|
||||
taosMemoryFreeClear(buf);
|
||||
if (terrno != TSDB_CODE_SUCCESS) {
|
||||
mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr());
|
||||
mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer == NULL ? 0 : pConsumer->consumerId,
|
||||
pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -850,7 +853,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
|
|||
|
||||
// add to current topic
|
||||
taosArrayPush(pOldConsumer->currentTopics, &addedTopic);
|
||||
taosArraySortString(pOldConsumer->currentTopics, taosArrayCompareString);
|
||||
taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString);
|
||||
|
||||
// set status
|
||||
if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {
|
||||
if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY ||
|
||||
|
|
|
|||
|
|
@ -147,6 +147,8 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SDbObj *pDb = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -156,10 +158,10 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj));
|
||||
pRow = sdbAllocRow(sizeof(SDbObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SDbObj *pDb = sdbGetRowObj(pRow);
|
||||
pDb = sdbGetRowObj(pRow);
|
||||
if (pDb == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -232,7 +234,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("db:%s, failed to decode from raw:%p since %s", pDb->name, pRaw, terrstr());
|
||||
mError("db:%s, failed to decode from raw:%p since %s", pDb == NULL ? "null" : pDb->name, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -76,11 +76,14 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
|
|||
|
||||
if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1;
|
||||
|
||||
// 3.0.20
|
||||
if (tEncodeI64(pEncoder, pObj->checkpointFreq) < 0) return -1;
|
||||
|
||||
tEndEncode(pEncoder);
|
||||
return pEncoder->pos;
|
||||
}
|
||||
|
||||
int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
|
||||
int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
|
||||
if (tStartDecode(pDecoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
|
||||
|
||||
|
|
@ -139,6 +142,10 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
|
|||
|
||||
if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1;
|
||||
|
||||
// 3.0.20
|
||||
if (sver >= 2) {
|
||||
if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1;
|
||||
}
|
||||
tEndDecode(pDecoder);
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -154,8 +154,9 @@ _OVER:
|
|||
}
|
||||
|
||||
static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
|
||||
SSdbRow *pRow = NULL;
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SDnodeObj *pDnode = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -166,7 +167,8 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
pRow = sdbAllocRow(sizeof(SDnodeObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
SDnodeObj *pDnode = sdbGetRowObj(pRow);
|
||||
|
||||
pDnode = sdbGetRowObj(pRow);
|
||||
if (pDnode == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -181,7 +183,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("dnode:%d, failed to decode from raw:%p since %s", pDnode->id, pRaw, terrstr());
|
||||
mError("dnode:%d, failed to decode from raw:%p since %s", pDnode == NULL ? 0 : pDnode->id, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -375,14 +377,18 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
|
|||
}
|
||||
bool roleChanged = false;
|
||||
for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
|
||||
if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) {
|
||||
if (pVgroup->vnodeGid[vg].syncState != pVload->syncState ||
|
||||
pVgroup->vnodeGid[vg].syncRestore != pVload->syncRestore) {
|
||||
mInfo("vgId:%d, state changed by status msg, old state:%s restored:%d new state:%s restored:%d",
|
||||
pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore,
|
||||
syncStr(pVload->syncState), pVload->syncRestore);
|
||||
pVgroup->vnodeGid[vg].syncState = pVload->syncState;
|
||||
pVgroup->vnodeGid[vg].syncRestore = pVload->syncRestore;
|
||||
SVnodeGid *pGid = &pVgroup->vnodeGid[vg];
|
||||
if (pGid->dnodeId == statusReq.dnodeId) {
|
||||
if (pGid->syncState != pVload->syncState || pGid->syncRestore != pVload->syncRestore ||
|
||||
pGid->syncCanRead != pVload->syncCanRead) {
|
||||
mInfo(
|
||||
"vgId:%d, state changed by status msg, old state:%s restored:%d canRead:%d new state:%s restored:%d "
|
||||
"canRead:%d",
|
||||
pVgroup->vgId, syncStr(pGid->syncState), pGid->syncRestore, pGid->syncCanRead,
|
||||
syncStr(pVload->syncState), pVload->syncRestore, pVload->syncCanRead);
|
||||
pGid->syncState = pVload->syncState;
|
||||
pGid->syncRestore = pVload->syncRestore;
|
||||
pGid->syncCanRead = pVload->syncCanRead;
|
||||
roleChanged = true;
|
||||
}
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -101,6 +101,8 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SFuncObj *pFunc = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -110,10 +112,10 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SFuncObj));
|
||||
pRow = sdbAllocRow(sizeof(SFuncObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SFuncObj *pFunc = sdbGetRowObj(pRow);
|
||||
pFunc = sdbGetRowObj(pRow);
|
||||
if (pFunc == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -148,7 +150,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("func:%s, failed to decode from raw:%p since %s", pFunc->name, pRaw, terrstr());
|
||||
mError("func:%s, failed to decode from raw:%p since %s", pFunc == NULL ? "null" : pFunc->name, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -85,6 +85,21 @@ static void *mndBuildTimerMsg(int32_t *pContLen) {
|
|||
return pReq;
|
||||
}
|
||||
|
||||
static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) {
|
||||
SMStreamTickReq timerReq = {
|
||||
.tick = sec,
|
||||
};
|
||||
|
||||
int32_t contLen = tSerializeSMStreamTickMsg(NULL, 0, &timerReq);
|
||||
if (contLen <= 0) return NULL;
|
||||
void *pReq = rpcMallocCont(contLen);
|
||||
if (pReq == NULL) return NULL;
|
||||
|
||||
tSerializeSMStreamTickMsg(pReq, contLen, &timerReq);
|
||||
*pContLen = contLen;
|
||||
return pReq;
|
||||
}
|
||||
|
||||
static void mndPullupTrans(SMnode *pMnode) {
|
||||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildTimerMsg(&contLen);
|
||||
|
|
@ -105,7 +120,24 @@ static void mndCalMqRebalance(SMnode *pMnode) {
|
|||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildTimerMsg(&contLen);
|
||||
if (pReq != NULL) {
|
||||
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen};
|
||||
SRpcMsg rpcMsg = {
|
||||
.msgType = TDMT_MND_TMQ_TIMER,
|
||||
.pCont = pReq,
|
||||
.contLen = contLen,
|
||||
};
|
||||
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
|
||||
}
|
||||
}
|
||||
|
||||
static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) {
|
||||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildCheckpointTickMsg(&contLen, sec);
|
||||
if (pReq != NULL) {
|
||||
SRpcMsg rpcMsg = {
|
||||
.msgType = TDMT_MND_STREAM_CHECKPOINT_TIMER,
|
||||
.pCont = pReq,
|
||||
.contLen = contLen,
|
||||
};
|
||||
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
|
||||
}
|
||||
}
|
||||
|
|
@ -150,12 +182,16 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs)
|
|||
|
||||
bool roleChanged = false;
|
||||
for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
|
||||
if (pVgroup->vnodeGid[vg].dnodeId == dnodeId) {
|
||||
if (pVgroup->vnodeGid[vg].syncState != TAOS_SYNC_STATE_ERROR) {
|
||||
mInfo("vgId:%d, state changed by offline check, old state:%s restored:%d new state:error restored:0",
|
||||
pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore);
|
||||
pVgroup->vnodeGid[vg].syncState = TAOS_SYNC_STATE_ERROR;
|
||||
pVgroup->vnodeGid[vg].syncRestore = 0;
|
||||
SVnodeGid *pGid = &pVgroup->vnodeGid[vg];
|
||||
if (pGid->dnodeId == dnodeId) {
|
||||
if (pGid->syncState != TAOS_SYNC_STATE_OFFLINE) {
|
||||
mInfo(
|
||||
"vgId:%d, state changed by offline check, old state:%s restored:%d canRead:%d new state:error restored:0 "
|
||||
"canRead:0",
|
||||
pVgroup->vgId, syncStr(pGid->syncState), pGid->syncRestore, pGid->syncCanRead);
|
||||
pGid->syncState = TAOS_SYNC_STATE_OFFLINE;
|
||||
pGid->syncRestore = 0;
|
||||
pGid->syncCanRead = 0;
|
||||
roleChanged = true;
|
||||
}
|
||||
break;
|
||||
|
|
@ -220,6 +256,12 @@ static void *mndThreadFp(void *param) {
|
|||
mndCalMqRebalance(pMnode);
|
||||
}
|
||||
|
||||
#if 0
|
||||
if (sec % tsStreamCheckpointTickInterval == 0) {
|
||||
mndStreamCheckpointTick(pMnode, sec);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (sec % tsTelemInterval == (TMIN(60, (tsTelemInterval - 1)))) {
|
||||
mndPullupTelem(pMnode);
|
||||
}
|
||||
|
|
@ -491,6 +533,16 @@ void mndPreClose(SMnode *pMnode) {
|
|||
if (pMnode != NULL) {
|
||||
syncLeaderTransfer(pMnode->syncMgmt.sync);
|
||||
syncPreStop(pMnode->syncMgmt.sync);
|
||||
#if 0
|
||||
while (syncSnapshotRecving(pMnode->syncMgmt.sync)) {
|
||||
mInfo("vgId:1, snapshot is recving");
|
||||
taosMsleep(300);
|
||||
}
|
||||
while (syncSnapshotSending(pMnode->syncMgmt.sync)) {
|
||||
mInfo("vgId:1, snapshot is sending");
|
||||
taosMsleep(300);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -592,17 +644,6 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
static int32_t mndCheckMsgContent(SRpcMsg *pMsg) {
|
||||
if (!IsReq(pMsg)) return 0;
|
||||
if (pMsg->contLen != 0 && pMsg->pCont != NULL) return 0;
|
||||
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
mGError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen,
|
||||
pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
|
||||
terrno = TSDB_CODE_INVALID_MSG_LEN;
|
||||
return -1;
|
||||
}
|
||||
|
||||
int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
|
||||
SMnode *pMnode = pMsg->info.node;
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
|
|
@ -614,7 +655,6 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
if (mndCheckMsgContent(pMsg) != 0) return -1;
|
||||
if (mndCheckMnodeState(pMsg) != 0) return -1;
|
||||
|
||||
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
|
||||
|
|
@ -747,7 +787,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
|
|||
tstrncpy(desc.status, "ready", sizeof(desc.status));
|
||||
pClusterInfo->vgroups_alive++;
|
||||
}
|
||||
if (pVgid->syncState != TAOS_SYNC_STATE_ERROR) {
|
||||
if (pVgid->syncState != TAOS_SYNC_STATE_ERROR && pVgid->syncState != TAOS_SYNC_STATE_OFFLINE) {
|
||||
pClusterInfo->vnodes_alive++;
|
||||
}
|
||||
pClusterInfo->vnodes_total++;
|
||||
|
|
|
|||
|
|
@ -142,6 +142,8 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SMnodeObj *pObj = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
|
||||
|
|
@ -151,10 +153,10 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SMnodeObj));
|
||||
pRow = sdbAllocRow(sizeof(SMnodeObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SMnodeObj *pObj = sdbGetRowObj(pRow);
|
||||
pObj = sdbGetRowObj(pRow);
|
||||
if (pObj == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -167,7 +169,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("mnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr());
|
||||
mError("mnode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -185,7 +187,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
pObj->syncState = TAOS_SYNC_STATE_ERROR;
|
||||
pObj->syncState = TAOS_SYNC_STATE_OFFLINE;
|
||||
mndReloadSyncConfig(pSdb->pMnode);
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,10 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType
|
|||
int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) {
|
||||
return 0;
|
||||
}
|
||||
int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic) { return 0; }
|
||||
int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName) {
|
||||
return 0;
|
||||
}
|
||||
int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) {
|
||||
memcpy(pRsp->user, pUser->user, TSDB_USER_LEN);
|
||||
pRsp->superAuth = 1;
|
||||
|
|
|
|||
|
|
@ -100,6 +100,8 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SQnodeObj *pObj = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -109,10 +111,10 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SQnodeObj));
|
||||
pRow = sdbAllocRow(sizeof(SQnodeObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SQnodeObj *pObj = sdbGetRowObj(pRow);
|
||||
pObj = sdbGetRowObj(pRow);
|
||||
if (pObj == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -125,7 +127,7 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("qnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr());
|
||||
mError("qnode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -132,6 +132,8 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SSmaObj *pSma = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -141,10 +143,10 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SSmaObj));
|
||||
pRow = sdbAllocRow(sizeof(SSmaObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SSmaObj *pSma = sdbGetRowObj(pRow);
|
||||
pSma = sdbGetRowObj(pRow);
|
||||
if (pSma == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -200,7 +202,7 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr());
|
||||
mError("sma:%s, failed to decode from raw:%p since %s", pSma == NULL ? "null" : pSma->name, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pSma->expr);
|
||||
taosMemoryFreeClear(pSma->tagsFilter);
|
||||
taosMemoryFreeClear(pSma->sql);
|
||||
|
|
|
|||
|
|
@ -105,6 +105,8 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SSnodeObj *pObj = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -114,10 +116,10 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SSnodeObj));
|
||||
pRow = sdbAllocRow(sizeof(SSnodeObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SSnodeObj *pObj = sdbGetRowObj(pRow);
|
||||
pObj = sdbGetRowObj(pRow);
|
||||
if (pObj == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -130,7 +132,7 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("snode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr());
|
||||
mError("snode:%d, failed to decode from raw:%p since %s", pObj == NULL ? 0 : pObj->id, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -162,6 +162,8 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SStbObj *pStb = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -171,10 +173,10 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SStbObj));
|
||||
pRow = sdbAllocRow(sizeof(SStbObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SStbObj *pStb = sdbGetRowObj(pRow);
|
||||
pStb = sdbGetRowObj(pRow);
|
||||
if (pStb == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -254,10 +256,12 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("stb:%s, failed to decode from raw:%p since %s", pStb->name, pRaw, terrstr());
|
||||
mError("stb:%s, failed to decode from raw:%p since %s", pStb == NULL ? "null" : pStb->name, pRaw, terrstr());
|
||||
if (pStb != NULL) {
|
||||
taosMemoryFreeClear(pStb->pColumns);
|
||||
taosMemoryFreeClear(pStb->pTags);
|
||||
taosMemoryFreeClear(pStb->comment);
|
||||
}
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@
|
|||
#include "parser.h"
|
||||
#include "tname.h"
|
||||
|
||||
#define MND_STREAM_VER_NUMBER 1
|
||||
#define MND_STREAM_VER_NUMBER 2
|
||||
#define MND_STREAM_RESERVE_SIZE 64
|
||||
|
||||
static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
|
||||
|
|
@ -36,6 +36,8 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
|
|||
static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream);
|
||||
static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
|
||||
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
|
||||
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
|
||||
static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
|
||||
/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/
|
||||
static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq);
|
||||
static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
|
||||
|
|
@ -62,6 +64,10 @@ int32_t mndInitStream(SMnode *pMnode) {
|
|||
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp);
|
||||
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp);
|
||||
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
|
||||
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
|
||||
|
||||
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream);
|
||||
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream);
|
||||
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
|
||||
|
|
@ -120,21 +126,22 @@ STREAM_ENCODE_OVER:
|
|||
|
||||
SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SStreamObj *pStream = NULL;
|
||||
void *buf = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER;
|
||||
|
||||
if (sver != MND_STREAM_VER_NUMBER) {
|
||||
if (sver != 1 && sver != 2) {
|
||||
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
|
||||
goto STREAM_DECODE_OVER;
|
||||
}
|
||||
|
||||
int32_t size = sizeof(SStreamObj);
|
||||
SSdbRow *pRow = sdbAllocRow(size);
|
||||
pRow = sdbAllocRow(sizeof(SStreamObj));
|
||||
if (pRow == NULL) goto STREAM_DECODE_OVER;
|
||||
|
||||
SStreamObj *pStream = sdbGetRowObj(pRow);
|
||||
pStream = sdbGetRowObj(pRow);
|
||||
if (pStream == NULL) goto STREAM_DECODE_OVER;
|
||||
|
||||
int32_t tlen;
|
||||
|
|
@ -146,7 +153,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
SDecoder decoder;
|
||||
tDecoderInit(&decoder, buf, tlen + 1);
|
||||
if (tDecodeSStreamObj(&decoder, pStream) < 0) {
|
||||
if (tDecodeSStreamObj(&decoder, pStream, sver) < 0) {
|
||||
tDecoderClear(&decoder);
|
||||
goto STREAM_DECODE_OVER;
|
||||
}
|
||||
|
|
@ -157,7 +164,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
|
|||
STREAM_DECODE_OVER:
|
||||
taosMemoryFreeClear(buf);
|
||||
if (terrno != TSDB_CODE_SUCCESS) {
|
||||
mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr());
|
||||
mError("stream:%s, failed to decode from raw:%p since %s", pStream == NULL ? "null" : pStream->name, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -679,6 +686,164 @@ _OVER:
|
|||
tFreeStreamObj(&streamObj);
|
||||
return code;
|
||||
}
|
||||
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
void *pIter = NULL;
|
||||
SStreamObj *pStream = NULL;
|
||||
|
||||
// iterate all stream obj
|
||||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
|
||||
if (pIter == NULL) break;
|
||||
// incr tick
|
||||
int64_t currentTick = atomic_add_fetch_64(&pStream->currentTick, 1);
|
||||
// if >= checkpointFreq, build msg TDMT_MND_STREAM_BEGIN_CHECKPOINT, put into write q
|
||||
if (currentTick >= pStream->checkpointFreq) {
|
||||
atomic_store_64(&pStream->currentTick, 0);
|
||||
SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
|
||||
|
||||
pMsg->streamId = pStream->uid;
|
||||
pMsg->checkpointId = tGenIdPI64();
|
||||
memcpy(pMsg->streamName, pStream->name, TSDB_STREAM_FNAME_LEN);
|
||||
|
||||
SRpcMsg rpcMsg = {
|
||||
.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT,
|
||||
.pCont = pMsg,
|
||||
.contLen = sizeof(SMStreamDoCheckpointMsg),
|
||||
};
|
||||
|
||||
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, const SStreamTask *pTask,
|
||||
SMStreamDoCheckpointMsg *pMsg) {
|
||||
SStreamCheckpointSourceReq req = {0};
|
||||
req.checkpointId = pMsg->checkpointId;
|
||||
req.nodeId = pTask->nodeId;
|
||||
req.expireTime = -1;
|
||||
req.streamId = pTask->streamId;
|
||||
req.taskId = pTask->taskId;
|
||||
|
||||
int32_t code;
|
||||
int32_t blen;
|
||||
|
||||
tEncodeSize(tEncodeSStreamCheckpointSourceReq, &req, blen, code);
|
||||
if (code < 0) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
int32_t tlen = sizeof(SMsgHead) + blen;
|
||||
|
||||
void *buf = taosMemoryMalloc(tlen);
|
||||
if (buf == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, abuf, tlen);
|
||||
tEncodeSStreamCheckpointSourceReq(&encoder, &req);
|
||||
|
||||
SMsgHead *pMsgHead = (SMsgHead *)buf;
|
||||
pMsgHead->contLen = htonl(tlen);
|
||||
pMsgHead->vgId = htonl(pTask->nodeId);
|
||||
|
||||
tEncoderClear(&encoder);
|
||||
|
||||
*pBuf = buf;
|
||||
*pLen = tlen;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
|
||||
SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont;
|
||||
|
||||
SStreamObj *pStream = mndAcquireStream(pMnode, pMsg->streamName);
|
||||
|
||||
if (pStream == NULL || pStream->uid != pMsg->streamId) {
|
||||
mError("start checkpointing failed since stream %s not found", pMsg->streamName);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// build new transaction:
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "stream-checkpoint");
|
||||
if (pTrans == NULL) return -1;
|
||||
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
|
||||
taosRLockLatch(&pStream->lock);
|
||||
// 1. redo action: broadcast checkpoint source msg for all source vg
|
||||
int32_t totLevel = taosArrayGetSize(pStream->tasks);
|
||||
for (int32_t i = 0; i < totLevel; i++) {
|
||||
SArray *pLevel = taosArrayGetP(pStream->tasks, i);
|
||||
SStreamTask *pTask = taosArrayGetP(pLevel, 0);
|
||||
if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
|
||||
int32_t sz = taosArrayGetSize(pLevel);
|
||||
for (int32_t j = 0; j < sz; j++) {
|
||||
SStreamTask *pTask = taosArrayGetP(pLevel, j);
|
||||
ASSERT(pTask->nodeId > 0);
|
||||
SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId);
|
||||
if (pVgObj == NULL) {
|
||||
ASSERT(0);
|
||||
taosRUnLockLatch(&pStream->lock);
|
||||
mndReleaseStream(pMnode, pStream);
|
||||
mndTransDrop(pTrans);
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *buf;
|
||||
int32_t tlen;
|
||||
if (mndBuildStreamCheckpointSourceReq(&buf, &tlen, pTask, pMsg) < 0) {
|
||||
taosRUnLockLatch(&pStream->lock);
|
||||
mndReleaseStream(pMnode, pStream);
|
||||
mndTransDrop(pTrans);
|
||||
return -1;
|
||||
}
|
||||
|
||||
STransAction action = {0};
|
||||
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
|
||||
action.pCont = buf;
|
||||
action.contLen = tlen;
|
||||
action.msgType = TDMT_VND_STREAM_CHECK_POINT_SOURCE;
|
||||
|
||||
mndReleaseVgroup(pMnode, pVgObj);
|
||||
|
||||
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
|
||||
taosMemoryFree(buf);
|
||||
taosRUnLockLatch(&pStream->lock);
|
||||
mndReleaseStream(pMnode, pStream);
|
||||
mndTransDrop(pTrans);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// 2. reset tick
|
||||
atomic_store_64(&pStream->currentTick, 0);
|
||||
// 3. commit log: stream checkpoint info
|
||||
taosRUnLockLatch(&pStream->lock);
|
||||
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) {
|
||||
mError("failed to prepare trans rebalance since %s", terrstr());
|
||||
mndTransDrop(pTrans);
|
||||
mndReleaseStream(pMnode, pStream);
|
||||
return -1;
|
||||
}
|
||||
|
||||
mndReleaseStream(pMnode, pStream);
|
||||
mndTransDrop(pTrans);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
|
|
@ -747,71 +912,6 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
|
|||
return TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
SStreamObj *pStream = NULL;
|
||||
/*SDbObj *pDb = NULL;*/
|
||||
/*SUserObj *pUser = NULL;*/
|
||||
|
||||
SMRecoverStreamReq recoverReq = {0};
|
||||
if (tDeserializeSMRecoverStreamReq(pReq->pCont, pReq->contLen, &recoverReq) < 0) {
|
||||
ASSERT(0);
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return -1;
|
||||
}
|
||||
|
||||
pStream = mndAcquireStream(pMnode, recoverReq.name);
|
||||
|
||||
if (pStream == NULL) {
|
||||
if (recoverReq.igNotExists) {
|
||||
mInfo("stream:%s, not exist, ignore not exist is set", recoverReq.name);
|
||||
sdbRelease(pMnode->pSdb, pStream);
|
||||
return 0;
|
||||
} else {
|
||||
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq);
|
||||
if (pTrans == NULL) {
|
||||
mError("stream:%s, failed to recover since %s", recoverReq.name, terrstr());
|
||||
sdbRelease(pMnode->pSdb, pStream);
|
||||
return -1;
|
||||
}
|
||||
mInfo("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name);
|
||||
|
||||
// broadcast to recover all tasks
|
||||
if (mndRecoverStreamTasks(pMnode, pTrans, pStream) < 0) {
|
||||
mError("stream:%s, failed to recover task since %s", recoverReq.name, terrstr());
|
||||
sdbRelease(pMnode->pSdb, pStream);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// update stream status
|
||||
if (mndSetStreamRecover(pMnode, pTrans, pStream) < 0) {
|
||||
sdbRelease(pMnode->pSdb, pStream);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) {
|
||||
mError("trans:%d, failed to prepare recover stream trans since %s", pTrans->id, terrstr());
|
||||
sdbRelease(pMnode->pSdb, pStream);
|
||||
mndTransDrop(pTrans);
|
||||
return -1;
|
||||
}
|
||||
|
||||
sdbRelease(pMnode->pSdb, pStream);
|
||||
|
||||
return TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
void *pIter = NULL;
|
||||
|
|
@ -846,13 +946,6 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
|
|||
}
|
||||
}
|
||||
|
||||
#if 0
|
||||
if (mndSetDropOffsetStreamLogs(pMnode, pTrans, pStream) < 0) {
|
||||
sdbRelease(pSdb, pStream);
|
||||
goto END;
|
||||
}
|
||||
#endif
|
||||
|
||||
sdbRelease(pSdb, pStream);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -440,9 +440,9 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
|
|||
}
|
||||
|
||||
static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) {
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "persist-reb");
|
||||
mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL);
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
|
||||
if (pTrans == NULL) return -1;
|
||||
mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL);
|
||||
|
||||
// make txn:
|
||||
// 1. redo action: action to all vg
|
||||
|
|
@ -523,28 +523,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
|
|||
tDeleteSMqConsumerObj(pConsumerNew);
|
||||
taosMemoryFree(pConsumerNew);
|
||||
}
|
||||
#if 0
|
||||
if (consumerNum) {
|
||||
char topic[TSDB_TOPIC_FNAME_LEN];
|
||||
char cgroup[TSDB_CGROUP_LEN];
|
||||
mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true);
|
||||
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
|
||||
if (pTopic) {
|
||||
// TODO make topic complete
|
||||
SMqTopicObj topicObj = {0};
|
||||
memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
|
||||
topicObj.refConsumerCnt = pTopic->refConsumerCnt - consumerNum;
|
||||
// TODO is that correct?
|
||||
pTopic->refConsumerCnt = topicObj.refConsumerCnt;
|
||||
mInfo("subscribe topic %s unref %d consumer cgroup %s, refcnt %d", pTopic->name, consumerNum, cgroup,
|
||||
topicObj.refConsumerCnt);
|
||||
if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) {
|
||||
ASSERT(0);
|
||||
goto REB_FAIL;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// 4. TODO commit log: modification log
|
||||
|
||||
|
|
@ -743,6 +721,8 @@ SUB_ENCODE_OVER:
|
|||
|
||||
static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SMqSubscribeObj *pSub = NULL;
|
||||
void *buf = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
|
|
@ -753,11 +733,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
|
|||
goto SUB_DECODE_OVER;
|
||||
}
|
||||
|
||||
int32_t size = sizeof(SMqSubscribeObj);
|
||||
SSdbRow *pRow = sdbAllocRow(size);
|
||||
pRow = sdbAllocRow(sizeof(SMqSubscribeObj));
|
||||
if (pRow == NULL) goto SUB_DECODE_OVER;
|
||||
|
||||
SMqSubscribeObj *pSub = sdbGetRowObj(pRow);
|
||||
pSub = sdbGetRowObj(pRow);
|
||||
if (pSub == NULL) goto SUB_DECODE_OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -777,7 +756,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
|
|||
SUB_DECODE_OVER:
|
||||
taosMemoryFreeClear(buf);
|
||||
if (terrno != TSDB_CODE_SUCCESS) {
|
||||
mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr());
|
||||
mError("subscribe:%s, failed to decode from raw:%p since %s", pSub == NULL ? "null" : pSub->key, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -72,15 +72,11 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
|
|||
return code;
|
||||
}
|
||||
|
||||
void mndSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
void mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
SMnode *pMnode = pFsm->data;
|
||||
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
|
||||
SSdbRaw *pRaw = pMsg->pCont;
|
||||
|
||||
// delete msg handle
|
||||
SRpcMsg rpcMsg = {0};
|
||||
rpcMsg.info = pMsg->info;
|
||||
|
||||
int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw);
|
||||
pMgmt->errCode = pMeta->code;
|
||||
mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64
|
||||
|
|
@ -120,6 +116,12 @@ void mndSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMet
|
|||
}
|
||||
}
|
||||
|
||||
void mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
mndProcessWriteMsg(pFsm, pMsg, pMeta);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
pMsg->pCont = NULL;
|
||||
}
|
||||
|
||||
int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) {
|
||||
mInfo("start to read snapshot from sdb in atomic way");
|
||||
SMnode *pMnode = pFsm->data;
|
||||
|
|
@ -361,7 +363,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
|
|||
|
||||
void mndSyncStart(SMnode *pMnode) {
|
||||
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
|
||||
syncStart(pMgmt->sync);
|
||||
if (syncStart(pMgmt->sync) < 0) {
|
||||
mError("vgId:1, failed to start sync, id:%" PRId64, pMgmt->sync);
|
||||
return;
|
||||
}
|
||||
mInfo("vgId:1, sync started, id:%" PRId64, pMgmt->sync);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -152,8 +152,10 @@ TOPIC_ENCODE_OVER:
|
|||
|
||||
SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
||||
SSdbRow *pRow = NULL;
|
||||
SMqTopicObj *pTopic = NULL;
|
||||
void *buf = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER;
|
||||
|
||||
|
|
@ -162,11 +164,10 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
|
|||
goto TOPIC_DECODE_OVER;
|
||||
}
|
||||
|
||||
int32_t size = sizeof(SMqTopicObj);
|
||||
SSdbRow *pRow = sdbAllocRow(size);
|
||||
pRow = sdbAllocRow(sizeof(SMqTopicObj));
|
||||
if (pRow == NULL) goto TOPIC_DECODE_OVER;
|
||||
|
||||
SMqTopicObj *pTopic = sdbGetRowObj(pRow);
|
||||
pTopic = sdbGetRowObj(pRow);
|
||||
if (pTopic == NULL) goto TOPIC_DECODE_OVER;
|
||||
|
||||
int32_t len;
|
||||
|
|
@ -251,7 +252,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
|
|||
TOPIC_DECODE_OVER:
|
||||
taosMemoryFreeClear(buf);
|
||||
if (terrno != TSDB_CODE_SUCCESS) {
|
||||
mError("topic:%s, failed to decode from raw:%p since %s", pTopic->name, pRaw, terrstr());
|
||||
mError("topic:%s, failed to decode from raw:%p since %s", pTopic == NULL ? "null" : pTopic->name, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -288,7 +289,7 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic
|
|||
return 0;
|
||||
}
|
||||
|
||||
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName) {
|
||||
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName) {
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName);
|
||||
if (pTopic == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
|
||||
|
|
@ -573,7 +574,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) {
|
||||
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC) != 0) {
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
|
|
@ -633,6 +634,11 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
|
|||
}
|
||||
}
|
||||
|
||||
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC) != 0) {
|
||||
mndReleaseTopic(pMnode, pTopic);
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *pIter = NULL;
|
||||
SMqConsumerObj *pConsumer;
|
||||
while (1) {
|
||||
|
|
|
|||
|
|
@ -18,10 +18,11 @@
|
|||
#include "mndDb.h"
|
||||
#include "mndPrivilege.h"
|
||||
#include "mndShow.h"
|
||||
#include "mndTopic.h"
|
||||
#include "mndTrans.h"
|
||||
#include "tbase64.h"
|
||||
|
||||
#define USER_VER_NUMBER 1
|
||||
#define USER_VER_NUMBER 2
|
||||
#define USER_RESERVE_SIZE 64
|
||||
|
||||
static int32_t mndCreateDefaultUsers(SMnode *pMnode);
|
||||
|
|
@ -36,6 +37,8 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq);
|
|||
static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq);
|
||||
static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
|
||||
static void mndCancelGetNextUser(SMnode *pMnode, void *pIter);
|
||||
static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
|
||||
static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter);
|
||||
|
||||
int32_t mndInitUser(SMnode *pMnode) {
|
||||
SSdbTable table = {
|
||||
|
|
@ -56,6 +59,8 @@ int32_t mndInitUser(SMnode *pMnode) {
|
|||
|
||||
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER, mndRetrieveUsers);
|
||||
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_USER, mndCancelGetNextUser);
|
||||
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_PRIVILEGES, mndRetrievePrivileges);
|
||||
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_PRIVILEGES, mndCancelGetNextPrivileges);
|
||||
return sdbSetTable(pMnode->pSdb, table);
|
||||
}
|
||||
|
||||
|
|
@ -119,7 +124,9 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
|
|||
|
||||
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
|
||||
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
|
||||
int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
|
||||
int32_t numOfTopics = taosHashGetSize(pUser->topics);
|
||||
int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN +
|
||||
numOfTopics * TSDB_TOPIC_FNAME_LEN;
|
||||
|
||||
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
|
||||
if (pRaw == NULL) goto _OVER;
|
||||
|
|
@ -137,6 +144,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
|
|||
SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, numOfTopics, _OVER)
|
||||
|
||||
char *db = taosHashIterate(pUser->readDbs, NULL);
|
||||
while (db != NULL) {
|
||||
|
|
@ -150,6 +158,12 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
|
|||
db = taosHashIterate(pUser->writeDbs, db);
|
||||
}
|
||||
|
||||
char *topic = taosHashIterate(pUser->topics, NULL);
|
||||
while (topic != NULL) {
|
||||
SDB_SET_BINARY(pRaw, dataPos, topic, TSDB_TOPIC_FNAME_LEN, _OVER);
|
||||
db = taosHashIterate(pUser->topics, topic);
|
||||
}
|
||||
|
||||
SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
|
||||
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
|
||||
|
||||
|
|
@ -168,19 +182,21 @@ _OVER:
|
|||
|
||||
static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SUserObj *pUser = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
||||
if (sver != USER_VER_NUMBER) {
|
||||
if (sver != 1 && sver != 2) {
|
||||
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj));
|
||||
pRow = sdbAllocRow(sizeof(SUserObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SUserObj *pUser = sdbGetRowObj(pRow);
|
||||
pUser = sdbGetRowObj(pRow);
|
||||
if (pUser == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -197,12 +213,18 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
int32_t numOfReadDbs = 0;
|
||||
int32_t numOfWriteDbs = 0;
|
||||
int32_t numOfTopics = 0;
|
||||
SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER)
|
||||
if (sver >= 2) {
|
||||
SDB_GET_INT32(pRaw, dataPos, &numOfTopics, _OVER)
|
||||
}
|
||||
|
||||
pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
|
||||
pUser->writeDbs =
|
||||
taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
|
||||
if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto _OVER;
|
||||
pUser->topics = taosHashInit(numOfTopics, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
|
||||
if (pUser->readDbs == NULL || pUser->writeDbs == NULL || pUser->topics == NULL) goto _OVER;
|
||||
|
||||
for (int32_t i = 0; i < numOfReadDbs; ++i) {
|
||||
char db[TSDB_DB_FNAME_LEN] = {0};
|
||||
|
|
@ -218,6 +240,15 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
|
|||
taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN);
|
||||
}
|
||||
|
||||
if (sver >= 2) {
|
||||
for (int32_t i = 0; i < numOfTopics; ++i) {
|
||||
char topic[TSDB_TOPIC_FNAME_LEN] = {0};
|
||||
SDB_GET_BINARY(pRaw, dataPos, topic, TSDB_TOPIC_FNAME_LEN, _OVER)
|
||||
int32_t len = strlen(topic) + 1;
|
||||
taosHashPut(pUser->topics, topic, len, topic, TSDB_TOPIC_FNAME_LEN);
|
||||
}
|
||||
}
|
||||
|
||||
SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
|
||||
taosInitRWLatch(&pUser->lock);
|
||||
|
||||
|
|
@ -225,9 +256,12 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr());
|
||||
mError("user:%s, failed to decode from raw:%p since %s", pUser == NULL ? "null" : pUser->user, pRaw, terrstr());
|
||||
if (pUser != NULL) {
|
||||
taosHashCleanup(pUser->readDbs);
|
||||
taosHashCleanup(pUser->writeDbs);
|
||||
taosHashCleanup(pUser->topics);
|
||||
}
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -255,8 +289,10 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
|
|||
mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser);
|
||||
taosHashCleanup(pUser->readDbs);
|
||||
taosHashCleanup(pUser->writeDbs);
|
||||
taosHashCleanup(pUser->topics);
|
||||
pUser->readDbs = NULL;
|
||||
pUser->writeDbs = NULL;
|
||||
pUser->topics = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -270,6 +306,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
|
|||
memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN);
|
||||
TSWAP(pOld->readDbs, pNew->readDbs);
|
||||
TSWAP(pOld->writeDbs, pNew->writeDbs);
|
||||
TSWAP(pOld->topics, pNew->topics);
|
||||
taosWUnLockLatch(&pOld->lock);
|
||||
|
||||
return 0;
|
||||
|
|
@ -482,9 +519,10 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
|
|||
taosRLockLatch(&pUser->lock);
|
||||
newUser.readDbs = mndDupDbHash(pUser->readDbs);
|
||||
newUser.writeDbs = mndDupDbHash(pUser->writeDbs);
|
||||
newUser.topics = mndDupDbHash(pUser->topics);
|
||||
taosRUnLockLatch(&pUser->lock);
|
||||
|
||||
if (newUser.readDbs == NULL || newUser.writeDbs == NULL) {
|
||||
if (newUser.readDbs == NULL || newUser.writeDbs == NULL || newUser.topics == NULL) {
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
|
|
@ -507,14 +545,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
|
||||
if (strcmp(alterReq.dbname, "1.*") != 0) {
|
||||
int32_t len = strlen(alterReq.dbname) + 1;
|
||||
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
|
||||
if (strcmp(alterReq.objname, "1.*") != 0) {
|
||||
int32_t len = strlen(alterReq.objname) + 1;
|
||||
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
|
||||
if (pDb == NULL) {
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
goto _OVER;
|
||||
}
|
||||
if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
|
||||
if (taosHashPut(newUser.readDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
goto _OVER;
|
||||
}
|
||||
|
|
@ -531,14 +569,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
|
||||
if (strcmp(alterReq.dbname, "1.*") != 0) {
|
||||
int32_t len = strlen(alterReq.dbname) + 1;
|
||||
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
|
||||
if (strcmp(alterReq.objname, "1.*") != 0) {
|
||||
int32_t len = strlen(alterReq.objname) + 1;
|
||||
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
|
||||
if (pDb == NULL) {
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
goto _OVER;
|
||||
}
|
||||
if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
|
||||
if (taosHashPut(newUser.writeDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
goto _OVER;
|
||||
}
|
||||
|
|
@ -555,33 +593,53 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
|
||||
if (strcmp(alterReq.dbname, "1.*") != 0) {
|
||||
int32_t len = strlen(alterReq.dbname) + 1;
|
||||
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
|
||||
if (strcmp(alterReq.objname, "1.*") != 0) {
|
||||
int32_t len = strlen(alterReq.objname) + 1;
|
||||
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
|
||||
if (pDb == NULL) {
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
goto _OVER;
|
||||
}
|
||||
taosHashRemove(newUser.readDbs, alterReq.dbname, len);
|
||||
taosHashRemove(newUser.readDbs, alterReq.objname, len);
|
||||
} else {
|
||||
taosHashClear(newUser.readDbs);
|
||||
}
|
||||
}
|
||||
|
||||
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
|
||||
if (strcmp(alterReq.dbname, "1.*") != 0) {
|
||||
int32_t len = strlen(alterReq.dbname) + 1;
|
||||
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.dbname);
|
||||
if (strcmp(alterReq.objname, "1.*") != 0) {
|
||||
int32_t len = strlen(alterReq.objname) + 1;
|
||||
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
|
||||
if (pDb == NULL) {
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
goto _OVER;
|
||||
}
|
||||
taosHashRemove(newUser.writeDbs, alterReq.dbname, len);
|
||||
taosHashRemove(newUser.writeDbs, alterReq.objname, len);
|
||||
} else {
|
||||
taosHashClear(newUser.writeDbs);
|
||||
}
|
||||
}
|
||||
|
||||
if (alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) {
|
||||
int32_t len = strlen(alterReq.objname) + 1;
|
||||
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
|
||||
if (pTopic == NULL) {
|
||||
mndReleaseTopic(pMnode, pTopic);
|
||||
goto _OVER;
|
||||
}
|
||||
taosHashPut(newUser.topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
|
||||
}
|
||||
|
||||
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC) {
|
||||
int32_t len = strlen(alterReq.objname) + 1;
|
||||
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
|
||||
if (pTopic == NULL) {
|
||||
mndReleaseTopic(pMnode, pTopic);
|
||||
goto _OVER;
|
||||
}
|
||||
taosHashRemove(newUser.topics, alterReq.objname, len);
|
||||
}
|
||||
|
||||
code = mndAlterUser(pMnode, pUser, &newUser, pReq);
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
|
|
@ -594,6 +652,7 @@ _OVER:
|
|||
mndReleaseUser(pMnode, pUser);
|
||||
taosHashCleanup(newUser.writeDbs);
|
||||
taosHashCleanup(newUser.readDbs);
|
||||
taosHashCleanup(newUser.topics);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -756,6 +815,108 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) {
|
|||
sdbCancelFetch(pSdb, pIter);
|
||||
}
|
||||
|
||||
static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
int32_t numOfRows = 0;
|
||||
SUserObj *pUser = NULL;
|
||||
int32_t cols = 0;
|
||||
char *pWrite;
|
||||
|
||||
while (numOfRows < rows) {
|
||||
pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser);
|
||||
if (pShow->pIter == NULL) break;
|
||||
|
||||
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
|
||||
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
|
||||
int32_t numOfTopics = taosHashGetSize(pUser->topics);
|
||||
if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics >= rows) break;
|
||||
|
||||
char *db = taosHashIterate(pUser->readDbs, NULL);
|
||||
while (db != NULL) {
|
||||
cols = 0;
|
||||
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
|
||||
|
||||
char privilege[20] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(privilege, "read", pShow->pMeta->pSchemas[cols].bytes);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
|
||||
|
||||
SName name = {0};
|
||||
char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
tNameFromString(&name, db, T_NAME_ACCT | T_NAME_DB);
|
||||
tNameGetDbName(&name, varDataVal(objName));
|
||||
varDataSetLen(objName, strlen(varDataVal(objName)));
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
|
||||
|
||||
numOfRows++;
|
||||
db = taosHashIterate(pUser->readDbs, db);
|
||||
}
|
||||
|
||||
db = taosHashIterate(pUser->writeDbs, NULL);
|
||||
while (db != NULL) {
|
||||
cols = 0;
|
||||
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
|
||||
|
||||
char privilege[20] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(privilege, "write", pShow->pMeta->pSchemas[cols].bytes);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
|
||||
|
||||
SName name = {0};
|
||||
char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
tNameFromString(&name, db, T_NAME_ACCT | T_NAME_DB);
|
||||
tNameGetDbName(&name, varDataVal(objName));
|
||||
varDataSetLen(objName, strlen(varDataVal(objName)));
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)objName, false);
|
||||
|
||||
numOfRows++;
|
||||
db = taosHashIterate(pUser->writeDbs, db);
|
||||
}
|
||||
|
||||
char *topic = taosHashIterate(pUser->topics, NULL);
|
||||
while (topic != NULL) {
|
||||
cols = 0;
|
||||
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
|
||||
|
||||
char privilege[20] = {0};
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(privilege, "subscribe", pShow->pMeta->pSchemas[cols].bytes);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)privilege, false);
|
||||
|
||||
char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0};
|
||||
tstrncpy(varDataVal(topicName), mndGetDbStr(topic), TSDB_TOPIC_NAME_LEN - 2);
|
||||
varDataSetLen(topicName, strlen(varDataVal(topicName)));
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
|
||||
|
||||
numOfRows++;
|
||||
db = taosHashIterate(pUser->topics, topic);
|
||||
}
|
||||
|
||||
sdbRelease(pSdb, pUser);
|
||||
}
|
||||
|
||||
pShow->numOfRows += numOfRows;
|
||||
return numOfRows;
|
||||
}
|
||||
|
||||
static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter) {
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
}
|
||||
|
||||
int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp,
|
||||
int32_t *pRspLen) {
|
||||
SUserAuthBatchRsp batchRsp = {0};
|
||||
|
|
|
|||
|
|
@ -113,6 +113,8 @@ _OVER:
|
|||
|
||||
SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
SSdbRow *pRow = NULL;
|
||||
SVgObj *pVgroup = NULL;
|
||||
|
||||
int8_t sver = 0;
|
||||
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
|
||||
|
|
@ -122,10 +124,10 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj));
|
||||
pRow = sdbAllocRow(sizeof(SVgObj));
|
||||
if (pRow == NULL) goto _OVER;
|
||||
|
||||
SVgObj *pVgroup = sdbGetRowObj(pRow);
|
||||
pVgroup = sdbGetRowObj(pRow);
|
||||
if (pVgroup == NULL) goto _OVER;
|
||||
|
||||
int32_t dataPos = 0;
|
||||
|
|
@ -152,7 +154,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr());
|
||||
mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup == NULL ? 0 : pVgroup->vgId, pRaw, terrstr());
|
||||
taosMemoryFreeClear(pRow);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -179,6 +181,23 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) {
|
|||
pOld->hashEnd = pNew->hashEnd;
|
||||
pOld->replica = pNew->replica;
|
||||
pOld->isTsma = pNew->isTsma;
|
||||
for (int32_t i = 0; i < pNew->replica; ++i) {
|
||||
SVnodeGid *pNewGid = &pNew->vnodeGid[i];
|
||||
for (int32_t j = 0; j < pOld->replica; ++j) {
|
||||
SVnodeGid *pOldGid = &pOld->vnodeGid[j];
|
||||
if (pNewGid->dnodeId == pOldGid->dnodeId) {
|
||||
pNewGid->syncState = pOldGid->syncState;
|
||||
pNewGid->syncRestore = pOldGid->syncRestore;
|
||||
pNewGid->syncCanRead = pOldGid->syncCanRead;
|
||||
}
|
||||
}
|
||||
}
|
||||
pNew->numOfTables = pOld->numOfTables;
|
||||
pNew->numOfTimeSeries = pOld->numOfTimeSeries;
|
||||
pNew->totalStorage = pOld->totalStorage;
|
||||
pNew->compStorage = pOld->compStorage;
|
||||
pNew->pointsWritten = pOld->pointsWritten;
|
||||
pNew->compact = pOld->compact;
|
||||
memcpy(pOld->vnodeGid, pNew->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -659,11 +678,12 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
|
|||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->numOfTables, false);
|
||||
|
||||
// default 3 replica
|
||||
for (int32_t i = 0; i < 3; ++i) {
|
||||
// default 3 replica, add 1 replica if move vnode
|
||||
for (int32_t i = 0; i < 4; ++i) {
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
if (i < pVgroup->replica) {
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false);
|
||||
int16_t dnodeId = (int16_t)pVgroup->vnodeGid[i].dnodeId;
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)&dnodeId, false);
|
||||
|
||||
bool exist = false;
|
||||
bool online = false;
|
||||
|
|
@ -679,8 +699,16 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
|
|||
if (!exist) {
|
||||
strcpy(role, "dropping");
|
||||
} else if (online) {
|
||||
bool show = (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER && !pVgroup->vnodeGid[i].syncRestore);
|
||||
snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), show ? "*" : "");
|
||||
char *star = "";
|
||||
if (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER) {
|
||||
if (!pVgroup->vnodeGid[i].syncRestore && !pVgroup->vnodeGid[i].syncCanRead) {
|
||||
star = "**";
|
||||
} else if (!pVgroup->vnodeGid[i].syncRestore && pVgroup->vnodeGid[i].syncCanRead) {
|
||||
star = "*";
|
||||
} else {
|
||||
}
|
||||
}
|
||||
snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), star);
|
||||
} else {
|
||||
}
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes);
|
||||
|
|
@ -695,16 +723,8 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
|
|||
}
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppendNULL(pColInfo, numOfRows);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->cacheUsage, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppendNULL(pColInfo, numOfRows);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppendNULL(pColInfo, numOfRows);
|
||||
int32_t cacheUsage = (int32_t)pVgroup->cacheUsage;
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)&cacheUsage, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false);
|
||||
|
|
@ -851,7 +871,7 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) {
|
|||
sdbCancelFetch(pSdb, pIter);
|
||||
}
|
||||
|
||||
int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
|
||||
static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray) {
|
||||
taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
|
||||
SDnodeObj *pDnode = taosArrayGet(pArray, i);
|
||||
|
|
@ -887,12 +907,21 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
|
|||
}
|
||||
|
||||
pVgid->dnodeId = pDnode->id;
|
||||
pVgid->syncState = TAOS_SYNC_STATE_ERROR;
|
||||
pVgid->syncState = TAOS_SYNC_STATE_OFFLINE;
|
||||
mInfo("db:%s, vgId:%d, vn:%d is added, memory:%" PRId64 ", dnode:%d avail:%" PRId64 " used:%" PRId64,
|
||||
pVgroup->dbName, pVgroup->vgId, pVgroup->replica, vgMem, pVgid->dnodeId, pDnode->memAvail, pDnode->memUsed);
|
||||
|
||||
pVgroup->replica++;
|
||||
pDnode->numOfVnodes++;
|
||||
|
||||
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
|
||||
if (pVgRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
|
||||
sdbFreeRaw(pVgRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -901,7 +930,8 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) {
|
||||
static int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray,
|
||||
SVnodeGid *pDelVgid) {
|
||||
taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
|
||||
SDnodeObj *pDnode = taosArrayGet(pArray, i);
|
||||
|
|
@ -941,6 +971,15 @@ _OVER:
|
|||
SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
|
||||
mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId);
|
||||
}
|
||||
|
||||
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
|
||||
if (pVgRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
|
||||
sdbFreeRaw(pVgRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1088,7 +1127,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
|
|||
|
||||
if (!force) {
|
||||
mInfo("vgId:%d, will add 1 vnode", pVgroup->vgId);
|
||||
if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
|
||||
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;
|
||||
for (int32_t i = 0; i < newVg.replica - 1; ++i) {
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
|
||||
}
|
||||
|
|
@ -1100,6 +1139,16 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
|
|||
SVnodeGid del = newVg.vnodeGid[vnIndex];
|
||||
newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
|
||||
memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
|
||||
{
|
||||
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
|
||||
if (pRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pRaw) != 0) {
|
||||
sdbFreeRaw(pRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
}
|
||||
|
||||
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
|
||||
for (int32_t i = 0; i < newVg.replica; ++i) {
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
|
||||
|
|
@ -1107,11 +1156,20 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
|
|||
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
|
||||
} else {
|
||||
mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId);
|
||||
if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
|
||||
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;
|
||||
newVg.replica--;
|
||||
SVnodeGid del = newVg.vnodeGid[vnIndex];
|
||||
newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
|
||||
memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
|
||||
{
|
||||
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
|
||||
if (pRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pRaw) != 0) {
|
||||
sdbFreeRaw(pRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < newVg.replica; ++i) {
|
||||
if (i != vnIndex) {
|
||||
|
|
@ -1128,16 +1186,12 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
|
|||
|
||||
{
|
||||
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
|
||||
if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) return -1;
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
pRaw = NULL;
|
||||
if (pRaw == NULL) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
|
||||
sdbFreeRaw(pRaw);
|
||||
return -1;
|
||||
}
|
||||
|
||||
{
|
||||
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
|
||||
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1;
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
pRaw = NULL;
|
||||
}
|
||||
|
||||
mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica);
|
||||
|
|
@ -1193,7 +1247,15 @@ static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDb
|
|||
SVnodeGid *pGid = &pVgroup->vnodeGid[pVgroup->replica];
|
||||
pVgroup->replica++;
|
||||
pGid->dnodeId = newDnodeId;
|
||||
pGid->syncState = TAOS_SYNC_STATE_ERROR;
|
||||
pGid->syncState = TAOS_SYNC_STATE_OFFLINE;
|
||||
|
||||
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
|
||||
if (pVgRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
|
||||
sdbFreeRaw(pVgRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
|
||||
|
||||
for (int32_t i = 0; i < pVgroup->replica - 1; ++i) {
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1;
|
||||
|
|
@ -1224,6 +1286,14 @@ static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, S
|
|||
memcpy(pGid, &pVgroup->vnodeGid[pVgroup->replica], sizeof(SVnodeGid));
|
||||
memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid));
|
||||
|
||||
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
|
||||
if (pVgRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
|
||||
sdbFreeRaw(pVgRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
|
||||
|
||||
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1;
|
||||
for (int32_t i = 0; i < pVgroup->replica; ++i) {
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1;
|
||||
|
|
@ -1237,7 +1307,6 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
|
|||
SDnodeObj *pOld1, SDnodeObj *pNew2, SDnodeObj *pOld2, SDnodeObj *pNew3,
|
||||
SDnodeObj *pOld3) {
|
||||
int32_t code = -1;
|
||||
SSdbRaw *pRaw = NULL;
|
||||
STrans *pTrans = NULL;
|
||||
|
||||
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "red-vgroup");
|
||||
|
|
@ -1319,17 +1388,13 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
|
|||
}
|
||||
|
||||
{
|
||||
pRaw = mndVgroupActionEncode(&newVg);
|
||||
if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER;
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
pRaw = NULL;
|
||||
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
|
||||
if (pRaw == NULL) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
|
||||
sdbFreeRaw(pRaw);
|
||||
return -1;
|
||||
}
|
||||
|
||||
{
|
||||
pRaw = mndVgroupActionEncode(&newVg);
|
||||
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
pRaw = NULL;
|
||||
}
|
||||
|
||||
mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica);
|
||||
|
|
@ -1342,7 +1407,6 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
|
|||
|
||||
_OVER:
|
||||
mndTransDrop(pTrans);
|
||||
sdbFreeRaw(pRaw);
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
return code;
|
||||
}
|
||||
|
|
@ -1593,13 +1657,13 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
|
|||
mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
|
||||
pVgroup->vnodeGid[0].dnodeId);
|
||||
|
||||
if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
|
||||
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1;
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
|
||||
return -1;
|
||||
if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1;
|
||||
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
|
||||
|
||||
if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1;
|
||||
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1;
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
|
||||
return -1;
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0)
|
||||
|
|
@ -1612,7 +1676,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
|
|||
|
||||
SVnodeGid del1 = {0};
|
||||
SVnodeGid del2 = {0};
|
||||
if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1;
|
||||
if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del1) != 0) return -1;
|
||||
if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true) != 0) return -1;
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
|
||||
return -1;
|
||||
|
|
@ -1620,7 +1684,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
|
|||
return -1;
|
||||
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1;
|
||||
|
||||
if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1;
|
||||
if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del2) != 0) return -1;
|
||||
if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true) != 0) return -1;
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0)
|
||||
return -1;
|
||||
|
|
@ -1629,16 +1693,6 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
|
|||
return -1;
|
||||
}
|
||||
|
||||
{
|
||||
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
|
||||
if (pVgRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
|
||||
sdbFreeRaw(pVgRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
|
||||
}
|
||||
|
||||
{
|
||||
SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
|
||||
if (pVgRaw == NULL) return -1;
|
||||
|
|
@ -1658,7 +1712,6 @@ static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans,
|
|||
|
||||
static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) {
|
||||
int32_t code = -1;
|
||||
SSdbRaw *pRaw = NULL;
|
||||
STrans *pTrans = NULL;
|
||||
SArray *pArray = mndBuildDnodesArray(pMnode, 0);
|
||||
|
||||
|
|
@ -1676,13 +1729,13 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
|
|||
}
|
||||
|
||||
if (newVg1.replica == 1) {
|
||||
if (mndAddVnodeToVgroup(pMnode, &newVg1, pArray) != 0) goto _OVER;
|
||||
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray) != 0) goto _OVER;
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
|
||||
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER;
|
||||
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
|
||||
} else if (newVg1.replica == 3) {
|
||||
SVnodeGid del1 = {0};
|
||||
if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER;
|
||||
if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1) != 0) goto _OVER;
|
||||
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER;
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER;
|
||||
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER;
|
||||
|
|
@ -1727,17 +1780,23 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
|
|||
#endif
|
||||
|
||||
{
|
||||
pRaw = mndVgroupActionEncode(&newVg1);
|
||||
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
|
||||
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg1);
|
||||
if (pRaw == NULL) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
|
||||
sdbFreeRaw(pRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
pRaw = NULL;
|
||||
}
|
||||
|
||||
{
|
||||
pRaw = mndVgroupActionEncode(&newVg2);
|
||||
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
|
||||
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg2);
|
||||
if (pRaw == NULL) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
|
||||
sdbFreeRaw(pRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
pRaw = NULL;
|
||||
}
|
||||
|
||||
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
|
||||
|
|
@ -1757,7 +1816,6 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
|
|||
_OVER:
|
||||
taosArrayDestroy(pArray);
|
||||
mndTransDrop(pTrans);
|
||||
sdbFreeRaw(pRaw);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -1802,16 +1860,8 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD
|
|||
|
||||
{
|
||||
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
|
||||
if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) {
|
||||
sdbFreeRaw(pRaw);
|
||||
return -1;
|
||||
}
|
||||
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
|
||||
}
|
||||
|
||||
{
|
||||
SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
|
||||
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) {
|
||||
if (pRaw == NULL) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
|
||||
sdbFreeRaw(pRaw);
|
||||
return -1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
|
|||
|
||||
int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) {
|
||||
SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg;
|
||||
streamMetaRemoveTask1(pSnode->pMeta, pReq->taskId);
|
||||
streamMetaRemoveTask(pSnode->pMeta, pReq->taskId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ bool vnodeShouldRollback(SVnode* pVnode);
|
|||
|
||||
// vnodeSync.c
|
||||
int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
|
||||
void vnodeSyncStart(SVnode* pVnode);
|
||||
int32_t vnodeSyncStart(SVnode* pVnode);
|
||||
void vnodeSyncPreClose(SVnode* pVnode);
|
||||
void vnodeSyncClose(SVnode* pVnode);
|
||||
void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg);
|
||||
|
|
|
|||
|
|
@ -1425,7 +1425,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
|
|||
|
||||
int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
|
||||
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
|
||||
streamMetaRemoveTask1(pTq->pStreamMeta, pReq->taskId);
|
||||
streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -811,7 +811,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
|
|||
int32_t lino = 0;
|
||||
|
||||
memset(pCommitter, 0, sizeof(*pCommitter));
|
||||
ASSERT(pTsdb->mem && pTsdb->imem == NULL);
|
||||
ASSERT(pTsdb->mem && pTsdb->imem == NULL && "last tsdb commit incomplete");
|
||||
|
||||
taosThreadRwlockWrlock(&pTsdb->rwLock);
|
||||
pTsdb->imem = pTsdb->mem;
|
||||
|
|
@ -1261,7 +1261,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
|
|||
}
|
||||
}
|
||||
} else {
|
||||
ASSERT(0);
|
||||
ASSERT(0 && "dup rows not allowed");
|
||||
}
|
||||
|
||||
if (pBDataW->nRow >= pCommitter->maxRow) {
|
||||
|
|
|
|||
|
|
@ -775,7 +775,10 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_
|
|||
} else {
|
||||
varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData);
|
||||
ASSERT(pColVal->value.nData <= pColInfoData->info.bytes);
|
||||
if (pColVal->value.nData > 0) { // pData may be null, if nData is 0
|
||||
memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData);
|
||||
}
|
||||
|
||||
colDataAppend(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false);
|
||||
}
|
||||
} else {
|
||||
|
|
@ -2540,6 +2543,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
|
|||
goto _err;
|
||||
}
|
||||
|
||||
// TODO: opt the perf of read del index
|
||||
code = tsdbReadDelIdx(pDelFReader, aDelIdx);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
taosArrayDestroy(aDelIdx);
|
||||
|
|
|
|||
|
|
@ -728,7 +728,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
|
|||
taosArraySet(pMerger->pArray, iCol, pColVal);
|
||||
}
|
||||
} else {
|
||||
ASSERT(0);
|
||||
ASSERT(0 && "dup versions not allowed");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -902,7 +902,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
|
|||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto _clear;
|
||||
}
|
||||
|
||||
midx = (sidx + eidx) / 2;
|
||||
|
||||
code = tsdbBuildDeleteSkyline(aDelData, sidx, midx, aSkyline1);
|
||||
|
|
|
|||
|
|
@ -176,6 +176,9 @@ void vnodeBufPoolRef(SVBufPool *pPool) {
|
|||
}
|
||||
|
||||
void vnodeBufPoolUnRef(SVBufPool *pPool) {
|
||||
if (pPool == NULL) {
|
||||
return;
|
||||
}
|
||||
int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1);
|
||||
if (nRef == 0) {
|
||||
SVnode *pVnode = pPool->pVnode;
|
||||
|
|
|
|||
|
|
@ -209,8 +209,8 @@ int vnodeCommit(SVnode *pVnode) {
|
|||
SVnodeInfo info = {0};
|
||||
char dir[TSDB_FILENAME_LEN];
|
||||
|
||||
vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID,
|
||||
pVnode->state.applied);
|
||||
vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode),
|
||||
pVnode->state.commitID, pVnode->state.applied, pVnode->state.applyTerm);
|
||||
|
||||
// persist wal before starting
|
||||
if (walPersist(pVnode->pWal) < 0) {
|
||||
|
|
|
|||
|
|
@ -144,9 +144,9 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
|
|||
pVnode->config = info.config;
|
||||
pVnode->state.committed = info.state.committed;
|
||||
pVnode->state.commitTerm = info.state.commitTerm;
|
||||
pVnode->state.applied = info.state.committed;
|
||||
pVnode->state.commitID = info.state.commitID;
|
||||
pVnode->state.commitTerm = info.state.commitTerm;
|
||||
pVnode->state.applied = info.state.committed;
|
||||
pVnode->state.applyTerm = info.state.commitTerm;
|
||||
pVnode->pTfs = pTfs;
|
||||
pVnode->msgCb = msgCb;
|
||||
taosThreadMutexInit(&pVnode->lock, NULL);
|
||||
|
|
@ -269,10 +269,7 @@ void vnodeClose(SVnode *pVnode) {
|
|||
}
|
||||
|
||||
// start the sync timer after the queue is ready
|
||||
int32_t vnodeStart(SVnode *pVnode) {
|
||||
vnodeSyncStart(pVnode);
|
||||
return 0;
|
||||
}
|
||||
int32_t vnodeStart(SVnode *pVnode) { return vnodeSyncStart(pVnode); }
|
||||
|
||||
void vnodeStop(SVnode *pVnode) {}
|
||||
|
||||
|
|
|
|||
|
|
@ -380,6 +380,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
|
|||
pLoad->vgId = TD_VID(pVnode);
|
||||
pLoad->syncState = state.state;
|
||||
pLoad->syncRestore = state.restored;
|
||||
pLoad->syncCanRead = state.canRead;
|
||||
pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
|
||||
pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
|
||||
pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);
|
||||
|
|
|
|||
|
|
@ -178,8 +178,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
|
|||
return -1;
|
||||
}
|
||||
|
||||
if (version <= pVnode->state.applied) {
|
||||
vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version,
|
||||
pVnode->state.applied);
|
||||
pRsp->info.handle = NULL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
|
||||
version);
|
||||
ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm);
|
||||
|
||||
pVnode->state.applied = version;
|
||||
pVnode->state.applyTerm = pMsg->info.conn.applyTerm;
|
||||
|
|
|
|||
|
|
@ -295,36 +295,31 @@ static int32_t vnodeSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
SVnode *pVnode = pFsm->data;
|
||||
|
||||
SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen};
|
||||
rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
|
||||
memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
|
||||
rpcMsg.info = pMsg->info;
|
||||
rpcMsg.info.conn.applyIndex = pMeta->index;
|
||||
rpcMsg.info.conn.applyTerm = pMeta->term;
|
||||
pMsg->info.conn.applyIndex = pMeta->index;
|
||||
pMsg->info.conn.applyTerm = pMeta->term;
|
||||
|
||||
const STraceId *trace = &pMsg->info.traceId;
|
||||
vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64
|
||||
", weak:%d, code:%d, state:%d %s, type:%s",
|
||||
pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, pMeta->code,
|
||||
pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, pMsg->info.conn.applyIndex, pMeta->isWeak, pMeta->code,
|
||||
pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType));
|
||||
|
||||
tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
|
||||
tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg);
|
||||
}
|
||||
|
||||
static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
vnodeSyncApplyMsg(pFsm, pMsg, pMeta);
|
||||
}
|
||||
|
||||
static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
if (pMeta->isWeak == 1) {
|
||||
vnodeSyncApplyMsg(pFsm, pMsg, pMeta);
|
||||
}
|
||||
}
|
||||
|
||||
static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
|
||||
SVnode *pVnode = pFsm->data;
|
||||
vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s",
|
||||
pVnode->config.vgId, pFsm, pMeta->index, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state),
|
||||
|
|
@ -404,7 +399,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm) {
|
|||
walApplyVer(pVnode->pWal, pVnode->state.applied);
|
||||
|
||||
pVnode->restored = true;
|
||||
vDebug("vgId:%d, sync restore finished", pVnode->config.vgId);
|
||||
vInfo("vgId:%d, sync restore finished", pVnode->config.vgId);
|
||||
}
|
||||
|
||||
static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
|
||||
|
|
@ -506,15 +501,29 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
void vnodeSyncStart(SVnode *pVnode) {
|
||||
int32_t vnodeSyncStart(SVnode *pVnode) {
|
||||
vInfo("vgId:%d, start sync", pVnode->config.vgId);
|
||||
syncStart(pVnode->sync);
|
||||
if (syncStart(pVnode->sync) < 0) {
|
||||
vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, terrstr());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void vnodeSyncPreClose(SVnode *pVnode) {
|
||||
vInfo("vgId:%d, pre close sync", pVnode->config.vgId);
|
||||
syncLeaderTransfer(pVnode->sync);
|
||||
syncPreStop(pVnode->sync);
|
||||
#if 0
|
||||
while (syncSnapshotRecving(pVnode->sync)) {
|
||||
vInfo("vgId:%d, snapshot is recving", pVnode->config.vgId);
|
||||
taosMsleep(300);
|
||||
}
|
||||
while (syncSnapshotSending(pVnode->sync)) {
|
||||
vInfo("vgId:%d, snapshot is sending", pVnode->config.vgId);
|
||||
taosMsleep(300);
|
||||
}
|
||||
#endif
|
||||
taosThreadMutexLock(&pVnode->lock);
|
||||
if (pVnode->blocked) {
|
||||
vInfo("vgId:%d, post block after close sync", pVnode->config.vgId);
|
||||
|
|
@ -543,12 +552,12 @@ bool vnodeIsLeader(SVnode *pVnode) {
|
|||
} else {
|
||||
terrno = TSDB_CODE_APP_NOT_READY;
|
||||
}
|
||||
vDebug("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored);
|
||||
vInfo("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!pVnode->restored) {
|
||||
vDebug("vgId:%d, vnode not restored", pVnode->config.vgId);
|
||||
vInfo("vgId:%d, vnode not restored", pVnode->config.vgId);
|
||||
terrno = TSDB_CODE_APP_NOT_READY;
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -161,4 +161,6 @@ int32_t convertFillType(int32_t mode);
|
|||
int32_t resultrowComparAsc(const void* p1, const void* p2);
|
||||
int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified);
|
||||
|
||||
void printDataBlock(SSDataBlock* pBlock, const char* flag);
|
||||
|
||||
#endif // TDENGINE_QUERYUTIL_H
|
||||
|
|
|
|||
|
|
@ -332,6 +332,7 @@ typedef struct STableScanInfo {
|
|||
int32_t currentTable;
|
||||
int8_t scanMode;
|
||||
int8_t assignBlockUid;
|
||||
bool hasGroupByTag;
|
||||
} STableScanInfo;
|
||||
|
||||
typedef struct STableMergeScanInfo {
|
||||
|
|
@ -535,25 +536,9 @@ typedef struct SStreamIntervalOperatorInfo {
|
|||
SArray* pChildren;
|
||||
SStreamState* pState;
|
||||
SWinKey delKey;
|
||||
uint64_t numOfDatapack;
|
||||
} SStreamIntervalOperatorInfo;
|
||||
|
||||
typedef struct SFillOperatorInfo {
|
||||
struct SFillInfo* pFillInfo;
|
||||
SSDataBlock* pRes;
|
||||
SSDataBlock* pFinalRes;
|
||||
int64_t totalInputRows;
|
||||
void** p;
|
||||
SSDataBlock* existNewGroupBlock;
|
||||
STimeWindow win;
|
||||
SColMatchInfo matchInfo;
|
||||
int32_t primaryTsCol;
|
||||
int32_t primarySrcSlotId;
|
||||
uint64_t curGroupId; // current handled group id
|
||||
SExprInfo* pExprInfo;
|
||||
int32_t numOfExpr;
|
||||
SExprSupp noFillExprSupp;
|
||||
} SFillOperatorInfo;
|
||||
|
||||
typedef struct SDataGroupInfo {
|
||||
uint64_t groupId;
|
||||
int64_t numOfRows;
|
||||
|
|
@ -796,7 +781,7 @@ void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order,
|
|||
int32_t checkForQueryBuf(size_t numOfTables);
|
||||
|
||||
bool isTaskKilled(SExecTaskInfo* pTaskInfo);
|
||||
void setTaskKilled(SExecTaskInfo* pTaskInfo);
|
||||
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode);
|
||||
void doDestroyTask(SExecTaskInfo* pTaskInfo);
|
||||
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
|
||||
|
||||
|
|
@ -805,8 +790,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
|
|||
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
|
||||
int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
|
||||
|
||||
void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo);
|
||||
|
||||
int32_t getMaximumIdleDurationSec();
|
||||
|
||||
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
|
||||
|
|
@ -824,7 +807,6 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup);
|
|||
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup);
|
||||
void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
|
||||
uint64_t* pGp, void* pTbName);
|
||||
void printDataBlock(SSDataBlock* pBlock, const char* flag);
|
||||
uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
|
||||
void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock);
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ extern "C" {
|
|||
#include "tcommon.h"
|
||||
#include "tsimplehash.h"
|
||||
|
||||
#define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId)
|
||||
|
||||
struct SSDataBlock;
|
||||
|
||||
typedef struct SFillColInfo {
|
||||
|
|
@ -116,7 +118,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t
|
|||
void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
|
||||
void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp);
|
||||
void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
|
||||
struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr,
|
||||
SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr,
|
||||
int32_t numOfNotFillCols, const struct SNodeListNode* val);
|
||||
bool taosFillHasMoreResults(struct SFillInfo* pFillInfo);
|
||||
|
||||
|
|
@ -128,6 +130,8 @@ void* taosDestroyFillInfo(struct SFillInfo* pFillInfo);
|
|||
int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity);
|
||||
int64_t getFillInfoStart(struct SFillInfo* pFillInfo);
|
||||
|
||||
bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData,
|
||||
int32_t rowIndex);
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -15,26 +15,15 @@
|
|||
|
||||
#include "filter.h"
|
||||
#include "function.h"
|
||||
#include "functionMgt.h"
|
||||
#include "os.h"
|
||||
#include "querynodes.h"
|
||||
#include "tfill.h"
|
||||
#include "tname.h"
|
||||
#include "tref.h"
|
||||
|
||||
#include "tdatablock.h"
|
||||
#include "tglobal.h"
|
||||
#include "tmsg.h"
|
||||
#include "tsort.h"
|
||||
#include "ttime.h"
|
||||
|
||||
#include "executorimpl.h"
|
||||
#include "index.h"
|
||||
#include "query.h"
|
||||
#include "tcompare.h"
|
||||
#include "thash.h"
|
||||
#include "ttypes.h"
|
||||
#include "vnode.h"
|
||||
|
||||
typedef struct SFetchRspHandleWrapper {
|
||||
uint32_t exchangeId;
|
||||
|
|
@ -81,7 +70,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn
|
|||
tsem_wait(&pExchangeInfo->ready);
|
||||
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
|
||||
longjmp(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < totalSources; ++i) {
|
||||
|
|
@ -584,7 +573,7 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) {
|
|||
|
||||
tsem_wait(&pExchangeInfo->ready);
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
|
||||
longjmp(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
tsem_post(&pExchangeInfo->ready);
|
||||
|
|
@ -632,7 +621,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) {
|
|||
doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current);
|
||||
tsem_wait(&pExchangeInfo->ready);
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
|
||||
longjmp(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
|
||||
|
|
|
|||
|
|
@ -2002,3 +2002,13 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
|
|||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void printDataBlock(SSDataBlock* pBlock, const char* flag) {
|
||||
if (!pBlock || pBlock->info.rows == 0) {
|
||||
qDebug("===stream===printDataBlock: Block is Null or Empty");
|
||||
return;
|
||||
}
|
||||
char* pBuf = NULL;
|
||||
qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
|
||||
taosMemoryFree(pBuf);
|
||||
}
|
||||
|
|
@ -688,7 +688,7 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) {
|
|||
taosWUnLockLatch(&pTaskInfo->stopInfo.lock);
|
||||
}
|
||||
|
||||
int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
|
||||
int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
|
||||
|
||||
if (pTaskInfo == NULL) {
|
||||
|
|
@ -697,13 +697,27 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
|
|||
|
||||
qDebug("%s execTask async killed", GET_TASKID(pTaskInfo));
|
||||
|
||||
setTaskKilled(pTaskInfo);
|
||||
setTaskKilled(pTaskInfo, rspCode);
|
||||
|
||||
qStopTaskOperators(pTaskInfo);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) {
|
||||
STaskCostInfo* pSummary = &pTaskInfo->cost;
|
||||
|
||||
SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder;
|
||||
if (pSummary->pRecoder != NULL) {
|
||||
qDebug(
|
||||
"%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, "
|
||||
"load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64,
|
||||
GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime,
|
||||
pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows,
|
||||
pRecorder->totalCheckedRows);
|
||||
}
|
||||
}
|
||||
|
||||
void qDestroyTask(qTaskInfo_t qTaskHandle) {
|
||||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle;
|
||||
if (pTaskInfo == NULL) {
|
||||
|
|
|
|||
|
|
@ -91,7 +91,6 @@ static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock*
|
|||
|
||||
static void releaseQueryBuf(size_t numOfTables);
|
||||
|
||||
static void destroyFillOperatorInfo(void* param);
|
||||
static void destroyAggOperatorInfo(void* param);
|
||||
static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
|
||||
static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId);
|
||||
|
|
@ -611,21 +610,10 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
|
|||
}
|
||||
|
||||
bool isTaskKilled(SExecTaskInfo* pTaskInfo) {
|
||||
// query has been executed more than tsShellActivityTimer, and the retrieve has not arrived
|
||||
// abort current query execution.
|
||||
if (pTaskInfo->owner != 0 &&
|
||||
((taosGetTimestampSec() - pTaskInfo->cost.start / 1000) > 10 * getMaximumIdleDurationSec())
|
||||
/*(!needBuildResAfterQueryComplete(pTaskInfo))*/) {
|
||||
assert(pTaskInfo->cost.start != 0);
|
||||
// qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64
|
||||
// ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec());
|
||||
// return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
return (0 != pTaskInfo->code) ? true : false;
|
||||
}
|
||||
|
||||
void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED; }
|
||||
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; }
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) {
|
||||
|
|
@ -1157,20 +1145,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
|
|||
}
|
||||
}
|
||||
|
||||
void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) {
|
||||
STaskCostInfo* pSummary = &pTaskInfo->cost;
|
||||
|
||||
SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder;
|
||||
if (pSummary->pRecoder != NULL) {
|
||||
qDebug(
|
||||
"%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, "
|
||||
"load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64,
|
||||
GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime,
|
||||
pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows,
|
||||
pRecorder->totalCheckedRows);
|
||||
}
|
||||
}
|
||||
|
||||
// void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) {
|
||||
// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
|
||||
//
|
||||
|
|
@ -1419,6 +1393,78 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
|
|||
}
|
||||
}
|
||||
|
||||
static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock **ppBlock) {
|
||||
if (!tsCountAlwaysReturnValue) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SOperatorInfo* downstream = pOperator->pDownstream[0];
|
||||
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION ||
|
||||
(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN &&
|
||||
((STableScanInfo *)downstream->info)->hasGroupByTag == true)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
|
||||
bool hasCountFunc = false;
|
||||
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
|
||||
if ((strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "count") == 0) ||
|
||||
(strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0) ||
|
||||
(strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_partial") == 0) ||
|
||||
(strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_merge") == 0)) {
|
||||
hasCountFunc = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!hasCountFunc) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SSDataBlock* pBlock = createDataBlock();
|
||||
pBlock->info.rows = 1;
|
||||
pBlock->info.capacity = 0;
|
||||
|
||||
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
|
||||
SColumnInfoData colInfo = {0};
|
||||
colInfo.hasNull = true;
|
||||
colInfo.info.type = TSDB_DATA_TYPE_NULL;
|
||||
colInfo.info.bytes = 1;
|
||||
|
||||
SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i];
|
||||
for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) {
|
||||
SFunctParam* pFuncParam = &pOneExpr->base.pParam[j];
|
||||
if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) {
|
||||
int32_t slotId = pFuncParam->pCol->slotId;
|
||||
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
|
||||
if (slotId >= numOfCols) {
|
||||
taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1);
|
||||
for (int32_t k = numOfCols; k < slotId + 1; ++k) {
|
||||
taosArrayPush(pBlock->pDataBlock, &colInfo);
|
||||
}
|
||||
}
|
||||
} else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
blockDataEnsureCapacity(pBlock, pBlock->info.rows);
|
||||
*ppBlock = pBlock;
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppBlock) {
|
||||
if (!blockAllocated) {
|
||||
return;
|
||||
}
|
||||
|
||||
blockDataDestroy(*ppBlock);
|
||||
*ppBlock = NULL;
|
||||
}
|
||||
|
||||
|
||||
// this is a blocking operator
|
||||
static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
|
||||
if (OPTR_IS_OPENED(pOperator)) {
|
||||
|
|
@ -1436,22 +1482,36 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
|
|||
int32_t order = TSDB_ORDER_ASC;
|
||||
int32_t scanFlag = MAIN_SCAN;
|
||||
|
||||
bool hasValidBlock = false;
|
||||
bool blockAllocated = false;
|
||||
|
||||
while (1) {
|
||||
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
|
||||
if (pBlock == NULL) {
|
||||
if (!hasValidBlock) {
|
||||
createDataBlockForEmptyInput(pOperator, &pBlock);
|
||||
if (pBlock == NULL) {
|
||||
break;
|
||||
}
|
||||
blockAllocated = true;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
hasValidBlock = true;
|
||||
|
||||
int32_t code = getTableScanInfo(pOperator, &order, &scanFlag);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
|
||||
T_LONG_JMP(pTaskInfo->env, code);
|
||||
}
|
||||
|
||||
// there is an scalar expression that needs to be calculated before apply the group aggregation.
|
||||
if (pAggInfo->scalarExprSup.pExprInfo != NULL) {
|
||||
if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) {
|
||||
SExprSupp* pSup1 = &pAggInfo->scalarExprSup;
|
||||
code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
|
||||
T_LONG_JMP(pTaskInfo->env, code);
|
||||
}
|
||||
}
|
||||
|
|
@ -1461,8 +1521,12 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
|
|||
setInputDataBlock(pSup, pBlock, order, scanFlag, true);
|
||||
code = doAggregateImpl(pOperator, pSup->pCtx);
|
||||
if (code != 0) {
|
||||
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
|
||||
T_LONG_JMP(pTaskInfo->env, code);
|
||||
}
|
||||
|
||||
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
|
||||
|
||||
}
|
||||
|
||||
// the downstream operator may return with error code, so let's check the code before generating results.
|
||||
|
|
@ -1513,179 +1577,6 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
|
|||
return (rows == 0) ? NULL : pInfo->pRes;
|
||||
}
|
||||
|
||||
static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
|
||||
SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) {
|
||||
pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows;
|
||||
SSDataBlock* pResBlock = pInfo->pFinalRes;
|
||||
|
||||
int32_t order = TSDB_ORDER_ASC;
|
||||
int32_t scanFlag = MAIN_SCAN;
|
||||
getTableScanInfo(pOperator, &order, &scanFlag);
|
||||
|
||||
int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey;
|
||||
taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo));
|
||||
|
||||
blockDataCleanup(pInfo->pRes);
|
||||
doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag);
|
||||
|
||||
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey);
|
||||
taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes);
|
||||
|
||||
int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows;
|
||||
taosFillResultDataBlock(pInfo->pFillInfo, pResBlock, numOfResultRows);
|
||||
|
||||
pInfo->curGroupId = pInfo->existNewGroupBlock->info.id.groupId;
|
||||
pInfo->existNewGroupBlock = NULL;
|
||||
}
|
||||
|
||||
static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
|
||||
SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) {
|
||||
if (taosFillHasMoreResults(pInfo->pFillInfo)) {
|
||||
int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows;
|
||||
taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows);
|
||||
pInfo->pRes->info.id.groupId = pInfo->curGroupId;
|
||||
return;
|
||||
}
|
||||
|
||||
// handle the cached new group data block
|
||||
if (pInfo->existNewGroupBlock) {
|
||||
doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo);
|
||||
}
|
||||
}
|
||||
|
||||
static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) {
|
||||
SFillOperatorInfo* pInfo = pOperator->info;
|
||||
SExprSupp* pSup = &pOperator->exprSupp;
|
||||
setInputDataBlock(pSup, pBlock, order, scanFlag, false);
|
||||
projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, NULL);
|
||||
|
||||
// reset the row value before applying the no-fill functions to the input data block, which is "pBlock" in this case.
|
||||
pInfo->pRes->info.rows = 0;
|
||||
SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp;
|
||||
setInputDataBlock(pNoFillSupp, pBlock, order, scanFlag, false);
|
||||
|
||||
projectApplyFunctions(pNoFillSupp->pExprInfo, pInfo->pRes, pBlock, pNoFillSupp->pCtx, pNoFillSupp->numOfExprs, NULL);
|
||||
pInfo->pRes->info.id.groupId = pBlock->info.id.groupId;
|
||||
}
|
||||
|
||||
static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
|
||||
SFillOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
SResultInfo* pResultInfo = &pOperator->resultInfo;
|
||||
SSDataBlock* pResBlock = pInfo->pFinalRes;
|
||||
|
||||
blockDataCleanup(pResBlock);
|
||||
|
||||
int32_t order = TSDB_ORDER_ASC;
|
||||
int32_t scanFlag = MAIN_SCAN;
|
||||
getTableScanInfo(pOperator, &order, &scanFlag);
|
||||
|
||||
doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo);
|
||||
if (pResBlock->info.rows > 0) {
|
||||
pResBlock->info.id.groupId = pInfo->curGroupId;
|
||||
return pResBlock;
|
||||
}
|
||||
|
||||
SOperatorInfo* pDownstream = pOperator->pDownstream[0];
|
||||
while (1) {
|
||||
SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream);
|
||||
if (pBlock == NULL) {
|
||||
if (pInfo->totalInputRows == 0) {
|
||||
setOperatorCompleted(pOperator);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey);
|
||||
} else {
|
||||
blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId);
|
||||
|
||||
blockDataCleanup(pInfo->pRes);
|
||||
blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
|
||||
blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows);
|
||||
doApplyScalarCalculation(pOperator, pBlock, order, scanFlag);
|
||||
|
||||
if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) {
|
||||
pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block
|
||||
pInfo->totalInputRows += pInfo->pRes->info.rows;
|
||||
|
||||
if (order == pInfo->pFillInfo->order) {
|
||||
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey);
|
||||
} else {
|
||||
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey);
|
||||
}
|
||||
taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes);
|
||||
} else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block
|
||||
pInfo->existNewGroupBlock = pBlock;
|
||||
|
||||
// Fill the previous group data block, before handle the data block of new group.
|
||||
// Close the fill operation for previous group data block
|
||||
taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t numOfResultRows = pOperator->resultInfo.capacity - pResBlock->info.rows;
|
||||
taosFillResultDataBlock(pInfo->pFillInfo, pResBlock, numOfResultRows);
|
||||
|
||||
// current group has no more result to return
|
||||
if (pResBlock->info.rows > 0) {
|
||||
// 1. The result in current group not reach the threshold of output result, continue
|
||||
// 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately
|
||||
if (pResBlock->info.rows > pResultInfo->threshold || pBlock == NULL || pInfo->existNewGroupBlock != NULL) {
|
||||
pResBlock->info.id.groupId = pInfo->curGroupId;
|
||||
return pResBlock;
|
||||
}
|
||||
|
||||
doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo);
|
||||
if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) {
|
||||
pResBlock->info.id.groupId = pInfo->curGroupId;
|
||||
return pResBlock;
|
||||
}
|
||||
} else if (pInfo->existNewGroupBlock) { // try next group
|
||||
assert(pBlock != NULL);
|
||||
|
||||
blockDataCleanup(pResBlock);
|
||||
|
||||
doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo);
|
||||
if (pResBlock->info.rows > pResultInfo->threshold) {
|
||||
pResBlock->info.id.groupId = pInfo->curGroupId;
|
||||
return pResBlock;
|
||||
}
|
||||
} else {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static SSDataBlock* doFill(SOperatorInfo* pOperator) {
|
||||
SFillOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
if (pOperator->status == OP_EXEC_DONE) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
SSDataBlock* fillResult = NULL;
|
||||
while (true) {
|
||||
fillResult = doFillImpl(pOperator);
|
||||
if (fillResult == NULL) {
|
||||
setOperatorCompleted(pOperator);
|
||||
break;
|
||||
}
|
||||
|
||||
doFilter(fillResult, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo);
|
||||
if (fillResult->info.rows > 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (fillResult != NULL) {
|
||||
pOperator->resultInfo.totalRows += fillResult->info.rows;
|
||||
}
|
||||
|
||||
return fillResult;
|
||||
}
|
||||
|
||||
void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) {
|
||||
for (int32_t i = 0; i < numOfExprs; ++i) {
|
||||
SExprInfo* pExprInfo = &pExpr[i];
|
||||
|
|
@ -1955,167 +1846,6 @@ void destroyAggOperatorInfo(void* param) {
|
|||
taosMemoryFreeClear(param);
|
||||
}
|
||||
|
||||
void destroyFillOperatorInfo(void* param) {
|
||||
SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param;
|
||||
pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo);
|
||||
pInfo->pRes = blockDataDestroy(pInfo->pRes);
|
||||
pInfo->pFinalRes = blockDataDestroy(pInfo->pFinalRes);
|
||||
|
||||
cleanupExprSupp(&pInfo->noFillExprSupp);
|
||||
|
||||
taosMemoryFreeClear(pInfo->p);
|
||||
taosArrayDestroy(pInfo->matchInfo.pList);
|
||||
taosMemoryFreeClear(param);
|
||||
}
|
||||
|
||||
static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SExprInfo* pNotFillExpr,
|
||||
int32_t numOfNotFillCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity,
|
||||
const char* id, SInterval* pInterval, int32_t fillType, int32_t order) {
|
||||
SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode);
|
||||
|
||||
int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey;
|
||||
STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey);
|
||||
w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order);
|
||||
|
||||
pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo,
|
||||
pInfo->primaryTsCol, order, id);
|
||||
|
||||
if (order == TSDB_ORDER_ASC) {
|
||||
pInfo->win.skey = win.skey;
|
||||
pInfo->win.ekey = win.ekey;
|
||||
} else {
|
||||
pInfo->win.skey = win.ekey;
|
||||
pInfo->win.ekey = win.skey;
|
||||
}
|
||||
pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES);
|
||||
|
||||
if (pInfo->pFillInfo == NULL || pInfo->p == NULL) {
|
||||
taosMemoryFree(pInfo->pFillInfo);
|
||||
taosMemoryFree(pInfo->p);
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
} else {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
static bool isWstartColumnExist(SFillOperatorInfo* pInfo) {
|
||||
if (pInfo->noFillExprSupp.numOfExprs == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < pInfo->noFillExprSupp.numOfExprs; ++i) {
|
||||
SExprInfo* exprInfo = pInfo->noFillExprSupp.pExprInfo + i;
|
||||
if (exprInfo->pExpr->nodeType == QUERY_NODE_COLUMN && exprInfo->base.numOfParams == 1 &&
|
||||
exprInfo->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static int32_t createPrimaryTsExprIfNeeded(SFillOperatorInfo* pInfo, SFillPhysiNode* pPhyFillNode, SExprSupp* pExprSupp,
|
||||
const char* idStr) {
|
||||
bool wstartExist = isWstartColumnExist(pInfo);
|
||||
|
||||
if (wstartExist == false) {
|
||||
if (pPhyFillNode->pWStartTs->type != QUERY_NODE_TARGET) {
|
||||
qError("pWStartTs of fill physical node is not a target node, %s", idStr);
|
||||
return TSDB_CODE_QRY_SYS_ERROR;
|
||||
}
|
||||
|
||||
SExprInfo* pExpr = taosMemoryRealloc(pExprSupp->pExprInfo, (pExprSupp->numOfExprs + 1) * sizeof(SExprInfo));
|
||||
if (pExpr == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
createExprFromTargetNode(&pExpr[pExprSupp->numOfExprs], (STargetNode*)pPhyFillNode->pWStartTs);
|
||||
pExprSupp->numOfExprs += 1;
|
||||
pExprSupp->pExprInfo = pExpr;
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode,
|
||||
SExecTaskInfo* pTaskInfo) {
|
||||
SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo));
|
||||
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
|
||||
if (pInfo == NULL || pOperator == NULL) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
|
||||
SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr);
|
||||
pOperator->exprSupp.pExprInfo = pExprInfo;
|
||||
|
||||
SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp;
|
||||
pNoFillSupp->pExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pNoFillSupp->numOfExprs);
|
||||
int32_t code = createPrimaryTsExprIfNeeded(pInfo, pPhyFillNode, pNoFillSupp, pTaskInfo->id.str);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
SInterval* pInterval =
|
||||
QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType
|
||||
? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval
|
||||
: &((SIntervalAggOperatorInfo*)downstream->info)->interval;
|
||||
|
||||
int32_t order = (pPhyFillNode->inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
|
||||
int32_t type = convertFillType(pPhyFillNode->mode);
|
||||
|
||||
SResultInfo* pResultInfo = &pOperator->resultInfo;
|
||||
|
||||
initResultSizeInfo(&pOperator->resultInfo, 4096);
|
||||
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
|
||||
code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId;
|
||||
pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId;
|
||||
|
||||
int32_t numOfOutputCols = 0;
|
||||
code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols,
|
||||
COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
|
||||
|
||||
code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs,
|
||||
(SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity,
|
||||
pTaskInfo->id.str, pInterval, type, order);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
pInfo->pFinalRes = createOneDataBlock(pInfo->pRes, false);
|
||||
blockDataEnsureCapacity(pInfo->pFinalRes, pOperator->resultInfo.capacity);
|
||||
|
||||
code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _error;
|
||||
}
|
||||
|
||||
setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo);
|
||||
pOperator->exprSupp.numOfExprs = pInfo->numOfExpr;
|
||||
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL);
|
||||
|
||||
code = appendDownstream(pOperator, &downstream, 1);
|
||||
return pOperator;
|
||||
|
||||
_error:
|
||||
if (pInfo != NULL) {
|
||||
destroyFillOperatorInfo(pInfo);
|
||||
}
|
||||
|
||||
pTaskInfo->code = code;
|
||||
taosMemoryFreeClear(pOperator);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model, char* dbFName) {
|
||||
SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
|
||||
if (pTaskInfo == NULL) {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -629,7 +629,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
|
|||
|
||||
while (tsdbNextDataBlock(pTableScanInfo->base.dataReader)) {
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
|
||||
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
// process this data block based on the probabilities
|
||||
|
|
@ -895,6 +895,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
|
|||
|
||||
pInfo->currentGroupId = -1;
|
||||
pInfo->assignBlockUid = pTableScanNode->assignBlockUid;
|
||||
pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false;
|
||||
|
||||
setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo,
|
||||
pTaskInfo);
|
||||
|
|
@ -1051,6 +1052,9 @@ static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts,
|
|||
}
|
||||
|
||||
static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) {
|
||||
if (pBlock->info.rows == 0) {
|
||||
return false;
|
||||
}
|
||||
if ((*pRowIndex) == pBlock->info.rows) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -1109,7 +1113,7 @@ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t
|
|||
if (hasGroup) {
|
||||
(*pRowIndex) += 1;
|
||||
} else {
|
||||
while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] < endWin.ekey)) {
|
||||
while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] <= endWin.ekey)) {
|
||||
(*pRowIndex) += 1;
|
||||
if ((*pRowIndex) == pDataBlockInfo->rows) {
|
||||
break;
|
||||
|
|
@ -1183,10 +1187,10 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
|
|||
}
|
||||
|
||||
static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
|
||||
blockDataCleanup(pDestBlock);
|
||||
if (pSrcBlock->info.rows == 0) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
blockDataCleanup(pDestBlock);
|
||||
int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
|
|
@ -1335,30 +1339,6 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
|
|||
return code;
|
||||
}
|
||||
|
||||
static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* pResBlock) {
|
||||
if (pTagCalSup == NULL || pTagCalSup->numOfExprs == 0) return;
|
||||
if (pBlock == NULL || pBlock->info.rows == 0) return;
|
||||
|
||||
SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0);
|
||||
ASSERT(pSrcBlock->info.rows == 1);
|
||||
|
||||
blockDataEnsureCapacity(pResBlock, 1);
|
||||
|
||||
projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL);
|
||||
ASSERT(pResBlock->info.rows == 1);
|
||||
|
||||
// build tagArray
|
||||
/*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/
|
||||
/*STagVal tagVal = {*/
|
||||
/*.cid = 0,*/
|
||||
/*.type = 0,*/
|
||||
/*};*/
|
||||
// build STag
|
||||
// set STag
|
||||
|
||||
blockDataDestroy(pSrcBlock);
|
||||
}
|
||||
|
||||
void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
|
||||
SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup;
|
||||
SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState;
|
||||
|
|
@ -1836,6 +1816,12 @@ FETCH_NEXT_BLOCK:
|
|||
}
|
||||
setBlockGroupIdByUid(pInfo, pDelBlock);
|
||||
printDataBlock(pDelBlock, "stream scan delete recv filtered");
|
||||
if (pDelBlock->info.rows == 0) {
|
||||
if (pInfo->tqReader) {
|
||||
blockDataDestroy(pDelBlock);
|
||||
}
|
||||
goto FETCH_NEXT_BLOCK;
|
||||
}
|
||||
if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) {
|
||||
generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes);
|
||||
pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT;
|
||||
|
|
@ -2046,7 +2032,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
|
|||
|
||||
if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) {
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
|
||||
longjmp(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
int32_t rows = 0;
|
||||
|
|
@ -2543,7 +2529,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
|
|||
STsdbReader* reader = pInfo->base.dataReader;
|
||||
while (tsdbNextDataBlock(reader)) {
|
||||
if (isTaskKilled(pTaskInfo)) {
|
||||
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
|
||||
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
|
||||
}
|
||||
|
||||
// process this data block based on the probabilities
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue