xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix:conflict from 3.0

This commit is contained in:
wangmm0220 2023-02-13 14:04:36 +08:00
parent 0e630b8f81 6483b92f0c
commit b054ef9a9f
228 changed files with 10823 additions and 6166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -14,6 +14,12 @@
[![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master)
[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
<br />
[![Twitter Follow](https://img.shields.io/twitter/follow/tdenginedb?label=TDengine&style=social)](https://twitter.com/tdenginedb)
[![YouTube Channel](https://img.shields.io/badge/Subscribe_@tdengine--white?logo=youtube&style=social)](https://www.youtube.com/@tdengine)
[![Discord Community](https://img.shields.io/badge/Join_Discord--white?logo=discord&style=social)](https://discord.com/invite/VZdSuUg4pS)
[![LinkedIn](https://img.shields.io/badge/Follow_LinkedIn--white?logo=linkedin&style=social)](https://www.linkedin.com/company/tdengine)
[![StackOverflow](https://img.shields.io/badge/Ask_StackOverflow--white?logo=stackoverflow&style=social&logoColor=orange)](https://stackoverflow.com/questions/tagged/tdengine)
English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine.com) | [Learn more about TSDB](https://tdengine.com/tsdb/)

15
cmake/cmake.platform
View File

@ -37,6 +37,21 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
SET(TD_LINUX_32 TRUE)
ENDIF ()
EXECUTE_PROCESS(COMMAND chmod 777 ${CMAKE_CURRENT_LIST_DIR}/../packaging/tools/get_os.sh)
EXECUTE_PROCESS(COMMAND readlink /bin/sh OUTPUT_VARIABLE SHELL_LINK)
MESSAGE(STATUS "The shell is: " ${SHELL_LINK})
IF (${SHELL_LINK} MATCHES "dash")
EXECUTE_PROCESS(COMMAND ${CMAKE_CURRENT_LIST_DIR}/../packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO)
ELSE ()
EXECUTE_PROCESS(COMMAND sh ${CMAKE_CURRENT_LIST_DIR}/../packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO)
ENDIF ()
MESSAGE(STATUS "The current OS is " ${TD_OS_INFO})
IF (${TD_OS_INFO} MATCHES "Alpine")
SET(TD_ALPINE TRUE)
ADD_DEFINITIONS("-D_ALPINE")
ENDIF ()
ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
SET(TD_DARWIN TRUE)

2
cmake/cmake.version
View File

@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
SET(TD_VER_NUMBER "3.0.2.4")
SET(TD_VER_NUMBER "3.0.2.5")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)

2
cmake/taosadapter_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
GIT_TAG 3e08996
GIT_TAG db6c843
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 6a2d9fc
GIT_TAG 22627d7
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

2
cmake/taosws_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taosws-rs
ExternalProject_Add(taosws-rs
GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git
GIT_TAG f406d51
GIT_TAG main
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

65
docs/en/07-develop/03-insert-data/_py_kafka.mdx
View File

@ -53,8 +53,69 @@ for p in ps:
In addition to python's built-in multithreading and multiprocessing library, we can also use the third-party library gunicorn.
### Examples
### examples
<details>
<summary>kafka_example_perform</summary>
`kafka_example_perform` is the entry point of the examples.
```py
{{#include docs/examples/python/kafka_example.py}}
{{#include docs/examples/python/kafka_example_perform.py}}
```
</details>
<details>
<summary>kafka_example_common</summary>
`kafka_example_common` is the common code of the examples.
```py
{{#include docs/examples/python/kafka_example_common.py}}
```
</details>
<details>
<summary>kafka_example_producer</summary>
`kafka_example_producer` is `producer`, which is responsible for generating test data and sending it to kafka.
```py
{{#include docs/examples/python/kafka_example_producer.py}}
```
</details>
<details>
<summary>kafka_example_consumer</summary>
`kafka_example_consumer` is `consumer`，which is responsible for consuming data from kafka and writing it to TDengine.
```py
{{#include docs/examples/python/kafka_example_consumer.py}}
```
</details>
### execute Python examples
<details>
<summary>execute Python examples</summary>
1. install and start up `kafka`
2. install python3 and pip
3. install `taospy` by pip
4. install `kafka-python` by pip
5. execute this example
The entry point of this example is `kafka_example_perform.py`. For more information about usage, please use `--help` command.
```
python3 kafka_example_perform.py --help
```
For example, the following command is creating 100 sub-table and inserting 20000 data for each table and the kafka max poll is 100 and 1 thread and 1 process per thread.
```
python3 kafka_example_perform.py -table-count=100 -table-items=20000 -max-poll=100 -threads=1 -processes=1
```
</details>

4
docs/en/12-taos-sql/06-select.md
View File

@ -13,6 +13,7 @@ SELECT [DISTINCT] select_list
from_clause
[WHERE condition]
[partition_by_clause]
[interp_clause]
[window_clause]
[group_by_clause]
[order_by_clasue]
@ -53,6 +54,9 @@ window_clause: {
| STATE_WINDOW(col)
| INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)]
interp_clause:
RANGE(ts_val, ts_val), EVERY(every_val), FILL(fill_mod_and_val)
partition_by_clause:
PARTITION BY expr [, expr] ...

6
docs/en/12-taos-sql/10-function.md
View File

@ -873,9 +873,9 @@ INTERP(expr)
- `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter.
- The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input.
- `INTERP` must be used along with `RANGE`, `EVERY`, `FILL` keywords.
- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1<=timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified.
- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. The parameter `EVERY` must be an integer, with no quotes, with a time unit of: b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds.
- Interpolation is performed based on `FILL` parameter.
- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1 <= timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified.
- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY(time_unit)`. Starting from timestamp1, one interpolation is performed for every time interval specified `time_unit` parameter. The parameter `time_unit` must be an integer, with no quotes, with a time unit of: a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds.
- Interpolation is performed based on `FILL` parameter. For more information about FILL clause, see [FILL Clause](../distinguished/#fill-clause).
- `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable.
- Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4).
- Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.2.3).

BIN
docs/en/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 78 KiB

11
docs/en/14-reference/07-tdinsight/index.md
View File

@ -326,11 +326,12 @@ Currently, only the number of logins per minute is reported.
Support monitoring taosAdapter request statistics and status details. Includes.
1. **http_request_inflight**: number of real-time requests.
2. **http_request_total**: number of total requests.
3. **http_request_fail**: number of failed requets.
4. **CPU Used**: CPU usage of taosAdapter.
5. **Memory Used**: Memory usage of taosAdapter.
1. **Http Request Total**: number of total requests.
2. **Http Request Fail**: number of failed requests.
3. **CPU Used**: CPU usage of taosAdapter.
4. **Memory Used**: Memory usage of taosAdapter.
5. **Http Request Inflight**: number of real-time requests.
6. **Http Status Code**: taosAdapter http status code.
## Upgrade

11
docs/en/20-third-party/01-grafana.mdx vendored
View File

@ -156,13 +156,13 @@ You can setup a zero-configuration stack for TDengine + Grafana by [docker-compo
services:
tdengine:
image: tdengine/tdengine:2.6.0.2
image: tdengine/tdengine:3.0.2.4
environment:
TAOS_FQDN: tdengine
volumes:
- tdengine-data:/var/lib/taos/
grafana:
image: grafana/grafana:8.5.6
image: grafana/grafana:9.3.6
volumes:
- ./tdengine.yml/:/etc/grafana/provisioning/tdengine.yml
- grafana-data:/var/lib/grafana
@ -197,11 +197,18 @@ As shown above, select the `TDengine` data source in the `Query` and enter the c
- INPUT SQL: Enter the desired query (the results being two columns and multiple rows), such as `select _wstart, avg(mem_system) from log.dnodes_info where ts >= $from and ts < $to interval($interval)`. In this statement, $from, $to, and $interval are variables that Grafana replaces with the query time range and interval. In addition to the built-in variables, custom template variables are also supported.
- ALIAS BY: This allows you to set the current query alias.
- GENERATE SQL: Clicking this button will automatically replace the corresponding variables and generate the final executed statement.
- Group by column name(s): `group by` or `partition by` columns name split by comma. By setting `Group by column name(s)`, it can show multi-dimension data if Sql is `group by` or `partition by`. Such as, it can show data by `dnode_ep` if sql is `select _wstart as ts, avg(mem_system), dnode_ep from log.dnodes_info where ts>=$from and ts<=$to partition by dnode_ep interval($interval)` and `Group by column name(s)` is `dnode_ep`.
- Format to: format legend for `group by` or `partition by`. Such as it can display series data by `dnode_ep` if sql is `select _wstart as ts, avg(mem_system), dnode_ep from log.dnodes_info where ts>=$from and ts<=$to partition by dnode_ep interval($interval)` and `Group by column name(s)` is `dnode_ep` and `Format to` is `mem_system_{{dnode_ep}}`.
Follow the default prompt to query the average system memory usage for the specified interval on the server where the current TDengine deployment is located as follows.
![TDengine Database TDinsight plugin create dashboard 2](./grafana/create_dashboard2.webp)
查询每台 TDengine 服务器指定间隔系统内存平均使用量如下.
The example to query the average system memory usage for the specified interval on each server as follows.
![TDengine Database TDinsight plugin create dashboard 2](./grafana/create_dashboard3.webp)
> For more information on how to use Grafana to create the appropriate monitoring interface and for more details on using Grafana, refer to the official Grafana [documentation](https://grafana.com/docs/).
### Importing the Dashboard

BIN
docs/en/20-third-party/grafana/create_dashboard1.webp vendored
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 75 KiB

After

Width:  |  Height:  |  Size: 36 KiB

BIN
docs/en/20-third-party/grafana/create_dashboard2.webp vendored
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 50 KiB

BIN
docs/en/20-third-party/grafana/create_dashboard3.webp vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 59 KiB

4
docs/en/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w
import Release from "/components/ReleaseV3";
## 3.0.2.5
<Release type="tdengine" version="3.0.2.5" />
## 3.0.2.4
<Release type="tdengine" version="3.0.2.4" />

4
docs/en/28-releases/02-tools.md
View File

@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat
import Release from "/components/ReleaseV3";
## 2.4.3
<Release type="tools" version="2.4.3" />
## 2.4.2
<Release type="tools" version="2.4.2" />

241
docs/examples/python/kafka_example.py
View File

@ -1,241 +0,0 @@
#! encoding = utf-8
import json
import time
from json import JSONDecodeError
from typing import Callable
import logging
from concurrent.futures import ThreadPoolExecutor, Future
import taos
from kafka import KafkaConsumer
from kafka.consumer.fetcher import ConsumerRecord
class Consumer(object):
DEFAULT_CONFIGS = {
'kafka_brokers': 'localhost:9092',
'kafka_topic': 'python_kafka',
'kafka_group_id': 'taos',
'taos_host': 'localhost',
'taos_user': 'root',
'taos_password': 'taosdata',
'taos_database': 'power',
'taos_port': 6030,
'timezone': None,
'clean_after_testing': False,
'bath_consume': True,
'batch_size': 1000,
'async_model': True,
'workers': 10,
'testing': False
}
LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose',
'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale',
'California.SantaClara', 'California.Cupertino']
CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1'
USE_DATABASE_SQL = 'use {}'
DROP_TABLE_SQL = 'drop table if exists meters'
DROP_DATABASE_SQL = 'drop database if exists {}'
CREATE_STABLE_SQL = 'create stable meters (ts timestamp, current float, voltage int, phase float) ' \
'tags (location binary(64), groupId int)'
CREATE_TABLE_SQL = 'create table if not exists {} using meters tags (\'{}\', {})'
INSERT_SQL_HEADER = "insert into "
INSERT_PART_SQL = 'power.{} values (\'{}\', {}, {}, {})'
def __init__(self, **configs):
self.config: dict = self.DEFAULT_CONFIGS
self.config.update(configs)
if not self.config.get('testing'):
self.consumer = KafkaConsumer(
self.config.get('kafka_topic'), # topic
bootstrap_servers=self.config.get('kafka_brokers'),
group_id=self.config.get('kafka_group_id'),
)
self.taos = taos.connect(
host=self.config.get('taos_host'),
user=self.config.get('taos_user'),
password=self.config.get('taos_password'),
port=self.config.get('taos_port'),
timezone=self.config.get('timezone'),
)
if self.config.get('async_model'):
self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers'))
self.tasks = []
# tags and table mapping # key: {location}_{groupId} value:
self.tag_table_mapping = {}
i = 0
for location in self.LOCATIONS:
for j in range(1, 11):
table_name = 'd{}'.format(i)
self._cache_table(location=location, group_id=j, table_name=table_name)
i += 1
def init_env(self):
# create database and table
self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database')))
self.taos.execute(self.CREATE_DATABASE_SQL.format(self.config.get('taos_database')))
self.taos.execute(self.USE_DATABASE_SQL.format(self.config.get('taos_database')))
self.taos.execute(self.DROP_TABLE_SQL)
self.taos.execute(self.CREATE_STABLE_SQL)
for tags, table_name in self.tag_table_mapping.items():
location, group_id = _get_location_and_group(tags)
self.taos.execute(self.CREATE_TABLE_SQL.format(table_name, location, group_id))
def consume(self):
logging.warning('## start consumer topic-[%s]', self.config.get('kafka_topic'))
try:
if self.config.get('bath_consume'):
self._run_batch(self._to_taos_batch)
else:
self._run(self._to_taos)
except KeyboardInterrupt:
logging.warning("## caught keyboard interrupt, stopping")
finally:
self.stop()
def stop(self):
# close consumer
if self.consumer is not None:
self.consumer.commit()
self.consumer.close()
# multi thread
if self.config.get('async_model'):
for task in self.tasks:
while not task.done():
pass
if self.pool is not None:
self.pool.shutdown()
# clean data
if self.config.get('clean_after_testing'):
self.taos.execute(self.DROP_TABLE_SQL)
self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database')))
# close taos
if self.taos is not None:
self.taos.close()
def _run(self, f):
for message in self.consumer:
if self.config.get('async_model'):
self.pool.submit(f(message))
else:
f(message)
def _run_batch(self, f):
while True:
messages = self.consumer.poll(timeout_ms=500, max_records=self.config.get('batch_size'))
if messages:
if self.config.get('async_model'):
self.pool.submit(f, messages.values())
else:
f(list(messages.values()))
if not messages:
time.sleep(0.1)
def _to_taos(self, message: ConsumerRecord) -> bool:
sql = self.INSERT_SQL_HEADER + self._build_sql(message.value)
if len(sql) == 0: # decode error, skip
return True
logging.info('## insert sql %s', sql)
return self.taos.execute(sql=sql) == 1
def _to_taos_batch(self, messages):
sql = self._build_sql_batch(messages=messages)
if len(sql) == 0: # decode error, skip
return
self.taos.execute(sql=sql)
def _build_sql(self, msg_value: str) -> str:
try:
data = json.loads(msg_value)
except JSONDecodeError as e:
logging.error('## decode message [%s] error ', msg_value, e)
return ''
location = data.get('location')
group_id = data.get('groupId')
ts = data.get('ts')
current = data.get('current')
voltage = data.get('voltage')
phase = data.get('phase')
table_name = self._get_table_name(location=location, group_id=group_id)
return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase)
def _build_sql_batch(self, messages) -> str:
sql_list = []
for partition_messages in messages:
for message in partition_messages:
sql_list.append(self._build_sql(message.value))
return self.INSERT_SQL_HEADER + ' '.join(sql_list)
def _cache_table(self, location: str, group_id: int, table_name: str):
self.tag_table_mapping[_tag_table_mapping_key(location=location, group_id=group_id)] = table_name
def _get_table_name(self, location: str, group_id: int) -> str:
return self.tag_table_mapping.get(_tag_table_mapping_key(location=location, group_id=group_id))
def _tag_table_mapping_key(location: str, group_id: int):
return '{}_{}'.format(location, group_id)
def _get_location_and_group(key: str) -> (str, int):
fields = key.split('_')
return fields[0], fields[1]
def test_to_taos(consumer: Consumer):
msg = {
'location': 'California.SanFrancisco',
'groupId': 1,
'ts': '2022-12-06 15:13:38.643',
'current': 3.41,
'voltage': 105,
'phase': 0.02027,
}
record = ConsumerRecord(checksum=None, headers=None, offset=1, key=None, value=json.dumps(msg), partition=1,
topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None)
assert consumer._to_taos(message=record)
def test_to_taos_batch(consumer: Consumer):
records = [
[
ConsumerRecord(checksum=None, headers=None, offset=1, key=None,
value=json.dumps({'location': 'California.SanFrancisco',
'groupId': 1,
'ts': '2022-12-06 15:13:38.643',
'current': 3.41,
'voltage': 105,
'phase': 0.02027, }),
partition=1, topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None),
ConsumerRecord(checksum=None, headers=None, offset=1, key=None,
value=json.dumps({'location': 'California.LosAngles',
'groupId': 2,
'ts': '2022-12-06 15:13:39.643',
'current': 3.41,
'voltage': 102,
'phase': 0.02027, }),
partition=1, topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None),
]
]
consumer._to_taos_batch(messages=records)
if __name__ == '__main__':
consumer = Consumer(async_model=True, testing=True)
# init env
consumer.init_env()
# consumer.consume()
# test build sql
# test build sql batch
test_to_taos(consumer)
test_to_taos_batch(consumer)

65
docs/examples/python/kafka_example_common.py Normal file
View File

@ -0,0 +1,65 @@
#! encoding = utf-8
import taos
LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose',
'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale',
'California.SantaClara', 'California.Cupertino']
CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1'
USE_DATABASE_SQL = 'use {}'
DROP_TABLE_SQL = 'drop table if exists meters'
DROP_DATABASE_SQL = 'drop database if exists {}'
CREATE_STABLE_SQL = 'create stable meters (ts timestamp, current float, voltage int, phase float) tags ' \
'(location binary(64), groupId int)'
CREATE_TABLE_SQL = 'create table if not exists {} using meters tags (\'{}\', {})'
def create_database_and_tables(host, port, user, password, db, table_count):
tags_tables = _init_tags_table_names(table_count=table_count)
conn = taos.connect(host=host, port=port, user=user, password=password)
conn.execute(DROP_DATABASE_SQL.format(db))
conn.execute(CREATE_DATABASE_SQL.format(db))
conn.execute(USE_DATABASE_SQL.format(db))
conn.execute(DROP_TABLE_SQL)
conn.execute(CREATE_STABLE_SQL)
for tags in tags_tables:
location, group_id = _get_location_and_group(tags)
tables = tags_tables[tags]
for table_name in tables:
conn.execute(CREATE_TABLE_SQL.format(table_name, location, group_id))
conn.close()
def clean(host, port, user, password, db):
conn = taos.connect(host=host, port=port, user=user, password=password)
conn.execute(DROP_DATABASE_SQL.format(db))
conn.close()
def _init_tags_table_names(table_count):
tags_table_names = {}
group_id = 0
for i in range(table_count):
table_name = 'd{}'.format(i)
location_idx = i % len(LOCATIONS)
location = LOCATIONS[location_idx]
if location_idx == 0:
group_id += 1
if group_id > 10:
group_id -= 10
key = _tag_table_mapping_key(location=location, group_id=group_id)
if key not in tags_table_names:
tags_table_names[key] = []
tags_table_names[key].append(table_name)
return tags_table_names
def _tag_table_mapping_key(location, group_id):
return '{}_{}'.format(location, group_id)
def _get_location_and_group(key):
fields = key.split('_')
return fields[0], fields[1]

231
docs/examples/python/kafka_example_consumer.py Normal file
View File

@ -0,0 +1,231 @@
#! encoding = utf-8
import json
import logging
import time
from concurrent.futures import ThreadPoolExecutor, Future
from json import JSONDecodeError
from typing import Callable
import taos
from kafka import KafkaConsumer
from kafka.consumer.fetcher import ConsumerRecord
import kafka_example_common as common
class Consumer(object):
DEFAULT_CONFIGS = {
'kafka_brokers': 'localhost:9092', # kafka broker
'kafka_topic': 'tdengine_kafka_practices',
'kafka_group_id': 'taos',
'taos_host': 'localhost', # TDengine host
'taos_port': 6030, # TDengine port
'taos_user': 'root', # TDengine user name
'taos_password': 'taosdata', # TDengine password
'taos_database': 'power', # TDengine database
'message_type': 'json', # message format, 'json' or 'line'
'clean_after_testing': False, # if drop database after testing
'max_poll': 1000, # poll size for batch mode
'workers': 10, # thread count for multi-threading
'testing': False
}
INSERT_SQL_HEADER = "insert into "
INSERT_PART_SQL = '{} values (\'{}\', {}, {}, {})'
def __init__(self, **configs):
self.config = self.DEFAULT_CONFIGS
self.config.update(configs)
self.consumer = None
if not self.config.get('testing'):
self.consumer = KafkaConsumer(
self.config.get('kafka_topic'),
bootstrap_servers=self.config.get('kafka_brokers'),
group_id=self.config.get('kafka_group_id'),
)
self.conns = taos.connect(
host=self.config.get('taos_host'),
port=self.config.get('taos_port'),
user=self.config.get('taos_user'),
password=self.config.get('taos_password'),
db=self.config.get('taos_database'),
)
if self.config.get('workers') > 1:
self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers'))
self.tasks = []
# tags and table mapping # key: {location}_{groupId} value:
def consume(self):
"""
consume data from kafka and deal. Base on `message_type`, `bath_consume`, `insert_by_table`,
there are several deal function.
:return:
"""
self.conns.execute(common.USE_DATABASE_SQL.format(self.config.get('taos_database')))
try:
if self.config.get('message_type') == 'line': # line
self._run(self._line_to_taos)
if self.config.get('message_type') == 'json': # json
self._run(self._json_to_taos)
except KeyboardInterrupt:
logging.warning("## caught keyboard interrupt, stopping")
finally:
self.stop()
def stop(self):
"""
stop consuming
:return:
"""
# close consumer
if self.consumer is not None:
self.consumer.commit()
self.consumer.close()
# multi thread
if self.config.get('workers') > 1:
if self.pool is not None:
self.pool.shutdown()
for task in self.tasks:
while not task.done():
time.sleep(0.01)
# clean data
if self.config.get('clean_after_testing'):
self.conns.execute(common.DROP_TABLE_SQL)
self.conns.execute(common.DROP_DATABASE_SQL.format(self.config.get('taos_database')))
# close taos
if self.conns is not None:
self.conns.close()
def _run(self, f):
"""
run in batch consuming mode
:param f:
:return:
"""
i = 0 # just for test.
while True:
messages = self.consumer.poll(timeout_ms=100, max_records=self.config.get('max_poll'))
if messages:
if self.config.get('workers') > 1:
self.pool.submit(f, messages.values())
else:
f(list(messages.values()))
if not messages:
i += 1 # just for test.
time.sleep(0.1)
if i > 3: # just for test.
logging.warning('## test over.') # just for test.
return # just for test.
def _json_to_taos(self, messages):
"""
convert a batch of json data to sql, and insert into TDengine
:param messages:
:return:
"""
sql = self._build_sql_from_json(messages=messages)
self.conns.execute(sql=sql)
def _line_to_taos(self, messages):
"""
convert a batch of lines data to sql, and insert into TDengine
:param messages:
:return:
"""
lines = []
for partition_messages in messages:
for message in partition_messages:
lines.append(message.value.decode())
sql = self.INSERT_SQL_HEADER + ' '.join(lines)
self.conns.execute(sql=sql)
def _build_single_sql_from_json(self, msg_value):
try:
data = json.loads(msg_value)
except JSONDecodeError as e:
logging.error('## decode message [%s] error ', msg_value, e)
return ''
# location = data.get('location')
# group_id = data.get('groupId')
ts = data.get('ts')
current = data.get('current')
voltage = data.get('voltage')
phase = data.get('phase')
table_name = data.get('table_name')
return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase)
def _build_sql_from_json(self, messages):
sql_list = []
for partition_messages in messages:
for message in partition_messages:
sql_list.append(self._build_single_sql_from_json(message.value))
return self.INSERT_SQL_HEADER + ' '.join(sql_list)
def test_json_to_taos(consumer: Consumer):
records = [
[
ConsumerRecord(checksum=None, headers=None, offset=1, key=None,
value=json.dumps({'table_name': 'd0',
'ts': '2022-12-06 15:13:38.643',
'current': 3.41,
'voltage': 105,
'phase': 0.02027, }),
partition=1, topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None),
ConsumerRecord(checksum=None, headers=None, offset=1, key=None,
value=json.dumps({'table_name': 'd1',
'ts': '2022-12-06 15:13:39.643',
'current': 3.41,
'voltage': 102,
'phase': 0.02027, }),
partition=1, topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None),
]
]
consumer._json_to_taos(messages=records)
def test_line_to_taos(consumer: Consumer):
records = [
[
ConsumerRecord(checksum=None, headers=None, offset=1, key=None,
value="d0 values('2023-01-01 00:00:00.001', 3.49, 109, 0.02737)".encode('utf-8'),
partition=1, topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None),
ConsumerRecord(checksum=None, headers=None, offset=1, key=None,
value="d1 values('2023-01-01 00:00:00.002', 6.19, 112, 0.09171)".encode('utf-8'),
partition=1, topic='test', serialized_key_size=None, serialized_header_size=None,
serialized_value_size=None, timestamp=time.time(), timestamp_type=None),
]
]
consumer._line_to_taos(messages=records)
def consume(kafka_brokers, kafka_topic, kafka_group_id, taos_host, taos_port, taos_user,
taos_password, taos_database, message_type, max_poll, workers):
c = Consumer(kafka_brokers=kafka_brokers, kafka_topic=kafka_topic, kafka_group_id=kafka_group_id,
taos_host=taos_host, taos_port=taos_port, taos_user=taos_user, taos_password=taos_password,
taos_database=taos_database, message_type=message_type, max_poll=max_poll, workers=workers)
c.consume()
if __name__ == '__main__':
consumer = Consumer(testing=True)
common.create_database_and_tables(host='localhost', port=6030, user='root', password='taosdata', db='py_kafka_test',
table_count=10)
consumer.conns.execute(common.USE_DATABASE_SQL.format('py_kafka_test'))
test_json_to_taos(consumer)
test_line_to_taos(consumer)
common.clean(host='localhost', port=6030, user='root', password='taosdata', db='py_kafka_test')

103
docs/examples/python/kafka_example_perform.py Normal file
View File

@ -0,0 +1,103 @@
#! encoding=utf-8
import argparse
import logging
import multiprocessing
import time
from multiprocessing import pool
import kafka_example_common as common
import kafka_example_consumer as consumer
import kafka_example_producer as producer
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-kafka-broker', type=str, default='localhost:9092',
help='kafka borker host. default is `localhost:9200`')
parser.add_argument('-kafka-topic', type=str, default='tdengine-kafka-practices',
help='kafka topic. default is `tdengine-kafka-practices`')
parser.add_argument('-kafka-group', type=str, default='kafka_practices',
help='kafka consumer group. default is `kafka_practices`')
parser.add_argument('-taos-host', type=str, default='localhost',
help='TDengine host. default is `localhost`')
parser.add_argument('-taos-port', type=int, default=6030, help='TDengine port. default is 6030')
parser.add_argument('-taos-user', type=str, default='root', help='TDengine username, default is `root`')
parser.add_argument('-taos-password', type=str, default='taosdata', help='TDengine password, default is `taosdata`')
parser.add_argument('-taos-db', type=str, default='tdengine_kafka_practices',
help='TDengine db name, default is `tdengine_kafka_practices`')
parser.add_argument('-table-count', type=int, default=100, help='TDengine sub-table count, default is 100')
parser.add_argument('-table-items', type=int, default=1000, help='items in per sub-tables, default is 1000')
parser.add_argument('-message-type', type=str, default='line',
help='kafka message type. `line` or `json`. default is `line`')
parser.add_argument('-max-poll', type=int, default=1000, help='max poll for kafka consumer')
parser.add_argument('-threads', type=int, default=10, help='thread count for deal message')
parser.add_argument('-processes', type=int, default=1, help='process count')
args = parser.parse_args()
total = args.table_count * args.table_items
logging.warning("## start to prepare testing data...")
prepare_data_start = time.time()
producer.produce_total(100, args.kafka_broker, args.kafka_topic, args.message_type, total, args.table_count)
prepare_data_end = time.time()
logging.warning("## prepare testing data finished! spend-[%s]", prepare_data_end - prepare_data_start)
logging.warning("## start to create database and tables ...")
create_db_start = time.time()
# create database and table
common.create_database_and_tables(host=args.taos_host, port=args.taos_port, user=args.taos_user,
password=args.taos_password, db=args.taos_db, table_count=args.table_count)
create_db_end = time.time()
logging.warning("## create database and tables finished! spend [%s]", create_db_end - create_db_start)
processes = args.processes
logging.warning("## start to consume data and insert into TDengine...")
consume_start = time.time()
if processes > 1: # multiprocess
multiprocessing.set_start_method("spawn")
pool = pool.Pool(processes)
consume_start = time.time()
for _ in range(processes):
pool.apply_async(func=consumer.consume, args=(
args.kafka_broker, args.kafka_topic, args.kafka_group, args.taos_host, args.taos_port, args.taos_user,
args.taos_password, args.taos_db, args.message_type, args.max_poll, args.threads))
pool.close()
pool.join()
else:
consume_start = time.time()
consumer.consume(kafka_brokers=args.kafka_broker, kafka_topic=args.kafka_topic, kafka_group_id=args.kafka_group,
taos_host=args.taos_host, taos_port=args.taos_port, taos_user=args.taos_user,
taos_password=args.taos_password, taos_database=args.taos_db, message_type=args.message_type,
max_poll=args.max_poll, workers=args.threads)
consume_end = time.time()
logging.warning("## consume data and insert into TDengine over! spend-[%s]", consume_end - consume_start)
# print report
logging.warning(
"\n#######################\n"
" Prepare data \n"
"#######################\n"
"# data_type # %s \n"
"# total # %s \n"
"# spend # %s s\n"
"#######################\n"
" Create database \n"
"#######################\n"
"# stable # 1 \n"
"# sub-table # 100 \n"
"# spend # %s s \n"
"#######################\n"
" Consume \n"
"#######################\n"
"# data_type # %s \n"
"# threads # %s \n"
"# processes # %s \n"
"# total_count # %s \n"
"# spend # %s s\n"
"# per_second # %s \n"
"#######################\n",
args.message_type, total, prepare_data_end - prepare_data_start, create_db_end - create_db_start,
args.message_type, args.threads, processes, total, consume_end - consume_start,
total / (consume_end - consume_start))

97
docs/examples/python/kafka_example_producer.py Normal file
View File

@ -0,0 +1,97 @@
#! encoding = utf-8
import json
import random
import threading
from concurrent.futures import ThreadPoolExecutor, Future
from datetime import datetime
from kafka import KafkaProducer
locations = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose',
'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale',
'California.SantaClara', 'California.Cupertino']
producers: list[KafkaProducer] = []
lock = threading.Lock()
start = 1640966400
def produce_total(workers, broker, topic, message_type, total, table_count):
if len(producers) == 0:
lock.acquire()
if len(producers) == 0:
_init_kafka_producers(broker=broker, count=10)
lock.release()
pool = ThreadPoolExecutor(max_workers=workers)
futures = []
for _ in range(0, workers):
futures.append(pool.submit(_produce_total, topic, message_type, int(total / workers), table_count))
pool.shutdown()
for f in futures:
f.result()
_close_kafka_producers()
def _produce_total(topic, message_type, total, table_count):
producer = _get_kafka_producer()
for _ in range(total):
message = _get_fake_date(message_type=message_type, table_count=table_count)
producer.send(topic=topic, value=message.encode(encoding='utf-8'))
def _init_kafka_producers(broker, count):
for _ in range(count):
p = KafkaProducer(bootstrap_servers=broker, batch_size=64 * 1024, linger_ms=300, acks=0)
producers.append(p)
def _close_kafka_producers():
for p in producers:
p.close()
def _get_kafka_producer():
return producers[random.randint(0, len(producers) - 1)]
def _get_fake_date(table_count, message_type='json'):
if message_type == 'json':
return _get_json_message(table_count=table_count)
if message_type == 'line':
return _get_line_message(table_count=table_count)
return ''
def _get_json_message(table_count):
return json.dumps({
'ts': _get_timestamp(),
'current': random.randint(0, 1000) / 100,
'voltage': random.randint(105, 115),
'phase': random.randint(0, 32000) / 100000,
'location': random.choice(locations),
'groupId': random.randint(1, 10),
'table_name': _random_table_name(table_count)
})
def _get_line_message(table_count):
return "{} values('{}', {}, {}, {})".format(
_random_table_name(table_count), # table
_get_timestamp(), # ts
random.randint(0, 1000) / 100, # current
random.randint(105, 115), # voltage
random.randint(0, 32000) / 100000, # phase
)
def _random_table_name(table_count):
return 'd{}'.format(random.randint(0, table_count - 1))
def _get_timestamp():
global start
lock.acquire(blocking=True)
start += 0.001
lock.release()
return datetime.fromtimestamp(start).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]

66
docs/zh/07-develop/03-insert-data/_py_kafka.mdx
View File

@ -55,6 +55,70 @@ for p in ps:
### 完整示例
<details>
<summary>kafka_example_perform</summary>
`kafka_example_perform` 是示例程序的入口
```py
{{#include docs/examples/python/kafka_example.py}}
{{#include docs/examples/python/kafka_example_perform.py}}
```
</details>
<details>
<summary>kafka_example_common</summary>
`kafka_example_common` 是示例程序的公共代码
```py
{{#include docs/examples/python/kafka_example_common.py}}
```
</details>
<details>
<summary>kafka_example_producer</summary>
`kafka_example_producer` 是示例程序的 producer 代码，负责生成并发送测试数据到 kafka
```py
{{#include docs/examples/python/kafka_example_producer.py}}
```
</details>
<details>
<summary>kafka_example_consumer</summary>
`kafka_example_consumer` 是示例程序的 consumer 代码，负责从 kafka 消费数据，并写入到 TDengine
```py
{{#include docs/examples/python/kafka_example_consumer.py}}
```
</details>
### 执行步骤
<details>
<summary>执行 Python 示例程序</summary>
1. 安装并启动 kafka
2. python 环境准备
- 安装 python3
- 安装 taospy
- 安装 kafka-python
3. 执行示例程序
程序的执行入口是 `kafka_example_perform.py`，获取程序完整的执行参数，请执行 help 命令。
```
python3 kafka_example_perform.py --help
```
以下为创建 100 个子表，每个子表 20000 条数据，kafka max poll 为 100，一个进程，每个进程一个处理线程的程序执行命令
```
python3 kafka_example_perform.py -table-count=100 -table-items=20000 -max-poll=100 -threads=1 -processes=1
```
</details>

2
docs/zh/12-taos-sql/03-table.md
View File

@ -57,7 +57,7 @@ table_option: {
3. MAX_DELAY：用于控制推送计算结果的最大延迟，默认值为 interval 的值(但不能超过最大值)，最小单位毫秒，范围为 1 毫秒到 15 分钟，多个以逗号分隔。注：不建议 MAX_DELAY 设置太小，否则会过于频繁的推送结果，影响存储和查询性能，如无特殊需求，取默认值即可。只可用于超级表，且只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。
4. ROLLUP：Rollup 指定的聚合函数，提供基于多层级的降采样聚合结果。只可用于超级表。只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。作用于超级表除 TS 列外的其它所有列，但是只能定义一个聚合函数。 聚合函数支持 avg, sum, min, max, last, first。
5. SMA：Small Materialized Aggregates，提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。
6. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。
6. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
## 创建子表

4
docs/zh/12-taos-sql/06-select.md
View File

@ -13,6 +13,7 @@ SELECT [DISTINCT] select_list
from_clause
[WHERE condition]
[partition_by_clause]
[interp_clause]
[window_clause]
[group_by_clause]
[order_by_clasue]
@ -53,6 +54,9 @@ window_clause: {
| STATE_WINDOW(col)
| INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)]
interp_clause:
RANGE(ts_val, ts_val), EVERY(every_val), FILL(fill_mod_and_val)
partition_by_clause:
PARTITION BY expr [, expr] ...

6
docs/zh/12-taos-sql/10-function.md
View File

@ -875,9 +875,9 @@ INTERP(expr)
- INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。
- INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。
- INTERP 需要同时与 RANGE，EVERY 和 FILL 关键字一起使用。
- INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1<=timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。
- INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。
- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。
- INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1 <= timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。
- INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（time_unit 值）进行插值，time_unit 可取值时间单位：1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)，1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值.
- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](../distinguished/#fill-子句)
- INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 partition by tbname 一起使用。
- INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。
- INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.2.3版本以后支持)。

BIN
docs/zh/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 78 KiB

11
docs/zh/14-reference/07-tdinsight/index.mdx
View File

@ -345,11 +345,12 @@ TDinsight 仪表盘旨在提供 TDengine 相关资源的使用情况和状态，
支持监控 taosAdapter 请求统计和状态详情。包括：
1. **http_request_inflight**: 即时处理请求数
2. **http_request_total**: 请求总数。
3. **http_request_fail**: 请求总数。
4. **CPU Used**: taosAdapter CPU 使用情况。
5. **Memory Used**: taosAdapter 内存使用情况。
1. **Http Request Total**: 请求总数。
2. **Http Request Fail**: 请求总数。
3. **CPU Used**: taosAdapter CPU 使用情况。
4. **Memory Used**: taosAdapter 内存使用情况。
5. **Http Request Inflight**: 即时处理请求数。
6. **Http Status Code**: taosAdapter http 状态码。
## 升级

12
docs/zh/20-third-party/01-grafana.mdx vendored
View File

@ -77,7 +77,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource
或者从 [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) 或 [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) 下载 .zip 文件到本地并解压到 Grafana 插件目录。命令行下载示例如下：
```bash
GF_VERSION=3.2.7
GF_VERSION=3.2.9
# from GitHub
wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip
# from Grafana
@ -156,13 +156,13 @@ docker run -d \
services:
tdengine:
image: tdengine/tdengine:2.6.0.2
image: tdengine/tdengine:3.0.2.4
environment:
TAOS_FQDN: tdengine
volumes:
- tdengine-data:/var/lib/taos/
grafana:
image: grafana/grafana:8.5.6
image: grafana/grafana:9.3.6
volumes:
- ./tdengine.yml/:/etc/grafana/provisioning/tdengine.yml
- grafana-data:/var/lib/grafana
@ -197,11 +197,17 @@ docker run -d \
- INPUT SQL：输入要查询的语句（该 SQL 语句的结果集应为两列多行），例如：`select _wstart, avg(mem_system) from log.dnodes_info where ts >= $from and ts < $to interval($interval)` ，其中，from、to 和 interval 为 TDengine 插件的内置变量，表示从 Grafana 插件面板获取的查询范围和时间间隔。除了内置变量外，`也支持可以使用自定义模板变量`。
- ALIAS BY：可设置当前查询别名。
- GENERATE SQL： 点击该按钮会自动替换相应变量，并生成最终执行的语句。
- Group by column name(s)： **半角**逗号分隔的 `group by` 或 `partition by` 列名。如果是 `group by` or `partition by` 查询语句，设置 `Group by` 列，可以展示多维数据。例如：INPUT SQL 为 `select _wstart as ts, avg(mem_system), dnode_ep from log.dnodes_info where ts>=$from and ts<=$to partition by dnode_ep interval($interval)`，设置 Group by 列名为 `dnode_ep`，可以按 `dnode_ep` 展示数据。
- Format to： Group by 或 Partition by 场景下多维数据 legend 格式化格式。例如上述 INPUT SQL，将 Format to 设置为 `mem_system_{{dnode_ep}}`，展示的 legend 名字为格式化的列名。
按照默认提示查询当前 TDengine 部署所在服务器指定间隔系统内存平均使用量如下：
![TDengine Database Grafana plugin create dashboard](./create_dashboard2.webp)
查询每台 TDengine 服务器指定间隔系统内存平均使用量如下：
![TDengine Database Grafana plugin create dashboard](./create_dashboard3.webp)
> 关于如何使用 Grafana 创建相应的监测界面以及更多有关使用 Grafana 的信息，请参考 Grafana 官方的[文档](https://grafana.com/docs/)。
### 导入 Dashboard

BIN
docs/zh/20-third-party/create_dashboard1.webp vendored
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 75 KiB

After

Width:  |  Height:  |  Size: 36 KiB

BIN
docs/zh/20-third-party/create_dashboard2.webp vendored
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 50 KiB

BIN
docs/zh/20-third-party/create_dashboard3.webp vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 59 KiB

6
docs/zh/27-train-faq/01-faq.md
View File

@ -243,8 +243,8 @@ sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist
```
launchctl limit maxfiles
```
### 19 建库时提示Out of dnode
该提示是创建db的vnode数量不够了，需要的vnode不能超过了dnode中vnode的上限。因为系统默认是一个dnode中有cpu核数两倍的vnode，也可以通过配置文件中的参数supportVnodes控制。
正常调大taos.cfg种这个supportVnodes参数即可。
### 20 建库时提示 Out of dnodes
该提示是创建 db 的 vnode 数量不够了，需要的 vnode 不能超过了 dnode 中 vnode 的上限。因为系统默认是一个 dnode 中有 CPU 核数两倍的 vnode，也可以通过配置文件中的参数 supportVnodes 控制。
正常调大 taos.cfg 中 supportVnodes 参数即可。

4
docs/zh/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
import Release from "/components/ReleaseV3";
## 3.0.2.5
<Release type="tdengine" version="3.0.2.5" />
## 3.0.2.4
<Release type="tdengine" version="3.0.2.4" />

4
docs/zh/28-releases/02-tools.md
View File

@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下：
import Release from "/components/ReleaseV3";
## 2.4.3
<Release type="tools" version="2.4.3" />
## 2.4.2
<Release type="tools" version="2.4.2" />

2
examples/JDBC/readme.md
View File

@ -10,4 +10,4 @@
| 6 | taosdemo | This is an internal tool for testing Our JDBC-JNI, JDBC-RESTful, RESTful interfaces |
more detail: https://docs.taosdata.com/reference/connector/java/
more detail: https://docs.taosdata.com/connector/java/

6
include/common/tdataformat.h
View File

@ -109,7 +109,7 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
void tRowDestroy(SRow *pRow);
void tRowSort(SArray *aRowP);
int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
// SRowIter ================================
int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);
@ -137,6 +137,7 @@ void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn)
void tColDataClear(SColData *pColData);
void tColDataDeepClear(SColData *pColData);
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward);
void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal);
int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg);
@ -200,6 +201,9 @@ struct SColData {
int16_t cid;
int8_t type;
int8_t smaOn;
int32_t numOfNone; // # of none
int32_t numOfNull; // # of null
int32_t numOfValue; // # of vale
int32_t nVal;
uint8_t flag;
uint8_t *pBitMap;

16
include/common/tmsg.h
View File

@ -58,7 +58,7 @@ extern int32_t tMsgDict[];
#define TMSG_INFO(TYPE) \
((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \
(TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) || \
(TYPE) < TDMT_VND_STREAM_MSG || (TYPE) < TDMT_VND_TMQ_MSG \
(TYPE) < TDMT_VND_STREAM_MSG || (TYPE) < TDMT_VND_TMQ_MSG || (TYPE) < TDMT_VND_TMQ_MAX_MSG \
? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \
: 0
@ -147,10 +147,12 @@ typedef enum _mgmt_table {
#define TSDB_FILL_NONE 0
#define TSDB_FILL_NULL 1
#define TSDB_FILL_SET_VALUE 2
#define TSDB_FILL_LINEAR 3
#define TSDB_FILL_PREV 4
#define TSDB_FILL_NEXT 5
#define TSDB_FILL_NULL_F 2
#define TSDB_FILL_SET_VALUE 3
#define TSDB_FILL_SET_VALUE_F 4
#define TSDB_FILL_LINEAR 5
#define TSDB_FILL_PREV 6
#define TSDB_FILL_NEXT 7
#define TSDB_ALTER_USER_PASSWD 0x1
#define TSDB_ALTER_USER_SUPERUSER 0x2
@ -1241,7 +1243,7 @@ int32_t tDeserializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq
typedef struct {
int64_t dbUid;
char db[TSDB_DB_FNAME_LEN];
int64_t reserved[8];
int64_t compactStartTime;
} SCompactVnodeReq;
int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
@ -1781,6 +1783,7 @@ typedef struct {
#define STREAM_FILL_HISTORY_ON 1
#define STREAM_FILL_HISTORY_OFF 0
#define STREAM_DEFAULT_FILL_HISTORY STREAM_FILL_HISTORY_OFF
#define STREAM_DEFAULT_IGNORE_UPDATE 0
#define STREAM_CREATE_STABLE_TRUE 1
#define STREAM_CREATE_STABLE_FALSE 0
@ -1810,6 +1813,7 @@ typedef struct {
int8_t createStb;
uint64_t targetStbUid;
SArray* fillNullCols; // array of SColLocation
int8_t igUpdate;
} SCMCreateStreamReq;
typedef struct {

530
include/common/ttokendef.h
View File

@ -77,270 +77,272 @@
#define TK_USE 59
#define TK_FLUSH 60
#define TK_TRIM 61
#define TK_IF 62
#define TK_NOT 63
#define TK_EXISTS 64
#define TK_BUFFER 65
#define TK_CACHEMODEL 66
#define TK_CACHESIZE 67
#define TK_COMP 68
#define TK_DURATION 69
#define TK_NK_VARIABLE 70
#define TK_MAXROWS 71
#define TK_MINROWS 72
#define TK_KEEP 73
#define TK_PAGES 74
#define TK_PAGESIZE 75
#define TK_TSDB_PAGESIZE 76
#define TK_PRECISION 77
#define TK_REPLICA 78
#define TK_VGROUPS 79
#define TK_SINGLE_STABLE 80
#define TK_RETENTIONS 81
#define TK_SCHEMALESS 82
#define TK_WAL_LEVEL 83
#define TK_WAL_FSYNC_PERIOD 84
#define TK_WAL_RETENTION_PERIOD 85
#define TK_WAL_RETENTION_SIZE 86
#define TK_WAL_ROLL_PERIOD 87
#define TK_WAL_SEGMENT_SIZE 88
#define TK_STT_TRIGGER 89
#define TK_TABLE_PREFIX 90
#define TK_TABLE_SUFFIX 91
#define TK_NK_COLON 92
#define TK_MAX_SPEED 93
#define TK_TABLE 94
#define TK_NK_LP 95
#define TK_NK_RP 96
#define TK_STABLE 97
#define TK_ADD 98
#define TK_COLUMN 99
#define TK_MODIFY 100
#define TK_RENAME 101
#define TK_TAG 102
#define TK_SET 103
#define TK_NK_EQ 104
#define TK_USING 105
#define TK_TAGS 106
#define TK_COMMENT 107
#define TK_BOOL 108
#define TK_TINYINT 109
#define TK_SMALLINT 110
#define TK_INT 111
#define TK_INTEGER 112
#define TK_BIGINT 113
#define TK_FLOAT 114
#define TK_DOUBLE 115
#define TK_BINARY 116
#define TK_TIMESTAMP 117
#define TK_NCHAR 118
#define TK_UNSIGNED 119
#define TK_JSON 120
#define TK_VARCHAR 121
#define TK_MEDIUMBLOB 122
#define TK_BLOB 123
#define TK_VARBINARY 124
#define TK_DECIMAL 125
#define TK_MAX_DELAY 126
#define TK_WATERMARK 127
#define TK_ROLLUP 128
#define TK_TTL 129
#define TK_SMA 130
#define TK_DELETE_MARK 131
#define TK_FIRST 132
#define TK_LAST 133
#define TK_SHOW 134
#define TK_PRIVILEGES 135
#define TK_DATABASES 136
#define TK_TABLES 137
#define TK_STABLES 138
#define TK_MNODES 139
#define TK_QNODES 140
#define TK_FUNCTIONS 141
#define TK_INDEXES 142
#define TK_ACCOUNTS 143
#define TK_APPS 144
#define TK_CONNECTIONS 145
#define TK_LICENCES 146
#define TK_GRANTS 147
#define TK_QUERIES 148
#define TK_SCORES 149
#define TK_TOPICS 150
#define TK_VARIABLES 151
#define TK_CLUSTER 152
#define TK_BNODES 153
#define TK_SNODES 154
#define TK_TRANSACTIONS 155
#define TK_DISTRIBUTED 156
#define TK_CONSUMERS 157
#define TK_SUBSCRIPTIONS 158
#define TK_VNODES 159
#define TK_ALIVE 160
#define TK_LIKE 161
#define TK_TBNAME 162
#define TK_QTAGS 163
#define TK_AS 164
#define TK_INDEX 165
#define TK_FUNCTION 166
#define TK_INTERVAL 167
#define TK_COUNT 168
#define TK_LAST_ROW 169
#define TK_TOPIC 170
#define TK_WITH 171
#define TK_META 172
#define TK_CONSUMER 173
#define TK_GROUP 174
#define TK_DESC 175
#define TK_DESCRIBE 176
#define TK_RESET 177
#define TK_QUERY 178
#define TK_CACHE 179
#define TK_EXPLAIN 180
#define TK_ANALYZE 181
#define TK_VERBOSE 182
#define TK_NK_BOOL 183
#define TK_RATIO 184
#define TK_NK_FLOAT 185
#define TK_OUTPUTTYPE 186
#define TK_AGGREGATE 187
#define TK_BUFSIZE 188
#define TK_STREAM 189
#define TK_INTO 190
#define TK_TRIGGER 191
#define TK_AT_ONCE 192
#define TK_WINDOW_CLOSE 193
#define TK_IGNORE 194
#define TK_EXPIRED 195
#define TK_FILL_HISTORY 196
#define TK_SUBTABLE 197
#define TK_KILL 198
#define TK_CONNECTION 199
#define TK_TRANSACTION 200
#define TK_BALANCE 201
#define TK_VGROUP 202
#define TK_MERGE 203
#define TK_REDISTRIBUTE 204
#define TK_SPLIT 205
#define TK_DELETE 206
#define TK_INSERT 207
#define TK_NULL 208
#define TK_NK_QUESTION 209
#define TK_NK_ARROW 210
#define TK_ROWTS 211
#define TK_QSTART 212
#define TK_QEND 213
#define TK_QDURATION 214
#define TK_WSTART 215
#define TK_WEND 216
#define TK_WDURATION 217
#define TK_IROWTS 218
#define TK_ISFILLED 219
#define TK_CAST 220
#define TK_NOW 221
#define TK_TODAY 222
#define TK_TIMEZONE 223
#define TK_CLIENT_VERSION 224
#define TK_SERVER_VERSION 225
#define TK_SERVER_STATUS 226
#define TK_CURRENT_USER 227
#define TK_CASE 228
#define TK_END 229
#define TK_WHEN 230
#define TK_THEN 231
#define TK_ELSE 232
#define TK_BETWEEN 233
#define TK_IS 234
#define TK_NK_LT 235
#define TK_NK_GT 236
#define TK_NK_LE 237
#define TK_NK_GE 238
#define TK_NK_NE 239
#define TK_MATCH 240
#define TK_NMATCH 241
#define TK_CONTAINS 242
#define TK_IN 243
#define TK_JOIN 244
#define TK_INNER 245
#define TK_SELECT 246
#define TK_DISTINCT 247
#define TK_WHERE 248
#define TK_PARTITION 249
#define TK_BY 250
#define TK_SESSION 251
#define TK_STATE_WINDOW 252
#define TK_EVENT_WINDOW 253
#define TK_START 254
#define TK_SLIDING 255
#define TK_FILL 256
#define TK_VALUE 257
#define TK_NONE 258
#define TK_PREV 259
#define TK_LINEAR 260
#define TK_NEXT 261
#define TK_HAVING 262
#define TK_RANGE 263
#define TK_EVERY 264
#define TK_ORDER 265
#define TK_SLIMIT 266
#define TK_SOFFSET 267
#define TK_LIMIT 268
#define TK_OFFSET 269
#define TK_ASC 270
#define TK_NULLS 271
#define TK_ABORT 272
#define TK_AFTER 273
#define TK_ATTACH 274
#define TK_BEFORE 275
#define TK_BEGIN 276
#define TK_BITAND 277
#define TK_BITNOT 278
#define TK_BITOR 279
#define TK_BLOCKS 280
#define TK_CHANGE 281
#define TK_COMMA 282
#define TK_COMPACT 283
#define TK_CONCAT 284
#define TK_CONFLICT 285
#define TK_COPY 286
#define TK_DEFERRED 287
#define TK_DELIMITERS 288
#define TK_DETACH 289
#define TK_DIVIDE 290
#define TK_DOT 291
#define TK_EACH 292
#define TK_FAIL 293
#define TK_FILE 294
#define TK_FOR 295
#define TK_GLOB 296
#define TK_ID 297
#define TK_IMMEDIATE 298
#define TK_IMPORT 299
#define TK_INITIALLY 300
#define TK_INSTEAD 301
#define TK_ISNULL 302
#define TK_KEY 303
#define TK_MODULES 304
#define TK_NK_BITNOT 305
#define TK_NK_SEMI 306
#define TK_NOTNULL 307
#define TK_OF 308
#define TK_PLUS 309
#define TK_PRIVILEGE 310
#define TK_RAISE 311
#define TK_REPLACE 312
#define TK_RESTRICT 313
#define TK_ROW 314
#define TK_SEMI 315
#define TK_STAR 316
#define TK_STATEMENT 317
#define TK_STRICT 318
#define TK_STRING 319
#define TK_TIMES 320
#define TK_UPDATE 321
#define TK_VALUES 322
#define TK_VARIABLE 323
#define TK_VIEW 324
#define TK_WAL 325
#define TK_COMPACT 62
#define TK_IF 63
#define TK_NOT 64
#define TK_EXISTS 65
#define TK_BUFFER 66
#define TK_CACHEMODEL 67
#define TK_CACHESIZE 68
#define TK_COMP 69
#define TK_DURATION 70
#define TK_NK_VARIABLE 71
#define TK_MAXROWS 72
#define TK_MINROWS 73
#define TK_KEEP 74
#define TK_PAGES 75
#define TK_PAGESIZE 76
#define TK_TSDB_PAGESIZE 77
#define TK_PRECISION 78
#define TK_REPLICA 79
#define TK_VGROUPS 80
#define TK_SINGLE_STABLE 81
#define TK_RETENTIONS 82
#define TK_SCHEMALESS 83
#define TK_WAL_LEVEL 84
#define TK_WAL_FSYNC_PERIOD 85
#define TK_WAL_RETENTION_PERIOD 86
#define TK_WAL_RETENTION_SIZE 87
#define TK_WAL_ROLL_PERIOD 88
#define TK_WAL_SEGMENT_SIZE 89
#define TK_STT_TRIGGER 90
#define TK_TABLE_PREFIX 91
#define TK_TABLE_SUFFIX 92
#define TK_NK_COLON 93
#define TK_MAX_SPEED 94
#define TK_TABLE 95
#define TK_NK_LP 96
#define TK_NK_RP 97
#define TK_STABLE 98
#define TK_ADD 99
#define TK_COLUMN 100
#define TK_MODIFY 101
#define TK_RENAME 102
#define TK_TAG 103
#define TK_SET 104
#define TK_NK_EQ 105
#define TK_USING 106
#define TK_TAGS 107
#define TK_COMMENT 108
#define TK_BOOL 109
#define TK_TINYINT 110
#define TK_SMALLINT 111
#define TK_INT 112
#define TK_INTEGER 113
#define TK_BIGINT 114
#define TK_FLOAT 115
#define TK_DOUBLE 116
#define TK_BINARY 117
#define TK_TIMESTAMP 118
#define TK_NCHAR 119
#define TK_UNSIGNED 120
#define TK_JSON 121
#define TK_VARCHAR 122
#define TK_MEDIUMBLOB 123
#define TK_BLOB 124
#define TK_VARBINARY 125
#define TK_DECIMAL 126
#define TK_MAX_DELAY 127
#define TK_WATERMARK 128
#define TK_ROLLUP 129
#define TK_TTL 130
#define TK_SMA 131
#define TK_DELETE_MARK 132
#define TK_FIRST 133
#define TK_LAST 134
#define TK_SHOW 135
#define TK_PRIVILEGES 136
#define TK_DATABASES 137
#define TK_TABLES 138
#define TK_STABLES 139
#define TK_MNODES 140
#define TK_QNODES 141
#define TK_FUNCTIONS 142
#define TK_INDEXES 143
#define TK_ACCOUNTS 144
#define TK_APPS 145
#define TK_CONNECTIONS 146
#define TK_LICENCES 147
#define TK_GRANTS 148
#define TK_QUERIES 149
#define TK_SCORES 150
#define TK_TOPICS 151
#define TK_VARIABLES 152
#define TK_CLUSTER 153
#define TK_BNODES 154
#define TK_SNODES 155
#define TK_TRANSACTIONS 156
#define TK_DISTRIBUTED 157
#define TK_CONSUMERS 158
#define TK_SUBSCRIPTIONS 159
#define TK_VNODES 160
#define TK_ALIVE 161
#define TK_LIKE 162
#define TK_TBNAME 163
#define TK_QTAGS 164
#define TK_AS 165
#define TK_INDEX 166
#define TK_FUNCTION 167
#define TK_INTERVAL 168
#define TK_COUNT 169
#define TK_LAST_ROW 170
#define TK_TOPIC 171
#define TK_WITH 172
#define TK_META 173
#define TK_CONSUMER 174
#define TK_GROUP 175
#define TK_DESC 176
#define TK_DESCRIBE 177
#define TK_RESET 178
#define TK_QUERY 179
#define TK_CACHE 180
#define TK_EXPLAIN 181
#define TK_ANALYZE 182
#define TK_VERBOSE 183
#define TK_NK_BOOL 184
#define TK_RATIO 185
#define TK_NK_FLOAT 186
#define TK_OUTPUTTYPE 187
#define TK_AGGREGATE 188
#define TK_BUFSIZE 189
#define TK_STREAM 190
#define TK_INTO 191
#define TK_TRIGGER 192
#define TK_AT_ONCE 193
#define TK_WINDOW_CLOSE 194
#define TK_IGNORE 195
#define TK_EXPIRED 196
#define TK_FILL_HISTORY 197
#define TK_UPDATE 198
#define TK_SUBTABLE 199
#define TK_KILL 200
#define TK_CONNECTION 201
#define TK_TRANSACTION 202
#define TK_BALANCE 203
#define TK_VGROUP 204
#define TK_MERGE 205
#define TK_REDISTRIBUTE 206
#define TK_SPLIT 207
#define TK_DELETE 208
#define TK_INSERT 209
#define TK_NULL 210
#define TK_NK_QUESTION 211
#define TK_NK_ARROW 212
#define TK_ROWTS 213
#define TK_QSTART 214
#define TK_QEND 215
#define TK_QDURATION 216
#define TK_WSTART 217
#define TK_WEND 218
#define TK_WDURATION 219
#define TK_IROWTS 220
#define TK_ISFILLED 221
#define TK_CAST 222
#define TK_NOW 223
#define TK_TODAY 224
#define TK_TIMEZONE 225
#define TK_CLIENT_VERSION 226
#define TK_SERVER_VERSION 227
#define TK_SERVER_STATUS 228
#define TK_CURRENT_USER 229
#define TK_CASE 230
#define TK_END 231
#define TK_WHEN 232
#define TK_THEN 233
#define TK_ELSE 234
#define TK_BETWEEN 235
#define TK_IS 236
#define TK_NK_LT 237
#define TK_NK_GT 238
#define TK_NK_LE 239
#define TK_NK_GE 240
#define TK_NK_NE 241
#define TK_MATCH 242
#define TK_NMATCH 243
#define TK_CONTAINS 244
#define TK_IN 245
#define TK_JOIN 246
#define TK_INNER 247
#define TK_SELECT 248
#define TK_DISTINCT 249
#define TK_WHERE 250
#define TK_PARTITION 251
#define TK_BY 252
#define TK_SESSION 253
#define TK_STATE_WINDOW 254
#define TK_EVENT_WINDOW 255
#define TK_START 256
#define TK_SLIDING 257
#define TK_FILL 258
#define TK_VALUE 259
#define TK_VALUE_F 260
#define TK_NONE 261
#define TK_PREV 262
#define TK_NULL_F 263
#define TK_LINEAR 264
#define TK_NEXT 265
#define TK_HAVING 266
#define TK_RANGE 267
#define TK_EVERY 268
#define TK_ORDER 269
#define TK_SLIMIT 270
#define TK_SOFFSET 271
#define TK_LIMIT 272
#define TK_OFFSET 273
#define TK_ASC 274
#define TK_NULLS 275
#define TK_ABORT 276
#define TK_AFTER 277
#define TK_ATTACH 278
#define TK_BEFORE 279
#define TK_BEGIN 280
#define TK_BITAND 281
#define TK_BITNOT 282
#define TK_BITOR 283
#define TK_BLOCKS 284
#define TK_CHANGE 285
#define TK_COMMA 286
#define TK_CONCAT 287
#define TK_CONFLICT 288
#define TK_COPY 289
#define TK_DEFERRED 290
#define TK_DELIMITERS 291
#define TK_DETACH 292
#define TK_DIVIDE 293
#define TK_DOT 294
#define TK_EACH 295
#define TK_FAIL 296
#define TK_FILE 297
#define TK_FOR 298
#define TK_GLOB 299
#define TK_ID 300
#define TK_IMMEDIATE 301
#define TK_IMPORT 302
#define TK_INITIALLY 303
#define TK_INSTEAD 304
#define TK_ISNULL 305
#define TK_KEY 306
#define TK_MODULES 307
#define TK_NK_BITNOT 308
#define TK_NK_SEMI 309
#define TK_NOTNULL 310
#define TK_OF 311
#define TK_PLUS 312
#define TK_PRIVILEGE 313
#define TK_RAISE 314
#define TK_REPLACE 315
#define TK_RESTRICT 316
#define TK_ROW 317
#define TK_SEMI 318
#define TK_STAR 319
#define TK_STATEMENT 320
#define TK_STRICT 321
#define TK_STRING 322
#define TK_TIMES 323
#define TK_VALUES 324
#define TK_VARIABLE 325
#define TK_VIEW 326
#define TK_WAL 327
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

27
include/libs/cache/cache.h vendored
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_CACHE_H_
#define _TD_CACHE_H_
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_CACHE_H_*/

1
include/libs/function/functionMgt.h
View File

@ -219,6 +219,7 @@ bool fmIsKeepOrderFunc(int32_t funcId);
bool fmIsCumulativeFunc(int32_t funcId);
bool fmIsInterpPseudoColumnFunc(int32_t funcId);
bool fmIsGroupKeyFunc(int32_t funcId);
bool fmIsBlockDistFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType);

6
include/libs/nodes/cmdnodes.h
View File

@ -124,6 +124,11 @@ typedef struct STrimDatabaseStmt {
int32_t maxSpeed;
} STrimDatabaseStmt;
typedef struct SCompactDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
} SCompactDatabaseStmt;
typedef struct STableOptions {
ENodeType type;
bool commentNull;
@ -395,6 +400,7 @@ typedef struct SStreamOptions {
SNode* pDeleteMark;
int8_t fillHistory;
int8_t ignoreExpired;
int8_t ignoreUpdate;
} SStreamOptions;
typedef struct SCreateStreamStmt {

2
include/libs/nodes/nodes.h
View File

@ -155,7 +155,7 @@ typedef enum ENodeType {
QUERY_NODE_EXPLAIN_STMT,
QUERY_NODE_DESCRIBE_STMT,
QUERY_NODE_RESET_QUERY_CACHE_STMT,
QUERY_NODE_COMPACT_STMT,
QUERY_NODE_COMPACT_DATABASE_STMT,
QUERY_NODE_CREATE_FUNCTION_STMT,
QUERY_NODE_DROP_FUNCTION_STMT,
QUERY_NODE_CREATE_STREAM_STMT,

4
include/libs/nodes/plannodes.h
View File

@ -93,6 +93,7 @@ typedef struct SScanLogicNode {
int64_t watermark;
int64_t deleteMark;
int8_t igExpired;
int8_t igCheckUpdate;
SArray* pSmaIndexes;
SNodeList* pGroupTags;
bool groupSort;
@ -224,6 +225,7 @@ typedef struct SWindowLogicNode {
int64_t watermark;
int64_t deleteMark;
int8_t igExpired;
int8_t igCheckUpdate;
EWindowAlgorithm windowAlgo;
EOrder inputTsOrder;
EOrder outputTsOrder;
@ -364,6 +366,7 @@ typedef struct STableScanPhysiNode {
int64_t watermark;
int8_t igExpired;
bool assignBlockUid;
int8_t igCheckUpdate;
} STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode;
@ -562,6 +565,7 @@ typedef struct SQueryInserterNode {
char tableName[TSDB_TABLE_NAME_LEN];
int32_t vgId;
SEpSet epSet;
bool explain;
} SQueryInserterNode;
typedef struct SDataDeleterNode {

2
include/libs/nodes/querynodes.h
View File

@ -233,8 +233,10 @@ typedef struct SEventWindowNode {
typedef enum EFillMode {
FILL_MODE_NONE = 1,
FILL_MODE_VALUE,
FILL_MODE_VALUE_F,
FILL_MODE_PREV,
FILL_MODE_NULL,
FILL_MODE_NULL_F,
FILL_MODE_LINEAR,
FILL_MODE_NEXT
} EFillMode;

1
include/libs/planner/planner.h
View File

@ -36,6 +36,7 @@ typedef struct SPlanContext {
int64_t watermark;
int64_t deleteMark;
int8_t igExpired;
int8_t igCheckUpdate;
char* pMsg;
int32_t msgLen;
const char* pUser;

1
include/libs/wal/wal.h
View File

@ -191,6 +191,7 @@ int32_t walApplyVer(SWal *, int64_t ver);
// read
SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond);
void walCloseReader(SWalReader *pRead);
void walReadReset(SWalReader *pReader);
int32_t walReadVer(SWalReader *pRead, int64_t ver);
int32_t walReadSeekVer(SWalReader *pRead, int64_t ver);
int32_t walNextValidMsg(SWalReader *pRead);

4
include/os/os.h
View File

@ -27,7 +27,11 @@ extern "C" {
#if !defined(WINDOWS)
#include <dirent.h>
#if !defined(_ALPINE)
#include <execinfo.h>
#endif
#include <libgen.h>
#include <sched.h>
#include <unistd.h>

2
include/os/osFile.h
View File

@ -110,6 +110,8 @@ bool taosValidFile(TdFilePtr pFile);
int32_t taosGetErrorFile(TdFilePtr pFile);
int32_t taosCompressFile(char *srcFileName, char *destFileName);
#ifdef __cplusplus
}
#endif

7
include/os/osSysinfo.h
View File

@ -60,6 +60,13 @@ void taosSetCoreDump(bool enable);
#endif // WINDOWS
#if defined(_ALPINE)
#define _UTSNAME_LENGTH 65
#define _UTSNAME_MACHINE_LENGTH _UTSNAME_LENGTH
#endif
typedef struct {
char sysname[_UTSNAME_MACHINE_LENGTH];
char nodename[_UTSNAME_MACHINE_LENGTH];

144
include/os/osSystem.h
View File

@ -16,6 +16,11 @@
#ifndef _TD_OS_SYSTEM_H_
#define _TD_OS_SYSTEM_H_
#ifdef _ALPINE
#define UNW_LOCAL_ONLY
#include <libunwind.h>
#endif
#ifdef __cplusplus
extern "C" {
#endif
@ -29,46 +34,124 @@ extern "C" {
#define tcgetattr TCGETATTR_FUNC_TAOS_FORBID
#endif
typedef struct TdCmd* TdCmdPtr;
typedef struct TdCmd *TdCmdPtr;
TdCmdPtr taosOpenCmd(const char *cmd);
int64_t taosGetsCmd(TdCmdPtr pCmd, int32_t maxSize, char *__restrict buf);
int64_t taosGetLineCmd(TdCmdPtr pCmd, char **__restrict ptrBuf);
TdCmdPtr taosOpenCmd(const char* cmd);
int64_t taosGetsCmd(TdCmdPtr pCmd, int32_t maxSize, char* __restrict buf);
int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf);
int32_t taosEOFCmd(TdCmdPtr pCmd);
int64_t taosCloseCmd(TdCmdPtr* ppCmd);
void* taosLoadDll(const char* filename);
void* taosLoadSym(void* handle, char* name);
void taosCloseDll(void* handle);
int64_t taosCloseCmd(TdCmdPtr *ppCmd);
void *taosLoadDll(const char *filename);
void *taosLoadSym(void *handle, char *name);
void taosCloseDll(void *handle);
int32_t taosSetConsoleEcho(bool on);
void taosSetTerminalMode();
int32_t taosGetOldTerminalMode();
void taosResetTerminalMode();
#define STACKSIZE 100
#if !defined(WINDOWS)
#define taosLogTraceToBuf(buf, bufSize, ignoreNum) { \
void* array[STACKSIZE]; \
#if defined(_ALPINE)
#define taosLogTraceToBuf(buf, bufSize, ignoreNum) \
{ \
unw_cursor_t cursor; \
unw_context_t context; \
\
unw_getcontext(&context); \
unw_init_local(&cursor, &context); \
\
char *array[STACKSIZE]; \
int32_t size = 0; \
int32_t ignores = ignoreNum; \
int32_t offset = 0; \
while (unw_step(&cursor) > 0 && size < STACKSIZE) { \
unw_word_t offset, pc; \
char fname[64]; \
unw_get_reg(&cursor, UNW_REG_IP, &pc); \
fname[0] = '\0'; \
(void)unw_get_proc_name(&cursor, fname, sizeof(fname), &offset); \
size += 1; \
array[size] = (char *)taosMemoryMalloc(sizeof(char) * STACKSIZE + 1); \
snprintf(array[size], STACKSIZE, "0x%lx : (%s+0x%lx) [0x%lx]\n", (long)pc, fname, (long)offset, (long)pc); \
} \
if (ignoreNum < size && size > 0) { \
offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \
for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \
offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, \
array[i]); \
} \
} \
for (int i = 0; i < size; i++) { \
taosMemoryFree(array[i]); \
} \
}
#define taosPrintTrace(flags, level, dflag, ignoreNum) \
{ \
unw_cursor_t cursor; \
unw_context_t context; \
\
unw_getcontext(&context); \
unw_init_local(&cursor, &context); \
\
char *array[STACKSIZE]; \
int32_t size = 0; \
while (unw_step(&cursor) > 0 && size < STACKSIZE) { \
unw_word_t offset, pc; \
char fname[64]; \
unw_get_reg(&cursor, UNW_REG_IP, &pc); \
fname[0] = '\0'; \
(void)unw_get_proc_name(&cursor, fname, sizeof(fname), &offset); \
size += 1; \
array[size] = (char *)taosMemoryMalloc(sizeof(char) * STACKSIZE + 1); \
snprintf(array[size], STACKSIZE, "frame:%d, 0x%lx : (%s+0x%lx) [0x%lx]\n", size, (long)pc, fname, (long)offset, \
(long)pc); \
} \
if (ignoreNum < size && size > 0) { \
taosPrintLog(flags, level, dflag, "obtained %d stack frames", (ignoreNum > 0) ? size - ignoreNum : size); \
for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \
taosPrintLog(flags, level, dflag, "frame:%d, %s", (ignoreNum > 0) ? i - ignoreNum : i, array[i]); \
} \
} \
for (int i = 0; i < size; i++) { \
taosMemoryFree(array[i]); \
} \
}
#elif !defined(WINDOWS)
#define taosLogTraceToBuf(buf, bufSize, ignoreNum) \
{ \
void *array[STACKSIZE]; \
int32_t size = backtrace(array, STACKSIZE); \
char** strings = backtrace_symbols(array, size); \
char **strings = backtrace_symbols(array, size); \
int32_t offset = 0; \
if (strings != NULL) { \
offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \
for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \
offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, strings[i]); \
offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, \
strings[i]); \
} \
} \
\
taosMemoryFree(strings); \
}
}
#define taosPrintTrace(flags, level, dflag, ignoreNum) \
{ \
void* array[STACKSIZE]; \
void *array[STACKSIZE]; \
int32_t size = backtrace(array, STACKSIZE); \
char** strings = backtrace_symbols(array, size); \
char **strings = backtrace_symbols(array, size); \
if (strings != NULL) { \
taosPrintLog(flags, level, dflag, "obtained %d stack frames", (ignoreNum > 0) ? size - ignoreNum : size); \
for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \
@ -80,14 +163,15 @@ void taosResetTerminalMode();
}
#else
#include <windows.h>
#include <dbghelp.h>
#include <windows.h>
#define taosLogTraceToBuf(buf, bufSize, ignoreNum) { \
#define taosLogTraceToBuf(buf, bufSize, ignoreNum) \
{ \
unsigned int i; \
void* stack[STACKSIZE]; \
void *stack[STACKSIZE]; \
unsigned short frames; \
SYMBOL_INFO* symbol; \
SYMBOL_INFO *symbol; \
HANDLE process; \
int32_t offset = 0; \
\
@ -96,16 +180,18 @@ void taosResetTerminalMode();
SymInitialize(process, NULL, TRUE); \
\
frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \
symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \
symbol = (SYMBOL_INFO *)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \
if (symbol != NULL) { \
symbol->MaxNameLen = 255; \
symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \
\
if (frames > 0) { \
offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? frames - ignoreNum : frames); \
offset = \
snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? frames - ignoreNum : frames); \
for (i = (ignoreNum > 0) ? ignoreNum : 0; i < frames; i++) { \
SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \
offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%i, %s - 0x%0X\n", (ignoreNum > 0) ? i - ignoreNum : i, symbol->Name, symbol->Address); \
offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%i, %s - 0x%0X\n", \
(ignoreNum > 0) ? i - ignoreNum : i, symbol->Name, symbol->Address); \
} \
} \
free(symbol); \
@ -115,9 +201,9 @@ void taosResetTerminalMode();
#define taosPrintTrace(flags, level, dflag, ignoreNum) \
{ \
unsigned int i; \
void* stack[STACKSIZE]; \
void *stack[STACKSIZE]; \
unsigned short frames; \
SYMBOL_INFO* symbol; \
SYMBOL_INFO *symbol; \
HANDLE process; \
\
process = GetCurrentProcess(); \
@ -125,16 +211,18 @@ void taosResetTerminalMode();
SymInitialize(process, NULL, TRUE); \
\
frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \
symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \
symbol = (SYMBOL_INFO *)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \
if (symbol != NULL) { \
symbol->MaxNameLen = 255; \
symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \
\
if (frames > 0) { \
taosPrintLog(flags, level, dflag, "obtained %d stack frames\n", (ignoreNum > 0) ? frames - ignoreNum : frames); \
taosPrintLog(flags, level, dflag, "obtained %d stack frames\n", \
(ignoreNum > 0) ? frames - ignoreNum : frames); \
for (i = (ignoreNum > 0) ? ignoreNum : 0; i < frames; i++) { \
SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \
taosPrintLog(flags, level, dflag, "frame:%i, %s - 0x%0X\n", (ignoreNum > 0) ? i - ignoreNum : i, symbol->Name, symbol->Address); \
taosPrintLog(flags, level, dflag, "frame:%i, %s - 0x%0X\n", (ignoreNum > 0) ? i - ignoreNum : i, \
symbol->Name, symbol->Address); \
} \
} \
free(symbol); \

6
include/os/osThread.h
View File

@ -22,7 +22,7 @@
extern "C" {
#endif
#ifndef WINDOWS
#if !defined(WINDOWS) && !defined(_ALPINE)
#ifndef __USE_XOPEN2K
#define TD_USE_SPINLOCK_AS_MUTEX
typedef pthread_mutex_t pthread_spinlock_t;
@ -100,7 +100,11 @@ typedef pthread_key_t TdThreadKey;
#define pthread_condattr_init PTHREAD_CONDATTR_INIT_FUNC_TAOS_FORBID
#define pthread_condattr_setpshared PTHREAD_CONDATTR_SETPSHARED_FUNC_TAOS_FORBID
#define pthread_detach PTHREAD_DETACH_FUNC_TAOS_FORBID
#if !defined(_ALPINE)
#define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID
#endif
#define pthread_exit PTHREAD_EXIT_FUNC_TAOS_FORBID
#define pthread_getschedparam PTHREAD_GETSCHEDPARAM_FUNC_TAOS_FORBID
#define pthread_getspecific PTHREAD_GETSPECIFIC_FUNC_TAOS_FORBID

2
include/util/taoserror.h
View File

@ -111,6 +111,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x012A)
#define TSDB_CODE_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012B)
#define TSDB_CODE_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x012C)
#define TSDB_CODE_MSG_ENCODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x012D)
#define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) //
#define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) //
@ -357,6 +358,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_STREAM_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03F3)
#define TSDB_CODE_MND_STREAM_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x03F4)
#define TSDB_CODE_MND_MULTI_REPLICA_SOURCE_DB TAOS_DEF_ERROR_CODE(0, 0x03F5)
#define TSDB_CODE_MND_TOO_MANY_STREAMS TAOS_DEF_ERROR_CODE(0, 0x03F6)
// mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)

3
include/util/trbtree.h
View File

@ -33,9 +33,12 @@ typedef int32_t (*tRBTreeCmprFn)(const SRBTreeNode *, const SRBTreeNode *);
#define tRBTreeMax(T) ((T)->max == ((T)->NIL) ? NULL : (T)->max)
void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn);
void tRBTreeClear(SRBTree *pTree);
SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z);
void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z);
SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey);
SRBTreeNode *tRBTreeDropMin(SRBTree *pTree);
SRBTreeNode *tRBTreeDropMax(SRBTree *pTree);
SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode);
// SRBTreeIter =============================================

55
packaging/tools/install.sh
View File

@ -35,6 +35,20 @@ dumpName="taosdump"
demoName="taosdemo"
xname="taosx"
clientName2="taos"
serverName2="taosd"
productName2="TDengine"
emailName2="taosdata.com"
benchmarkName2="${clientName2}Benchmark"
dumpName2="${clientName2}dump"
uninstallScript2="rm${clientName2}"
historyFile="${clientName2}_history"
logDir="/var/log/${clientName2}"
configDir="/etc/${clientName2}"
installDir="/usr/local/${clientName}"
data_dir=${dataDir}
log_dir=${logDir}
cfg_install_dir=${configDir}
@ -217,6 +231,13 @@ function install_bin() {
[ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || :
[ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName2} || :
[ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName2} || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || :
fi
}
function install_lib() {
@ -518,7 +539,7 @@ function install_config() {
local_fqdn_check
echo
echo -e -n "${GREEN}Enter FQDN:port (like h1.${emailName}:6030) of an existing ${productName} cluster node to join${NC}"
echo -e -n "${GREEN}Enter FQDN:port (like h1.${emailName2}:6030) of an existing ${productName2} cluster node to join${NC}"
echo
echo -e -n "${GREEN}OR leave it blank to build one${NC}:"
read firstEp
@ -655,6 +676,9 @@ function clean_service_on_systemd() {
fi
${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null
${csudo}rm -f ${tarbitratord_service_config}
# if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
# ${csudo}rm -f ${service_config_dir}/${serverName2}.service
# fi
}
function install_service_on_systemd() {
@ -663,6 +687,13 @@ function install_service_on_systemd() {
[ -f ${script_dir}/cfg/${serverName}.service ] &&
${csudo}cp ${script_dir}/cfg/${serverName}.service \
${service_config_dir}/ || :
# if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
# [ -f ${script_dir}/cfg/${serverName}.service ] &&
# ${csudo}cp ${script_dir}/cfg/${serverName}.service \
# ${service_config_dir}/${serverName2}.service || :
# fi
${csudo}systemctl daemon-reload
${csudo}systemctl enable ${serverName}
@ -793,7 +824,7 @@ function updateProduct() {
tar -zxf ${tarName}
install_jemalloc
echo -e "${GREEN}Start to update ${productName}...${NC}"
echo -e "${GREEN}Start to update ${productName2}...${NC}"
# Stop the service if running
if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then
if ((${service_mod} == 0)); then
@ -830,25 +861,25 @@ function updateProduct() {
echo
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}"
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml"
echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml"
if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}"
echo -e "${GREEN_DARK}To start Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}"
elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}"
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}"
else
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}"
[ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}"
echo -e "${GREEN_DARK}To start ${clientName} Adapter ${NC}: taosadapter &${NC}"
fi
if [ ${openresty_work} = 'true' ]; then
echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}"
echo -e "${GREEN_DARK}To access ${productName2} ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}"
else
echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}"
echo -e "${GREEN_DARK}To access ${productName2} ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}"
fi
if ((${prompt_force} == 1)); then
@ -856,13 +887,13 @@ function updateProduct() {
echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName} $exist_version!${NC}"
fi
echo
echo -e "\033[44;32;1m${productName} is updated successfully!${NC}"
echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}"
else
install_bin
install_config
echo
echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}"
echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}"
fi
rm -rf $(tar -tf ${tarName} | grep -v "^\./$")
@ -876,7 +907,7 @@ function installProduct() {
fi
tar -zxf ${tarName}
echo -e "${GREEN}Start to install ${productName}...${NC}"
echo -e "${GREEN}Start to install ${productName2}...${NC}"
install_main_path
@ -965,7 +996,7 @@ serverFqdn=$(hostname)
if [ "$verType" == "server" ]; then
# Check default 2.x data file.
if [ -x ${data_dir}/dnode/dnodeCfg.json ]; then
echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of tdengine 2.x, please clear it before installing!\033[0m"
echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of ${productName2} 2.x, please clear it before installing!\033[0m"
else
# Install server and client
if [ -x ${bin_dir}/${serverName} ]; then

31
packaging/tools/install_client.sh
View File

@ -23,6 +23,16 @@ osType=Linux
pagMode=full
verMode=edge
clientName2="taos"
serverName2="taosd"
productName2="TDengine"
emailName2="taosdata.com"
benchmarkName2="${clientName2}Benchmark"
dumpName2="${clientName2}dump"
demoName2="${clientName2}demo"
uninstallScript2="rm${clientName2}"
if [ "$osType" != "Darwin" ]; then
script_dir=$(dirname $(readlink -f "$0"))
# Dynamic directory
@ -112,6 +122,15 @@ function install_bin() {
fi
[ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript} || :
[ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
#Make link
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || :
if [ "$osType" != "Darwin" ]; then
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/${demoName2} || :
fi
[ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || :
fi
}
function clean_lib() {
@ -263,9 +282,9 @@ function update_TDengine() {
exit 1
fi
tar -zxf ${tarName}
echo -e "${GREEN}Start to update ${productName} client...${NC}"
echo -e "${GREEN}Start to update ${productName2} client...${NC}"
# Stop the client shell if running
if ps aux | grep -v grep | grep ${clientName} &> /dev/null; then
if ps aux | grep -v grep | grep ${clientName2} &> /dev/null; then
kill_client
sleep 1
fi
@ -284,7 +303,7 @@ function update_TDengine() {
install_config
echo
echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}"
echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}"
rm -rf $(tar -tf ${tarName})
}
@ -296,7 +315,7 @@ function install_TDengine() {
exit 1
fi
tar -zxf ${tarName}
echo -e "${GREEN}Start to install ${productName} client...${NC}"
echo -e "${GREEN}Start to install ${productName2} client...${NC}"
install_main_path
install_log
@ -311,7 +330,7 @@ function install_TDengine() {
install_config
echo
echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}"
echo -e "\033[44;32;1m${productName2} client is installed successfully!${NC}"
rm -rf $(tar -tf ${tarName})
}
@ -321,7 +340,7 @@ function install_TDengine() {
# Install or updata client and client
# if server is already install, don't install client
if [ -e ${bin_dir}/${serverName} ]; then
echo -e "\033[44;32;1mThere are already installed ${productName} server, so don't need install client!${NC}"
echo -e "\033[44;32;1mThere are already installed ${productName2} server, so don't need install client!${NC}"
exit 0
fi

57
packaging/tools/make_install.bat
View File

@ -2,12 +2,20 @@
for /F %%a in ('echo prompt $E ^| cmd') do set "ESC=%%a"
goto %1
if "%1" NEQ "" goto %1
:needAdmin
if exist C:\\TDengine\\data\\dnode\\dnodeCfg.json (
echo The default data directory C:/TDengine/data contains old data of tdengine 2.x, please clear it before installing!
)
rem // stop and delete service
mshta vbscript:createobject("shell.application").shellexecute("%~s0",":stop_delete","","runas",1)(window.close)
echo This might take a few moment to accomplish deleting service taosd/taosadapter ...
call :check_svc taosd
call :check_svc taosadapter
set source_dir=%2
set source_dir=%source_dir:/=\\%
set binary_dir=%3
@ -87,7 +95,6 @@ if %Enterprise% == TRUE (
copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul
copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul
if exist %binary_dir%\\build\\bin\\taosadapter.exe (
copy %binary_dir%\\build\\bin\\taosadapter.exe %target_dir% > nul
)
@ -107,22 +114,23 @@ goto :eof
:hasAdmin
sc query "taosd" && sc stop taosd && sc delete taosd
sc query "taosadapter" && sc stop taosadapter && sc delete taosd
call :stop_delete
call :check_svc taosd
call :check_svc taosadapter
copy /y C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 > nul
if exist C:\\TDengine\\driver\\taosws.dll (
copy /y C:\\TDengine\\driver\\taosws.dll C:\\Windows\\System32 > nul
)
sc query "taosd" >nul || sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND
sc query "taosadapter" >nul || sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND
rem // create services
sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND
sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND
set "env=HKLM\System\CurrentControlSet\Control\Session Manager\Environment"
for /f "tokens=2*" %%I in ('reg query "%env%" /v Path ^| findstr /i "\<Path\>"') do (
rem // make addition persistent through reboots
reg add "%env%" /f /v Path /t REG_EXPAND_SZ /d "%%J;C:\TDengine"
call :append_if_not_exists %%J
rem // apply change to the current process
for %%a in ("%%J;C:\TDengine") do path %%~a
@ -132,3 +140,36 @@ rem // use setx to set a temporary throwaway value to trigger a WM_SETTINGCHANGE
rem // applies change to new console windows without requiring a reboot
(setx /m foo bar & reg delete "%env%" /f /v foo) >NUL 2>NUL
goto :end
:append_if_not_exists
set "_origin_paths=%*"
set "_paths=%*"
set "_found=0"
:loop
for /f "tokens=1* delims=;" %%x in ("%_paths%") do (
if "%%x" EQU "C:\TDengine" (
set "_found=1"
) else (
set "_paths=%%y"
goto :loop
)
)
if "%_found%" == "0" (
rem // make addition persistent through reboots
reg add "%env%" /f /v Path /t REG_EXPAND_SZ /d "%_origin_paths%;C:\TDengine"
)
exit /B 0
:stop_delete
sc stop taosd
sc delete taosd
sc stop taosadapter
sc delete taosadapter
exit /B 0
:check_svc
sc query %1 >nul 2>nul && goto :check_svc %1
exit /B 0
:end

4
packaging/tools/make_install.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
#
# This file is used to install TAOS time-series database on linux systems. The operating system
# is required to use systemd to manage services at boot
@ -340,7 +340,7 @@ function install_lib() {
#install_avro lib64
if [ "$osType" != "Darwin" ]; then
${csudo}ldconfig
${csudo}ldconfig /etc/ld.so.conf.d
fi
}

34
packaging/tools/makeclient.sh
View File

@ -13,7 +13,13 @@ osType=$5
verMode=$6
verType=$7
pagMode=$8
dbName=$9
#comVersion=$9
dbName=$10
productName2="${11}"
#serverName2="${12}d"
clientName2="${12}"
# cusEmail2=${13}
productName="TDengine"
clientName="taos"
@ -38,15 +44,21 @@ release_dir="${top_dir}/release"
#package_name='linux'
if [ "$verMode" == "cluster" ]; then
install_dir="${release_dir}/${productName}-enterprise-client-${version}"
install_dir="${release_dir}/${productName2}-enterprise-client-${version}"
elif [ "$verMode" == "cloud" ]; then
install_dir="${release_dir}/${productName}-cloud-client-${version}"
install_dir="${release_dir}/${productName2}-cloud-client-${version}"
else
install_dir="${release_dir}/${productName}-client-${version}"
install_dir="${release_dir}/${productName2}-client-${version}"
fi
# Directories and files.
if [ "$verMode" == "cluster" ]; then
sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh
fi
if [ "$osType" != "Darwin" ]; then
if [ "$pagMode" == "lite" ]; then
strip ${build_dir}/bin/${clientName}
@ -131,22 +143,28 @@ fi
cd ${curr_dir}
cp ${install_files} ${install_dir}
cp ${install_dir}/install_client.sh install_client_temp.sh
if [ "$osType" == "Darwin" ]; then
sed 's/osType=Linux/osType=Darwin/g' ${install_dir}/install_client.sh >>install_client_temp.sh
sed -i 's/osType=Linux/osType=Darwin/g' install_client_temp.sh
mv install_client_temp.sh ${install_dir}/install_client.sh
fi
if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install_client.sh >>install_client_temp.sh
sed -i 's/verMode=edge/verMode=cluster/g' install_client_temp.sh
sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_client_temp.sh
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_client_temp.sh
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_client_temp.sh
sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_client_temp.sh
mv install_client_temp.sh ${install_dir}/install_client.sh
fi
if [ "$verMode" == "cloud" ]; then
sed 's/verMode=edge/verMode=cloud/g' ${install_dir}/install_client.sh >>install_client_temp.sh
sed -i 's/verMode=edge/verMode=cloud/g' install_client_temp.sh
mv install_client_temp.sh ${install_dir}/install_client.sh
fi
if [ "$pagMode" == "lite" ]; then
sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install_client.sh >>install_client_temp.sh
sed -i 's/pagMode=full/pagMode=lite/g' install_client_temp.sh
mv install_client_temp.sh ${install_dir}/install_client.sh
fi
chmod a+x ${install_dir}/install_client.sh

29
packaging/tools/makepkg.sh
View File

@ -16,6 +16,10 @@ verType=$7
pagMode=$8
versionComp=$9
dbName=${10}
productName2="${11}"
serverName2="${12}d"
clientName2="${12}"
cusEmail2="${13}"
script_dir="$(dirname $(readlink -f $0))"
top_dir="$(readlink -f ${script_dir}/../..)"
@ -38,11 +42,11 @@ release_dir="${top_dir}/release"
#package_name='linux'
if [ "$verMode" == "cluster" ]; then
install_dir="${release_dir}/${productName}-enterprise-server-${version}"
install_dir="${release_dir}/${productName2}-enterprise-server-${version}"
elif [ "$verMode" == "cloud" ]; then
install_dir="${release_dir}/${productName}-cloud-server-${version}"
install_dir="${release_dir}/${productName2}-cloud-server-${version}"
else
install_dir="${release_dir}/${productName}-server-${version}"
install_dir="${release_dir}/${productName2}-server-${version}"
fi
if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then
@ -50,11 +54,11 @@ if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then
[ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
taostools_ver=$(git tag |grep -v taos | sort | tail -1)
taostools_install_dir="${release_dir}/${clientName}Tools-${taostools_ver}"
taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}"
cd ${curr_dir}
else
taostools_install_dir="${release_dir}/${clientName}Tools-${version}"
taostools_install_dir="${release_dir}/${clientName2}Tools-${version}"
fi
# Directories and files
@ -225,6 +229,10 @@ fi
if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" remove_temp.sh
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" remove_temp.sh
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" remove_temp.sh
sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" remove_temp.sh
mv remove_temp.sh ${install_dir}/bin/remove.sh
fi
if [ "$verMode" == "cloud" ]; then
@ -247,16 +255,21 @@ fi
cd ${curr_dir}
cp ${install_files} ${install_dir}
cp ${install_dir}/install.sh install_temp.sh
if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install.sh >>install_temp.sh
sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh
sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_temp.sh
sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_temp.sh
sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_temp.sh
sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_temp.sh
mv install_temp.sh ${install_dir}/install.sh
fi
if [ "$verMode" == "cloud" ]; then
sed 's/verMode=edge/verMode=cloud/g' ${install_dir}/install.sh >>install_temp.sh
sed -i 's/verMode=edge/verMode=cloud/g' install_temp.sh
mv install_temp.sh ${install_dir}/install.sh
fi
if [ "$pagMode" == "lite" ]; then
sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install.sh >>install_temp.sh
sed -i 's/pagMode=full/pagMode=lite/g' install_temp.sh
mv install_temp.sh ${install_dir}/install.sh
fi
chmod a+x ${install_dir}/install.sh

31
packaging/tools/remove.sh
View File

@ -36,6 +36,16 @@ clientName="taos"
uninstallScript="rmtaos"
productName="TDengine"
serverName2="taosd"
clientName2="taos"
productName2="TDengine"
benchmarkName2="${clientName2}Benchmark"
dumpName2="${clientName2}dump"
uninstallScript2="rm${clientName2}"
installDir="/usr/local/${clientName}"
#install main path
install_main_dir=${installDir}
data_link_dir=${installDir}/data
@ -107,6 +117,15 @@ function clean_bin() {
${csudo}rm -f ${bin_link_dir}/tarbitrator || :
${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/taosx || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || :
fi
}
function clean_local_bin() {
@ -147,7 +166,7 @@ function clean_log() {
function clean_service_on_systemd() {
taosd_service_config="${service_config_dir}/${taos_service_name}.service"
if systemctl is-active --quiet ${taos_service_name}; then
echo "${productName} ${serverName} is running, stopping it..."
echo "${productName2} ${serverName2} is running, stopping it..."
${csudo}systemctl stop ${taos_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null
@ -155,7 +174,7 @@ function clean_service_on_systemd() {
taosadapter_service_config="${service_config_dir}/taosadapter.service"
if systemctl is-active --quiet ${taosadapter_service_name}; then
echo "${productName} taosAdapter is running, stopping it..."
echo "${productName2} ${clientName2}Adapter is running, stopping it..."
${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
@ -163,7 +182,7 @@ function clean_service_on_systemd() {
tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service"
if systemctl is-active --quiet ${tarbitrator_service_name}; then
echo "${productName} tarbitrator is running, stopping it..."
echo "${productName2} tarbitrator is running, stopping it..."
${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
@ -172,12 +191,12 @@ function clean_service_on_systemd() {
function clean_service_on_sysvinit() {
if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then
echo "${productName} ${serverName} is running, stopping it..."
echo "${productName2} ${serverName2} is running, stopping it..."
${csudo}service ${serverName} stop || :
fi
if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then
echo "${productName} tarbitrator is running, stopping it..."
echo "${productName2} tarbitrator is running, stopping it..."
${csudo}service tarbitratord stop || :
fi
@ -272,5 +291,5 @@ if [ "$osType" = "Darwin" ]; then
${csudo}rm -rf /Applications/TDengine.app
fi
echo -e "${GREEN}${productName} is removed successfully!${NC}"
echo -e "${GREEN}${productName2} is removed successfully!${NC}"
echo

17
packaging/tools/remove_client.sh
View File

@ -12,6 +12,15 @@ installDir="/usr/local/taos"
clientName="taos"
uninstallScript="rmtaos"
clientName2="taos"
productName2="TDengine"
benchmarkName2="${clientName}Benchmark"
dumpName2="${clientName}dump"
uninstallScript2="rm${clientName}"
installDir="/usr/local/${clientName}"
#install main path
install_main_dir=${installDir}
@ -40,6 +49,12 @@ function clean_bin() {
${csudo}rm -f ${bin_link_dir}/taosdump || :
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
${csudo}rm -f ${bin_link_dir}/set_core || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || :
fi
}
function clean_lib() {
@ -82,5 +97,5 @@ clean_config
${csudo}rm -rf ${install_main_dir}
echo -e "${GREEN}TDengine client is removed successfully!${NC}"
echo -e "${GREEN}${productName2} client is removed successfully!${NC}"
echo

16
packaging/tools/tdengine.iss
View File

@ -18,7 +18,6 @@
#define MyAppDLLName "\driver\*.dll"
;#define MyAppVersion "3.0"
;#define MyAppInstallName "TDengine"
;#define MyAppInstallName "TDengine"
[Setup]
VersionInfoVersion={#MyAppVersion}
AppId={{A0F7A93C-79C4-485D-B2B8-F0D03DF42FAB}
@ -59,11 +58,14 @@ Source: {#MyAppSourceDir}{#MyAppDriverName}; DestDir: "{app}\driver"; Flags: igN
Source: {#MyAppSourceDir}{#MyAppIncludeName}; DestDir: "{app}\include"; Flags: igNoreversion recursesubdirs createallsubdirs
Source: {#MyAppSourceDir}{#MyAppExeName}; DestDir: "{app}"; Excludes: {#MyAppExcludeSource} ; Flags: igNoreversion recursesubdirs createallsubdirs
Source: {#MyAppSourceDir}{#MyAppTaosdemoExeName}; DestDir: "{app}"; Flags: igNoreversion recursesubdirs createallsubdirs
Source: {#MyAppSourceDir}\taos.exe; DestDir: "{app}"; DestName: "{#CusPrompt}.EXE"; Flags: igNoreversion recursesubdirs createallsubdirs
Source: {#MyAppSourceDir}\taosBenchmark.exe; DestDir: "{app}"; DestName: "{#CusPrompt}Benchmark.EXE"; Flags: igNoreversion recursesubdirs createallsubdirs
Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt}dump.EXE"; Flags: igNoreversion recursesubdirs createallsubdirs
[run]
Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden
Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden
Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\{#CusName}\\taosd.exe --win_service""" ; Flags: runhidden
Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\{#CusName}\\taosadapter.exe""" ; Flags: runhidden
[UninstallRun]
RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden
@ -73,8 +75,8 @@ RunOnceId: "deltaosadapter"; Filename: {sys}\sc.exe; Parameters: "delete taosada
[Registry]
Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; \
ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};C:\TDengine"; \
Check: NeedsAddPath('C:\TDengine')
ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};C:\{#CusName}"; \
Check: NeedsAddPath('C:\{#CusName}')
[Code]
function NeedsAddPath(Param: string): boolean;
@ -104,11 +106,11 @@ Name: "desktopicon";Description: "{cm:CreateDesktopIcon}"; GroupDescription:"{cm
[Icons]
Name:"{group}\Taos Shell"; Filename: "{app}\include\{#MyAppTaosExeName}" ; Parameters: "taos.exe" ; IconFilename: "{app}\include\{#MyAppIco}"
Name:"{group}\Open TDengine Directory"; Filename: "{app}\"
Name:"{group}\Open {#CusName} Directory"; Filename: "{app}\"
Name:"{group}\Taosdemo"; Filename: "{app}\include\{#MyAppTaosExeName}" ; Parameters: "taosdemo.exe" ; IconFilename: "{app}\include\{#MyAppIco}"
Name: "{group}\{cm:UninstallProgram,{#MyAppName}}"; Filename: "{uninstallexe}" ; IconFilename: "{app}\include\{#MyAppIco}"
Name:"{commondesktop}\Taos Shell"; Filename: "{app}\include\{#MyAppTaosExeName}" ; Parameters: "taos.exe" ; Tasks: desktopicon; WorkingDir: "{app}" ; IconFilename: "{app}\include\{#MyAppIco}"
[Messages]
ConfirmUninstall=Do you really want to uninstall TDengine from your computer?%n%nPress [Y] to completely delete %1 and all its components;%nPress [N] to keep the software on your computer.
ConfirmUninstall=Do you really want to uninstall {#CusName} from your computer?%n%nPress [Y] to completely delete %1 and all its components;%nPress [N] to keep the software on your computer.

7
source/client/inc/clientSml.h
View File

@ -99,6 +99,7 @@ typedef struct {
char *tags;
char *cols;
char *timestamp;
char *measureTag;
int32_t measureLen;
int32_t measureTagsLen;
@ -114,7 +115,7 @@ typedef struct {
int32_t sTableNameLen;
char childTableName[TSDB_TABLE_NAME_LEN];
uint64_t uid;
void *key; // for openTsdb
// void *key; // for openTsdb
SArray *tags;
@ -166,8 +167,8 @@ typedef struct {
int32_t ttl;
int32_t uid; // used for automatic create child table
NodeList *childTables;
NodeList *superTables;
SHashObj *childTables;
SHashObj *superTables;
SHashObj *pVgHash;
STscObj *taos;

14
source/client/src/clientImpl.c
View File

@ -138,6 +138,12 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
p->mgmtEp = epSet;
taosThreadMutexInit(&p->qnodeMutex, NULL);
p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores / 2);
if (p->pTransporter == NULL) {
taosThreadMutexUnlock(&appInfo.mutex);
taosMemoryFreeClear(key);
taosMemoryFree(p);
return NULL;
}
p->pAppHbMgr = appHbMgrInit(p, key);
if (NULL == p->pAppHbMgr) {
destroyAppInst(p);
@ -1386,8 +1392,6 @@ int32_t doProcessMsgFromServer(void* param) {
tscError("0x%" PRIx64 " rsp msg:%s, code:%s rspLen:%d, elapsed time:%d ms, reqId:0x%" PRIx64, pRequest->self,
TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed / 1000, pRequest->requestId);
}
taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId);
}
}
@ -1407,6 +1411,11 @@ int32_t doProcessMsgFromServer(void* param) {
}
pSendInfo->fp(pSendInfo->param, &buf, pMsg->code);
if (pTscObj) {
taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId);
}
rpcFreeCont(pMsg->pCont);
destroySendMsgInfo(pSendInfo);
@ -1444,6 +1453,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
tscError("failed to sched msg to tsc, tsc ready to quit");
rpcFreeCont(pMsg->pCont);
taosMemoryFree(arg->pEpset);
destroySendMsgInfo(pMsg->info.ahandle);
taosMemoryFree(arg);
}
}

18
source/client/src/clientRawBlockWrite.c
View File

@ -694,7 +694,11 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
pReq.numOfColumns = req.schemaRow.nCols;
pReq.numOfTags = req.schemaTag.nCols;
pReq.commentLen = -1;
<<<<<<< HEAD
pReq.suid = processSuid(req.suid, pRequest->pDb);;
=======
pReq.suid = processSuid(req.suid, pRequest->pDb);
>>>>>>> 6483b92f0c645b36facfb2f3415b8df9449264cf
pReq.source = TD_REQ_FROM_TAOX;
pReq.igExists = true;
@ -1513,10 +1517,10 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
if(fields == NULL){
if (fields == NULL) {
goto end;
}
for(int i = 0; i < pSW->nCols; i++){
for (int i = 0; i < pSW->nCols; i++) {
fields[i].type = pSW->pSchema[i].type;
fields[i].bytes = pSW->pSchema[i].bytes;
tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
@ -1651,9 +1655,9 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
tDecoderClear(&decoderTmp);
}
if(pCreateReqDst){
if (pCreateReqDst) {
strcpy(pName.tname, pCreateReqDst->ctb.stbName);
}else{
} else {
strcpy(pName.tname, tbName);
}
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
@ -1674,7 +1678,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
goto end;
}
if(pCreateReqDst){
if (pCreateReqDst) {
pTableMeta->vgId = vg.vgId;
pTableMeta->uid = pCreateReqDst->uid;
}
@ -1685,10 +1689,10 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
if(fields == NULL){
if (fields == NULL) {
goto end;
}
for(int i = 0; i < pSW->nCols; i++){
for (int i = 0; i < pSW->nCols; i++) {
fields[i].type = pSW->pSchema[i].type;
fields[i].bytes = pSW->pSchema[i].bytes;
tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));

128
source/client/src/clientSml.c
View File

@ -759,13 +759,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
conn.requestObjRefId = info->pRequest->self;
conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
NodeList *tmp = info->superTables;
SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
while (tmp) {
SSmlSTableMeta *sTableData = (SSmlSTableMeta *)tmp->data.value;
SSmlSTableMeta *sTableData = *tmp;
bool needCheckMeta = false; // for multi thread
size_t superTableLen = (size_t)tmp->data.keyLen;
const void *superTable = tmp->data.key;
size_t superTableLen = 0;
void *superTable = taosHashGetKey(tmp, &superTableLen);
memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
memcpy(pName.tname, superTable, superTableLen);
@ -914,7 +914,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
sTableData->tableMeta = pTableMeta;
tmp = tmp->next;
tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
}
return 0;
@ -1017,11 +1017,11 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
taosHashCleanup(kvHash);
}
if (info->parseJsonByLib) {
SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
if (key != NULL) taosMemoryFree(key->tags);
}
taosMemoryFree(tag->key);
// if (info->parseJsonByLib) {
// SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
// if (key != NULL) taosMemoryFree(key->tags);
// }
// taosMemoryFree(tag->key);
taosArrayDestroy(tag->cols);
taosArrayDestroy(tag->tags);
taosMemoryFree(tag);
@ -1042,29 +1042,23 @@ void smlDestroyInfo(SSmlHandle *info) {
qDestroyQuery(info->pQuery);
// destroy info->childTables
NodeList *tmp = info->childTables;
while (tmp) {
if (tmp->data.used) {
smlDestroyTableInfo(info, (SSmlTableInfo *)tmp->data.value);
}
NodeList *t = tmp->next;
taosMemoryFree(tmp);
tmp = t;
SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
while (oneTable) {
smlDestroyTableInfo(info, *oneTable);
oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
}
// destroy info->superTables
tmp = info->superTables;
while (tmp) {
if (tmp->data.used) {
smlDestroySTableMeta((SSmlSTableMeta *)tmp->data.value);
}
NodeList *t = tmp->next;
taosMemoryFree(tmp);
tmp = t;
SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
while (oneSTable) {
smlDestroySTableMeta(*oneSTable);
oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
}
// destroy info->pVgHash
taosHashCleanup(info->pVgHash);
taosHashCleanup(info->childTables);
taosHashCleanup(info->superTables);
for (int i = 0; i < taosArrayGetSize(info->tagJsonArray); i++) {
cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i);
@ -1088,6 +1082,7 @@ void smlDestroyInfo(SSmlHandle *info) {
if (info->parseJsonByLib) {
taosMemoryFree(info->lines[i].tags);
}
if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
}
taosMemoryFree(info->lines);
}
@ -1112,6 +1107,9 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
}
info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
info->id = smlGenId();
info->pQuery = smlInitHandle();
info->dataFormat = true;
@ -1122,7 +1120,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
info->maxTagKVs = taosArrayInit(8, sizeof(SSmlKv));
info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv));
if (NULL == info->pVgHash) {
if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables) {
uError("create SSmlHandle failed");
goto cleanup;
}
@ -1156,11 +1154,13 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
SSmlLineInfo *elements = info->lines + i;
SSmlTableInfo *tinfo = NULL;
if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->measure, elements->measureTagsLen, NULL);
tinfo = *(SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen);
} else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet);
tinfo = *(SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
elements->measureLen + elements->tagsLen);
} else {
tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet);
tinfo = *(SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
elements->measureLen + elements->tagsLen);
}
if (tinfo == NULL) {
@ -1184,12 +1184,12 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
return ret;
}
SSmlSTableMeta *tableMeta =
(SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL);
SSmlSTableMeta **tableMeta =
(SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
if (tableMeta) { // update meta
ret = smlUpdateMeta(tableMeta->colHash, tableMeta->cols, elements->colArray, false, &info->msgBuf);
ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf);
if (ret == TSDB_CODE_SUCCESS) {
ret = smlUpdateMeta(tableMeta->tagHash, tableMeta->tags, tinfo->tags, true, &info->msgBuf);
ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf);
}
if (ret != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
@ -1205,7 +1205,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
smlInsertMeta(meta->colHash, meta->cols, elements->colArray);
nodeListSet(&info->superTables, elements->measure, elements->measureLen, meta, NULL);
taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
}
}
@ -1215,9 +1215,9 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
static int32_t smlInsertData(SSmlHandle *info) {
int32_t code = TSDB_CODE_SUCCESS;
NodeList *tmp = info->childTables;
while (tmp) {
SSmlTableInfo *tableData = (SSmlTableInfo *)tmp->data.value;
SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
while (oneTable) {
SSmlTableInfo *tableData = *oneTable;
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
@ -1237,25 +1237,25 @@ static int32_t smlInsertData(SSmlHandle *info) {
}
taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg));
SSmlSTableMeta *pMeta =
(SSmlSTableMeta *)nodeListGet(info->superTables, tableData->sTableName, tableData->sTableNameLen, NULL);
if (unlikely(NULL == pMeta || NULL == pMeta->tableMeta)) {
SSmlSTableMeta **pMeta =
(SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen);
if (unlikely(NULL == pMeta || NULL == (*pMeta)->tableMeta)) {
uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName);
return TSDB_CODE_SML_INTERNAL_ERROR;
}
// use tablemeta of stable to save vgid and uid of child table
pMeta->tableMeta->vgId = vg.vgId;
pMeta->tableMeta->uid = tableData->uid; // one table merge data block together according uid
(*pMeta)->tableMeta->vgId = vg.vgId;
(*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid
code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, pMeta->cols, tableData->cols, pMeta->tableMeta,
tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->ttl,
info->msgBuf.buf, info->msgBuf.len);
code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
(*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
info->ttl, info->msgBuf.buf, info->msgBuf.len);
if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
return code;
}
tmp = tmp->next;
oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
}
code = smlBuildOutput(info->pQuery, info->pVgHash);
@ -1288,25 +1288,22 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
int32_t smlClearForRerun(SSmlHandle *info) {
info->reRun = false;
// clear info->childTables
NodeList *pList = info->childTables;
while (pList) {
if (pList->data.used) {
smlDestroyTableInfo(info, (SSmlTableInfo *)pList->data.value);
pList->data.used = false;
}
pList = pList->next;
SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
while (oneTable) {
smlDestroyTableInfo(info, *oneTable);
oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
}
// clear info->superTables
pList = info->superTables;
while (pList) {
if (pList->data.used) {
smlDestroySTableMeta((SSmlSTableMeta *)pList->data.value);
pList->data.used = false;
}
pList = pList->next;
SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
while (oneSTable) {
smlDestroySTableMeta(*oneSTable);
oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
}
taosHashClear(info->childTables);
taosHashClear(info->superTables);
if (!info->dataFormat) {
if (unlikely(info->lines != NULL)) {
uError("SML:0x%" PRIx64 " info->lines != NULL", info->id);
@ -1375,6 +1372,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
if (info->dataFormat) {
SSmlLineInfo element = {0};
code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element);
if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag);
} else {
code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i);
}
@ -1418,15 +1416,15 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
}
info->cost.lineNum = info->lineNum;
info->cost.numOfSTables = nodeListSize(info->superTables);
info->cost.numOfCTables = nodeListSize(info->childTables);
info->cost.numOfSTables = taosHashGetSize(info->superTables);
info->cost.numOfCTables = taosHashGetSize(info->childTables);
info->cost.schemaTime = taosGetTimestampUs();
do {
code = smlModifyDBSchemas(info);
if (code == 0) break;
} while (retryNum++ < nodeListSize(info->superTables) * MAX_RETRY_TIMES);
} while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES);
if (code != 0) {
uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code));
@ -1504,7 +1502,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
request->code = code;
info->cost.endTime = taosGetTimestampUs();
info->cost.code = code;
// smlPrintStatisticInfo(info);
smlPrintStatisticInfo(info);
end:
smlDestroyInfo(info);

70
source/client/src/clientSmlJson.c
View File

@ -256,7 +256,7 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset){
if(unlikely(index >= OTD_JSON_FIELDS_NUM)) {
uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start)
break;
return -1;
}
char *sTmp = *start;
@ -275,6 +275,7 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset){
}
if(unlikely(isInQuote && *(*start) == '"')){
element->measureLen = (*start) - element->measure;
(*start)++;
break;
}
(*start)++;
@ -384,7 +385,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){
if(unlikely(index >= OTD_JSON_FIELDS_NUM)) {
uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start)
break;
return -1;
}
if((*start)[1] == 'm'){
@ -393,6 +394,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){
while(*(*start)){
if(unlikely(*(*start) == '"')){
element->measureLen = (*start) - element->measure;
(*start)++;
break;
}
(*start)++;
@ -406,9 +408,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){
element->timestampLen = tmp - (*start);
*start = tmp;
}
continue;
}
}else{
while(*(*start)){
if(unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){
element->timestampLen = (*start) - element->timestamp;
@ -416,6 +416,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){
}
(*start)++;
}
}
}else if((*start)[1] == 'v'){
(*start) += offset[index++];
element->cols = *start;
@ -425,8 +426,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){
element->colsLen = tmp - (*start);
*start = tmp;
}
continue;
}
}else{
while(*(*start)){
if(unlikely( *(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){
element->colsLen = (*start) - element->cols;
@ -434,6 +434,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){
}
(*start)++;
}
}
}else if((*start)[1] == 't' && (*start)[2] == 'a'){
(*start) += offset[index++];
element->tags = (*start);
@ -442,7 +443,6 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){
element->tagsLen = tmp - (*start);
*start = tmp;
}
continue;
}
if(*(*start) == '}'){
(*start)++;
@ -694,9 +694,9 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
SArray *superKV = NULL;
if(info->dataFormat){
if(unlikely(!isSameMeasure)){
SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL);
if(unlikely(sMeta == NULL)){
SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
SSmlSTableMeta *sMeta = NULL;
if(unlikely(tmp == NULL)){
STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
if(pTableMeta == NULL){
info->dataFormat = false;
@ -705,10 +705,11 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
}
sMeta = smlBuildSTableMeta(info->dataFormat);
sMeta->tableMeta = pTableMeta;
nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL);
taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
tmp = &sMeta;
}
info->currSTableMeta = sMeta->tableMeta;
superKV = sMeta->tags;
info->currSTableMeta = (*tmp)->tableMeta;
superKV = (*tmp)->tags;
if(unlikely(taosArrayGetSize(superKV) == 0)){
isSuperKVInit = false;
@ -761,13 +762,13 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt);
if(unlikely(kv.length > maxKV->length)){
maxKV->length = kv.length;
SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL);
SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
if(unlikely(NULL == tableMeta)){
uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
return TSDB_CODE_SML_INTERNAL_ERROR;
}
SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt);
SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt);
oldKV->length = kv.length;
info->needModifySchema = true;
}
@ -808,8 +809,14 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
cnt++;
}
SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet);
if (unlikely(tinfo == NULL)) {
elements->measureTag = (char*)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
memcpy(elements->measureTag, elements->measure, elements->measureLen);
memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
elements->measureTagsLen = elements->measureLen + elements->tagsLen;
SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
SSmlTableInfo *tinfo = NULL;
if (unlikely(tmp == NULL)) {
tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
if (unlikely(!tinfo)) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -828,17 +835,18 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
}
}
SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
*key = *elements;
if(info->parseJsonByLib){
key->tags = taosMemoryMalloc(elements->tagsLen + 1);
memcpy(key->tags, elements->tags, elements->tagsLen);
key->tags[elements->tagsLen] = 0;
// SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
// *key = *elements;
// if(info->parseJsonByLib){
// key->tags = taosMemoryMalloc(elements->tagsLen + 1);
// memcpy(key->tags, elements->tags, elements->tagsLen);
// key->tags[elements->tagsLen] = 0;
// }
// tinfo->key = key;
taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, POINTER_BYTES);
tmp = &tinfo;
}
tinfo->key = key;
nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet);
}
if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
return ret;
}
@ -1011,6 +1019,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
elements->tags = NULL;
return ret;
}
}else{
elements->measureTag = info->preLine.measureTag;
}
if(needFree){
@ -1189,6 +1199,8 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id);
return ret;
}
}else{
elements->measureTag = info->preLine.measureTag;
}
if(unlikely(info->reRun)){
@ -1259,6 +1271,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) {
if(info->dataFormat) {
SSmlLineInfo element = {0};
ret = smlParseJSONString(info, &dataPointStart, &element);
if(element.measureTagsLen != 0) taosMemoryFree(element.measureTag);
}else{
if(cnt >= payloadNum){
payloadNum = payloadNum << 1;
@ -1269,6 +1282,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) {
}
}
ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt);
if((info->lines + cnt)->measure == NULL) break;
}
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
uError("SML:0x%" PRIx64 " Invalid JSON Payload 1:%s", info->id, payload);

42
source/client/src/clientSmlLine.c
View File

@ -148,9 +148,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd,
SArray *superKV = NULL;
if(info->dataFormat){
if(unlikely(!isSameMeasure)){
SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL);
if(unlikely(sMeta == NULL)){
SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
SSmlSTableMeta *sMeta = NULL;
if(unlikely(tmp == NULL)){
STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
if(pTableMeta == NULL){
info->dataFormat = false;
@ -159,10 +159,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd,
}
sMeta = smlBuildSTableMeta(info->dataFormat);
sMeta->tableMeta = pTableMeta;
nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL);
taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
tmp = &sMeta;
}
info->currSTableMeta = sMeta->tableMeta;
superKV = sMeta->tags;
info->currSTableMeta = (*tmp)->tableMeta;
superKV = (*tmp)->tags;
if(unlikely(taosArrayGetSize(superKV) == 0)){
isSuperKVInit = false;
@ -258,13 +259,13 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd,
SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt);
if(unlikely(kv.length > maxKV->length)){
maxKV->length = kv.length;
SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL);
SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
if(unlikely(NULL == tableMeta)){
uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
return TSDB_CODE_SML_INTERNAL_ERROR;
}
SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt);
SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt);
oldKV->length = kv.length;
info->needModifySchema = true;
}
@ -310,7 +311,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd,
(*sql)++;
}
void* oneTable = nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL);
void* oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
if ((oneTable != NULL)) {
return TSDB_CODE_SUCCESS;
}
@ -332,7 +333,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd,
}
}
nodeListSet(&info->childTables, currElement->measure, currElement->measureTagsLen, tinfo, NULL);
taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES);
return TSDB_CODE_SUCCESS;
}
@ -345,18 +346,18 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd,
SArray *superKV = NULL;
if(info->dataFormat){
if(unlikely(!isSameCTable)){
SSmlTableInfo *oneTable = (SSmlTableInfo *)nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL);
SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
if (unlikely(oneTable == NULL)) {
smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure);
return TSDB_CODE_SML_INVALID_DATA;
}
info->currTableDataCtx = oneTable->tableDataCtx;
info->currTableDataCtx = (*oneTable)->tableDataCtx;
}
if(unlikely(!isSameMeasure)){
SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL);
if(unlikely(sMeta == NULL)){
SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
SSmlSTableMeta *sMeta = NULL;
if(unlikely(tmp == NULL)){
STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
if(pTableMeta == NULL){
info->dataFormat = false;
@ -365,10 +366,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd,
}
sMeta = smlBuildSTableMeta(info->dataFormat);
sMeta->tableMeta = pTableMeta;
nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL);
taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
tmp = &sMeta;
}
info->currSTableMeta = sMeta->tableMeta;
superKV = sMeta->cols;
info->currSTableMeta = (*tmp)->tableMeta;
superKV = (*tmp)->cols;
if(unlikely(taosArrayGetSize(superKV) == 0)){
isSuperKVInit = false;
}
@ -487,13 +489,13 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd,
if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)){
maxKV->length = kv.length;
SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL);
SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
if(unlikely(NULL == tableMeta)){
uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
return TSDB_CODE_SML_INTERNAL_ERROR;
}
SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->cols, cnt);
SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->cols, cnt);
oldKV->length = kv.length;
info->needModifySchema = true;
}

40
source/client/src/clientSmlTelnet.c
View File

@ -71,6 +71,7 @@ static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t
static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) {
if(is_same_child_table_telnet(elements, &info->preLine) == 0){
elements->measureTag = info->preLine.measureTag;
return TSDB_CODE_SUCCESS;
}
@ -83,9 +84,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
SArray *superKV = NULL;
if(info->dataFormat){
if(!isSameMeasure){
SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL);
if(unlikely(sMeta == NULL)){
SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
SSmlSTableMeta *sMeta = NULL;
if(unlikely(tmp == NULL)){
STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
if(pTableMeta == NULL){
info->dataFormat = false;
@ -94,10 +95,11 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
}
sMeta = smlBuildSTableMeta(info->dataFormat);
sMeta->tableMeta = pTableMeta;
nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL);
taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
tmp = &sMeta;
}
info->currSTableMeta = sMeta->tableMeta;
superKV = sMeta->tags;
info->currSTableMeta = (*tmp)->tableMeta;
superKV = (*tmp)->tags;
if(unlikely(taosArrayGetSize(superKV) == 0)){
isSuperKVInit = false;
@ -183,13 +185,13 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt);
if(unlikely(kv.length > maxKV->length)){
maxKV->length = kv.length;
SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL);
SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
if(unlikely(NULL == tableMeta)){
uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
return TSDB_CODE_SML_INTERNAL_ERROR;
}
SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt);
SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt);
oldKV->length = kv.length;
info->needModifySchema = true;
}
@ -229,8 +231,15 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
taosArrayPush(preLineKV, &kv);
cnt++;
}
SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet);
if (unlikely(tinfo == NULL)) {
elements->measureTag = (char*)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
memcpy(elements->measureTag, elements->measure, elements->measureLen);
memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
elements->measureTagsLen = elements->measureLen + elements->tagsLen;
SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
SSmlTableInfo *tinfo = NULL;
if (unlikely(tmp == NULL)) {
tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
if (!tinfo) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -249,12 +258,13 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
}
}
SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
*key = *elements;
tinfo->key = key;
nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet);
// SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
// *key = *elements;
// tinfo->key = key;
taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, POINTER_BYTES);
tmp = &tinfo;
}
if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
return TSDB_CODE_SUCCESS;
}

12
source/client/test/clientTests.cpp
View File

@ -112,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
}
taos_free_result(pRes);
for(int32_t i = 0; i < 20; i += 20) {
for(int32_t i = 0; i < 2000; i += 20) {
char sql[1024] = {0};
sprintf(sql,
"insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
@ -692,6 +692,7 @@ TEST(testCase, insert_test) {
taos_free_result(pRes);
taos_close(pConn);
}
#endif
TEST(testCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@ -725,7 +726,7 @@ TEST(testCase, projection_query_tables) {
}
taos_free_result(pRes);
for (int32_t i = 0; i < 200000; ++i) {
for (int32_t i = 0; i < 2; ++i) {
printf("create table :%d\n", i);
createNewTable(pConn, i);
}
@ -751,6 +752,7 @@ TEST(testCase, projection_query_tables) {
taos_close(pConn);
}
#if 0
TEST(testCase, tsbs_perf_test) {
TdThread qid[20] = {0};
@ -760,8 +762,6 @@ TEST(testCase, tsbs_perf_test) {
getchar();
}
#endif
TEST(testCase, projection_query_stables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
@ -790,7 +790,6 @@ TEST(testCase, projection_query_stables) {
taos_close(pConn);
}
#if 0
TEST(testCase, agg_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
@ -831,7 +830,7 @@ TEST(testCase, async_api_test) {
ASSERT_NE(pConn, nullptr);
taos_query(pConn, "use abc1");
#if 0
TAOS_RES* pRes = taos_query(pConn, "insert into tu(ts) values('2022-02-27 12:12:61')");
if (taos_errno(pRes) != 0) {
printf("failed, reason:%s\n", taos_errstr(pRes));
@ -854,7 +853,6 @@ TEST(testCase, async_api_test) {
printf("%s\n", str);
memset(str, 0, sizeof(str));
}
#endif
taos_query_a(pConn, "select count(*) from tu", queryCallback, pConn);
getchar();

1
source/common/src/systable.c
View File

@ -230,6 +230,7 @@ static const SSysDbTableSchema vgroupsSchema[] = {
{.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
// {.name = "compact_start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema smaSchema[] = {

429
source/common/src/tdataformat.c
View File

@ -21,6 +21,7 @@
#include "tlog.h"
static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData);
static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward);
// SBuffer ================================
void tBufferDestroy(SBuffer *pBuffer) {
@ -850,19 +851,19 @@ _exit:
return &pIter->cv;
}
static int32_t tRowAppendNoneToColData(SColData *aColData, int32_t nColData) {
static int32_t tRowNoneUpsertColData(SColData *aColData, int32_t nColData, int32_t flag) {
int32_t code = 0;
if (flag) return code;
for (int32_t iColData = 0; iColData < nColData; iColData++) {
SColData *pColData = &aColData[iColData];
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
code = tColDataAppendValueImpl[aColData[iColData].flag][CV_FLAG_NONE](&aColData[iColData], NULL, 0);
if (code) return code;
}
_exit:
return code;
}
static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STSchema *pSchema) {
static int32_t tRowNullUpsertColData(SColData *aColData, int32_t nColData, STSchema *pSchema, int32_t flag) {
int32_t code = 0;
int32_t iColData = 0;
@ -873,21 +874,23 @@ static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STS
while (pColData) {
if (pTColumn) {
if (pTColumn->colId == pColData->cid) { // NULL
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0, flag > 0);
}
if (code) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL;
} else if (pTColumn->colId > pColData->cid) { // NONE
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
} else {
pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL;
}
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
} else { // NONE
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
}
}
@ -895,7 +898,8 @@ static int32_t tRowAppendNullToColData(SColData *aColData, int32_t nColData, STS
_exit:
return code;
}
static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) {
static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData,
int32_t flag) {
int32_t code = 0;
int32_t iColData = 0;
@ -955,11 +959,15 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData
}
if (bv == BIT_FLG_NONE) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0)))
goto _exit;
goto _continue;
} else if (bv == BIT_FLG_NULL) {
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0, flag > 0);
}
if (code) goto _exit;
goto _continue;
}
@ -969,11 +977,20 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData
uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset);
uint32_t nData;
pData += tGetU32v(pData, &nData);
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData, flag > 0);
}
if (code) goto _exit;
} else {
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset,
TYPE_BYTES[pColData->type]);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset,
TYPE_BYTES[pColData->type], flag > 0);
}
if (code) goto _exit;
}
@ -981,17 +998,13 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
} else if (pTColumn->colId > pColData->cid) { // NONE
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
} else {
pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
}
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
}
}
@ -999,7 +1012,7 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData
_exit:
return code;
}
static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) {
static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag) {
int32_t code = 0;
SKVIdx *pKVIdx = (SKVIdx *)pRow->data;
@ -1040,18 +1053,25 @@ static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *a
if (TABS(cid) == pTColumn->colId) {
if (cid < 0) {
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
if (code) goto _exit;
} else {
if (IS_VAR_DATA_TYPE(pTColumn->type)) {
uint32_t nData;
pData += tGetU32v(pData, &nData);
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData);
if (code) goto _exit;
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, 0);
if (code) goto _exit;
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0, flag > 0);
}
if (code) goto _exit;
} else {
uint32_t nData;
if (IS_VAR_DATA_TYPE(pTColumn->type)) {
pData += tGetU32v(pData, &nData);
} else {
nData = 0;
}
if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData);
} else {
code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData, flag > 0);
}
if (code) goto _exit;
}
iCol++;
goto _continue;
@ -1062,22 +1082,19 @@ static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *a
}
}
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
_continue:
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
} else if (pTColumn->colId > pColData->cid) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
} else {
pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL;
}
} else {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
if (code) goto _exit;
if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit;
pColData = (++iColData < nColData) ? &aColData[iColData] : NULL;
}
}
@ -1085,30 +1102,23 @@ static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *a
_exit:
return code;
}
int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData) {
/* flag > 0: forward update
* flag == 0: append
* flag < 0: backward update
*/
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag) {
ASSERT(pRow->sver == pTSchema->version);
ASSERT(nColData > 0);
int32_t code = 0;
if (pRow->flag == HAS_NONE) {
code = tRowAppendNoneToColData(aColData, nColData);
goto _exit;
return tRowNoneUpsertColData(aColData, nColData, flag);
} else if (pRow->flag == HAS_NULL) {
code = tRowAppendNullToColData(aColData, nColData, pTSchema);
goto _exit;
return tRowNullUpsertColData(aColData, nColData, pTSchema, flag);
} else if (pRow->flag >> 4) { // KV row
return tRowKVUpsertColData(pRow, pTSchema, aColData, nColData, flag);
} else { // TUPLE row
return tRowTupleUpsertColData(pRow, pTSchema, aColData, nColData, flag);
}
if (pRow->flag >> 4) { // KV row
code = tRowAppendKVToColData(pRow, pTSchema, aColData, nColData);
if (code) goto _exit;
} else {
code = tRowAppendTupleToColData(pRow, pTSchema, aColData, nColData);
if (code) goto _exit;
}
_exit:
return code;
}
// STag ========================================
@ -1549,6 +1559,9 @@ void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
}
void tColDataClear(SColData *pColData) {
pColData->numOfNone = 0;
pColData->numOfNull = 0;
pColData->numOfValue = 0;
pColData->nVal = 0;
pColData->flag = 0;
pColData->nData = 0;
@ -1594,15 +1607,18 @@ _exit:
}
static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->flag = HAS_VALUE;
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->flag = HAS_NONE;
pColData->numOfNone++;
pColData->nVal++;
return 0;
}
static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->flag = HAS_NULL;
pColData->numOfNull++;
pColData->nVal++;
return 0;
}
@ -1617,6 +1633,7 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, uint8_t *p
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->flag |= HAS_VALUE;
pColData->numOfValue++;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
@ -1636,6 +1653,7 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, uint8_t *p
}
static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->nVal++;
pColData->numOfNone++;
return 0;
}
static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, uint8_t *pData, uint32_t nData) {
@ -1649,6 +1667,7 @@ static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, uint8_t *p
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->flag |= HAS_NULL;
pColData->numOfNull++;
pColData->nVal++;
return code;
@ -1664,6 +1683,7 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, uint8_t *p
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->flag |= HAS_VALUE;
pColData->numOfValue++;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
@ -1692,18 +1712,21 @@ static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, uint8_t *p
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->flag |= HAS_NONE;
pColData->numOfNone++;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->nVal++;
pColData->numOfNull++;
return 0;
}
static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, uint8_t *pData, uint32_t nData) {
int32_t code = 0;
pColData->flag |= HAS_VALUE;
pColData->numOfValue++;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
@ -1740,6 +1763,7 @@ static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, uint8_t *p
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->numOfNone++;
pColData->nVal++;
return code;
@ -1751,15 +1775,20 @@ static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, uint8_t *p
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->numOfNull++;
pColData->nVal++;
return code;
}
#define tColDataAppendValue40 tColDataPutValue
static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, uint8_t *pData, uint32_t nData) {
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, uint8_t *pData, uint32_t nData) {
int32_t code = 0;
pColData->flag |= HAS_NONE;
pColData->numOfNone++;
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
@ -1774,6 +1803,7 @@ static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, uint8_t *p
int32_t code = 0;
pColData->flag |= HAS_NULL;
pColData->numOfNull++;
int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit);
@ -1791,6 +1821,7 @@ static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, uint8_t *p
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
@ -1801,6 +1832,7 @@ static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, uint8_t *p
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->numOfNone++;
return tColDataPutValue(pColData, NULL, 0);
}
@ -1808,6 +1840,7 @@ static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, uint8_t *p
int32_t code = 0;
pColData->flag |= HAS_NULL;
pColData->numOfNull++;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
@ -1829,6 +1862,7 @@ static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
@ -1836,6 +1870,7 @@ static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, uint8_t *p
int32_t code = 0;
pColData->flag |= HAS_NONE;
pColData->numOfNone++;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
@ -1857,6 +1892,7 @@ static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->numOfNull++;
return tColDataPutValue(pColData, NULL, 0);
}
@ -1866,6 +1902,7 @@ static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 2);
pColData->numOfValue++;
return tColDataPutValue(pColData, pData, nData);
}
@ -1875,6 +1912,7 @@ static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 0);
pColData->numOfNone++;
return tColDataPutValue(pColData, NULL, 0);
}
@ -1884,6 +1922,7 @@ static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, uint8_t *p
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 1);
pColData->numOfNull++;
return tColDataPutValue(pColData, NULL, 0);
}
@ -1896,6 +1935,8 @@ static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pDa
{tColDataAppendValue50, tColDataAppendValue51, tColDataAppendValue52}, // HAS_VALUE|HAS_NONE
{tColDataAppendValue60, tColDataAppendValue61, tColDataAppendValue62}, // HAS_VALUE|HAS_NULL
{tColDataAppendValue70, tColDataAppendValue71, tColDataAppendValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE
// VALUE NONE NULL
};
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
@ -1904,6 +1945,282 @@ int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
pColVal->value.nData);
}
static FORCE_INLINE int32_t tColDataUpdateValue10(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue10(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue00(pColData, pData, nData);
}
}
static FORCE_INLINE int32_t tColDataUpdateValue12(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue12(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue02(pColData, pData, nData);
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue20(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward) {
pColData->numOfNull--;
pColData->nVal--;
if (pColData->numOfNull) {
return tColDataAppendValue20(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue00(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue30(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
uint8_t bv = GET_BIT1(pColData->pBitMap, pColData->nVal - 1);
if (bv == 0) { // NONE == > VALUE
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue30(pColData, pData, nData);
} else {
pColData->flag = HAS_NULL;
return tColDataAppendValue20(pColData, pData, nData);
}
} else if (forward) { // NULL == > VALUE
pColData->numOfNull--;
pColData->nVal--;
if (pColData->numOfNull) {
return tColDataAppendValue30(pColData, pData, nData);
} else {
pColData->flag = HAS_NONE;
return tColDataAppendValue10(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue32(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) { // NONE ==> NULL
pColData->numOfNone--;
pColData->numOfNull++;
if (pColData->numOfNone) {
SET_BIT1(pColData->pBitMap, pColData->nVal - 1, 1);
} else {
pColData->flag = HAS_NULL;
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue40(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward) {
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue40(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue00(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue42(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward) {
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue42(pColData, pData, nData);
} else {
pColData->flag = 0;
return tColDataAppendValue02(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue50(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
uint8_t bv = GET_BIT1(pColData->pBitMap, pColData->nVal - 1);
if (bv == 0) { // NONE ==> VALUE
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue50(pColData, pData, nData);
} else {
pColData->flag = HAS_VALUE;
return tColDataAppendValue40(pColData, pData, nData);
}
} else if (forward) { // VALUE ==> VALUE
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue50(pColData, pData, nData);
} else {
pColData->flag = HAS_NONE;
return tColDataAppendValue10(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue52(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
uint8_t bv = GET_BIT1(pColData->pBitMap, pColData->nVal - 1);
if (bv == 0) { // NONE ==> NULL
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue52(pColData, pData, nData);
} else {
pColData->flag &= ~HAS_NONE;
return tColDataAppendValue42(pColData, pData, nData);
}
} else if (forward) { // VALUE ==> NULL
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue52(pColData, pData, nData);
} else {
pColData->flag &= ~HAS_VALUE;
return tColDataAppendValue12(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue60(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward) {
if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) { // NULL ==> VALUE
pColData->numOfNull--;
pColData->nVal--;
if (pColData->numOfNull) {
return tColDataAppendValue60(pColData, pData, nData);
} else {
pColData->flag = HAS_VALUE;
return tColDataAppendValue40(pColData, pData, nData);
}
} else { // VALUE ==> VALUE
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue60(pColData, pData, nData);
} else {
pColData->flag = HAS_NULL;
return tColDataAppendValue20(pColData, pData, nData);
}
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue62(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
if (forward && (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 1)) { // VALUE ==> NULL
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue62(pColData, pData, nData);
} else {
pColData->flag = HAS_NULL;
return tColDataAppendValue20(pColData, pData, nData);
}
}
return 0;
}
static FORCE_INLINE int32_t tColDataUpdateValue70(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
int32_t code = 0;
uint8_t bv = GET_BIT2(pColData->pBitMap, pColData->nVal - 1);
if (bv == 0) { // NONE ==> VALUE
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue70(pColData, pData, nData);
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) - 1);
}
pColData->flag &= ~HAS_NONE;
return tColDataAppendValue60(pColData, pData, nData);
}
} else if (bv == 1) { // NULL ==> VALUE
if (forward) {
pColData->numOfNull--;
pColData->nVal--;
if (pColData->numOfNull) {
return tColDataAppendValue70(pColData, pData, nData);
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) ? 1 : 0);
}
pColData->flag &= ~HAS_NULL;
return tColDataAppendValue50(pColData, pData, nData);
}
}
} else if (bv == 2) { // VALUE ==> VALUE
if (forward) {
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue70(pColData, pData, nData);
} else {
return tColDataPutValue(pColData, pData, nData);
}
}
} else {
ASSERT(0);
}
return 0;
}
static int32_t tColDataUpdateValue72(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) {
uint8_t bv = GET_BIT2(pColData->pBitMap, pColData->nVal - 1);
ASSERT(bv < 3);
if (bv == 0) { // NONE ==> NULL
pColData->numOfNone--;
pColData->nVal--;
if (pColData->numOfNone) {
return tColDataAppendValue72(pColData, pData, nData);
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) - 1);
}
pColData->flag &= ~HAS_NONE;
return tColDataAppendValue62(pColData, pData, nData);
}
} else if (bv == 2 && forward) { // VALUE ==> NULL
pColData->numOfValue--;
pColData->nVal--;
if (pColData->numOfValue) {
return tColDataAppendValue72(pColData, pData, nData);
} else {
for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) {
SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal));
}
pColData->flag &= ~HAS_VALUE;
return tColDataAppendValue32(pColData, pData, nData);
}
}
return 0;
}
static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) = {
{NULL, NULL, NULL}, // 0
{tColDataUpdateValue10, NULL, tColDataUpdateValue12}, // HAS_NONE
{tColDataUpdateValue20, NULL, NULL}, // HAS_NULL
{tColDataUpdateValue30, NULL, tColDataUpdateValue32}, // HAS_NULL|HAS_NONE
{tColDataUpdateValue40, NULL, tColDataUpdateValue42}, // HAS_VALUE
{tColDataUpdateValue50, NULL, tColDataUpdateValue52}, // HAS_VALUE|HAS_NONE
{tColDataUpdateValue60, NULL, tColDataUpdateValue62}, // HAS_VALUE|HAS_NULL
{tColDataUpdateValue70, NULL, tColDataUpdateValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE
// VALUE NONE NULL
};
int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward) {
ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
ASSERT(pColData->nVal > 0);
if (tColDataUpdateValueImpl[pColData->flag][pColVal->flag] == NULL) return 0;
return tColDataUpdateValueImpl[pColData->flag][pColVal->flag](
pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val,
pColVal->value.nData, forward);
}
static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE
*pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
}

2
source/common/src/tglobal.c
View File

@ -351,7 +351,9 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, 0) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
#if !defined(_ALPINE)
if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1;
#endif
if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1;

12
source/common/src/tmsg.c
View File

@ -3992,9 +3992,7 @@ int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) {
if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1;
}
if (tEncodeI64(&encoder, pReq->compactStartTime) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -4009,9 +4007,7 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
for (int32_t i = 0; i < 8; ++i) {
if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1;
}
if (tDecodeI64(&decoder, &pReq->compactStartTime) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -5489,6 +5485,7 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
}
if (tEncodeI8(&encoder, pReq->createStb) < 0) return -1;
if (tEncodeU64(&encoder, pReq->targetStbUid) < 0) return -1;
if (tEncodeI32(&encoder, taosArrayGetSize(pReq->fillNullCols)) < 0) return -1;
@ -5498,6 +5495,7 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeI16(&encoder, pCol->colId) < 0) return -1;
if (tEncodeI8(&encoder, pCol->type) < 0) return -1;
}
if (tEncodeI8(&encoder, pReq->igUpdate) < 0) return -1;
tEndEncode(&encoder);
@ -5581,6 +5579,8 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
}
}
if (tDecodeI8(&decoder, &pReq->igUpdate) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);

2
source/dnode/mgmt/mgmt_dnode/inc/dmInt.h
View File

@ -48,7 +48,7 @@ int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t dmProcessGrantReq(SRpcMsg *pMsg);
int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg);
// dmWorker.c
int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);

2
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -234,7 +234,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
code = dmProcessRetrieve(pMgmt, pMsg);
break;
case TDMT_MND_GRANT:
code = dmProcessGrantReq(pMsg);
code = dmProcessGrantReq(&pMgmt->pData->clusterId, pMsg);
break;
default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED;

2
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -49,7 +49,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
pMsg->info.node = pMgmt->pMnode;
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, get from mnode queue", pMsg);
dGTrace("msg:%p, get from mnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
int32_t code = mndProcessRpcMsg(pMsg);

8
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -311,7 +311,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t srcVgId = req.srcVgId;
int32_t dstVgId = req.dstVgId;
dInfo("vgId:%d, start to alter vnode hashrange[%u, %u), dstVgId:%d", req.srcVgId, req.hashBegin, req.hashEnd,
dInfo("vgId:%d, start to alter vnode hashrange:[%u, %u], dstVgId:%d", req.srcVgId, req.hashBegin, req.hashEnd,
req.dstVgId);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, srcVgId);
@ -342,7 +342,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
dInfo("vgId:%d, start to open vnode", dstVgId);
dInfo("vgId:%d, open vnode", dstVgId);
SVnode *pImpl = vnodeOpen(dstPath, pMgmt->pTfs, pMgmt->msgCb);
if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode at %s since %s", dstVgId, dstPath, terrstr());
@ -376,7 +376,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
int32_t vgId = alterReq.vgId;
dInfo("vgId:%d, start to alter vnode, replica:%d selfIndex:%d strict:%d", vgId, alterReq.replica, alterReq.selfIndex,
dInfo("vgId:%d, start to alter vnode replica:%d selfIndex:%d strict:%d", vgId, alterReq.replica, alterReq.selfIndex,
alterReq.strict);
for (int32_t i = 0; i < alterReq.replica; ++i) {
SReplica *pReplica = &alterReq.replicas[i];
@ -423,7 +423,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1;
}
dInfo("vgId:%d, start to open vnode", vgId);
dInfo("vgId:%d, close vnode", vgId);
SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb);
if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr());

2
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -57,7 +57,7 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
if (IsReq(pMsg)) {
if (code != 0) {
if (terrno != 0) code = terrno;
dGError("msg:%p, failed to process since %s", pMsg, terrstr(code));
dGError("msg:%p, failed to process since %s, type:%s", pMsg, terrstr(code), TMSG_INFO(pMsg->msgType));
}
vmSendRsp(pMsg, code);
}

2
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -328,6 +328,7 @@ typedef struct {
SDbCfg cfg;
SRWLatch lock;
int64_t stateTs;
int64_t compactStartTime;
} SDbObj;
typedef struct {
@ -649,6 +650,7 @@ typedef struct {
int64_t checkpointFreq; // ms
int64_t currentTick; // do not serialize
int64_t deleteMark;
int8_t igCheckUpdate;
} SStreamObj;
int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj);

2
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -81,7 +81,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
int32_t mndTransProcessRsp(SRpcMsg *pRsp);
void mndTransPullup(SMnode *pMnode);
int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans);
void mndTransExecute(SMnode *pMnode, STrans *pTrans);
void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader);
int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname);
#ifdef __cplusplus

1
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -43,6 +43,7 @@ int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVg
int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId, bool force);
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup,
SArray *pArray);
int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs);
void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

68
source/dnode/mnode/impl/src/mndDb.c
View File

@ -30,7 +30,7 @@
#include "systable.h"
#define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 54
#define DB_RESERVE_SIZE 46
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -126,6 +126,7 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashPrefix, _OVER)
SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashSuffix, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER)
SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -215,6 +216,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashPrefix, _OVER)
SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashSuffix, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER)
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pDb->lock);
@ -274,6 +276,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.replications = pNew->cfg.replications;
pOld->cfg.sstTrigger = pNew->cfg.sstTrigger;
pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
pOld->compactStartTime = pNew->compactStartTime;
taosWUnLockLatch(&pOld->lock);
return 0;
}
@ -1378,7 +1381,63 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
return 0;
}
static int32_t mndCompactDb(SMnode *pMnode, SDbObj *pDb) { return 0; }
static int32_t mndSetCompactDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) {
SDbObj dbObj = {0};
memcpy(&dbObj, pDb, sizeof(SDbObj));
dbObj.compactStartTime = compactTs;
SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
sdbFreeRaw(pCommitRaw);
return -1;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
return 0;
}
static int32_t mndSetCompactDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (mndVgroupInDb(pVgroup, pDb->uid)) {
if (mndBuildCompactVgroupAction(pMnode, pTrans, pDb, pVgroup, compactTs) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
return -1;
}
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndCompactDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
int64_t compactTs = taosGetTimestampMs();
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "compact-db");
if (pTrans == NULL) goto _OVER;
mInfo("trans:%d, used to compact db:%s", pTrans->id, pDb->name);
mndTransSetDbName(pTrans, pDb->name, NULL);
if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER;
if (mndSetCompactDbCommitLogs(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER;
if (mndSetCompactDbRedoActions(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
}
static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
@ -1402,10 +1461,11 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) {
goto _OVER;
}
code = mndCompactDb(pMnode, pDb);
code = mndCompactDb(pMnode, pReq, pDb);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER:
if (code != 0) {
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("db:%s, failed to process compact db req since %s", compactReq.db, terrstr());
}

4
source/dnode/mnode/impl/src/mndDef.c
View File

@ -78,6 +78,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
// 3.0.20
if (tEncodeI64(pEncoder, pObj->checkpointFreq) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->igCheckUpdate) < 0) return -1;
tEndEncode(pEncoder);
return pEncoder->pos;
@ -145,6 +146,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
// 3.0.20
if (sver >= 2) {
if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->igCheckUpdate) < 0) return -1;
}
tEndDecode(pDecoder);
return 0;
@ -493,7 +495,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
tlen += tEncodeSMqConsumerEp(buf, pConsumerEp);
cnt++;
}
if(cnt != sz) return -1;
if (cnt != sz) return -1;
tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp);
tlen += taosEncodeString(buf, pSub->dbName);
return tlen;

2
source/dnode/mnode/impl/src/mndGrant.c
View File

@ -129,7 +129,7 @@ void grantParseParameter() { mError("can't parsed parameter k"); }
void grantReset(SMnode *pMnode, EGrantType grant, uint64_t value) {}
void grantAdd(EGrantType grant, uint64_t value) {}
void grantRestore(EGrantType grant, uint64_t value) {}
int32_t dmProcessGrantReq(SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
int32_t dmProcessGrantReq(void* pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
#endif

4
source/dnode/mnode/impl/src/mndMain.c
View File

@ -669,9 +669,9 @@ _OVER:
mGDebug(
"msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d "
"role:%s, redirect numOfEps:%d inUse:%d",
"role:%s, redirect numOfEps:%d inUse:%d, type:%s",
pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored,
syncStr(state.restored), epSet.numOfEps, epSet.inUse);
syncStr(state.restored), epSet.numOfEps, epSet.inUse, TMSG_INFO(pMsg->msgType));
if (epSet.numOfEps <= 0) return -1;

33
source/dnode/mnode/impl/src/mndStream.c
View File

@ -31,6 +31,8 @@
#define MND_STREAM_VER_NUMBER 2
#define MND_STREAM_RESERVE_SIZE 64
#define MND_STREAM_MAX_NUM 10
static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream);
@ -295,6 +297,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
pObj->triggerParam = pCreate->maxDelay;
pObj->watermark = pCreate->watermark;
pObj->fillHistory = pCreate->fillHistory;
pObj->igCheckUpdate = pCreate->igUpdate;
memcpy(pObj->sourceDb, pCreate->sourceDB, TSDB_DB_FNAME_LEN);
SDbObj *pSourceDb = mndAcquireDb(pMnode, pCreate->sourceDB);
@ -377,6 +380,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
.triggerType = pObj->trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->trigger,
.watermark = pObj->watermark,
.igExpired = pObj->igExpired,
.igCheckUpdate = pObj->igCheckUpdate,
};
// using ast and param to build physical plan
@ -666,6 +670,35 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER;
}
{
int32_t numOfStream = 0;
SStreamObj *pStream = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) {
if (numOfStream > MND_STREAM_MAX_NUM) {
mError("too many streams, no more than 10 for each database");
terrno = TSDB_CODE_MND_TOO_MANY_STREAMS;
goto _OVER;
}
break;
}
if (pStream->sourceDbUid == streamObj.sourceDbUid) {
++numOfStream;
}
sdbRelease(pMnode->pSdb, pStream);
if (numOfStream > MND_STREAM_MAX_NUM) {
mError("too many streams, no more than 10 for each database");
terrno = TSDB_CODE_MND_TOO_MANY_STREAMS;
goto _OVER;
}
}
}
pDb = mndAcquireDb(pMnode, streamObj.sourceDb);
if (pDb->cfg.replications != 1) {
mError("stream source db must have only 1 replica, but %s has %d", pDb->name, pDb->cfg.replications);

13
source/dnode/mnode/impl/src/mndSync.c
View File

@ -85,7 +85,11 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta
pRaw, pMgmt->transSec, pMgmt->transSeq);
if (pMeta->code == 0) {
sdbWriteWithoutFree(pMnode->pSdb, pRaw);
int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pRaw);
if (code != 0) {
mError("trans:%d, failed to write to sdb since %s", transId, terrstr());
return 0;
}
sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex);
}
@ -110,8 +114,9 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta
taosThreadMutexUnlock(&pMgmt->lock);
STrans *pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans != NULL) {
mInfo("trans:%d, execute in mnode which not leader or sync timeout", transId);
mndTransExecute(pMnode, pTrans);
mInfo("trans:%d, execute in mnode which not leader or sync timeout, createTime:%" PRId64 " saved trans:%d",
transId, pTrans->createdTime, pMgmt->transId);
mndTransExecute(pMnode, pTrans, false);
mndReleaseTrans(pMnode, pTrans);
// sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA);
} else {
@ -368,7 +373,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
taosThreadMutexLock(&pMgmt->lock);
pMgmt->errCode = 0;
if (pMgmt->transId != 0) {
if (pMgmt->transId != 0 /* && pMgmt->transId != transId*/) {
mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId);
taosThreadMutexUnlock(&pMgmt->lock);
rpcFreeCont(req.pCont);

50
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -572,8 +572,20 @@ static void mndTransUpdateActions(SArray *pOldArray, SArray *pNewArray) {
}
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
mTrace("trans:%d, perform update action, old row:%p stage:%s, new row:%p stage:%s", pOld->id, pOld,
mndTransStr(pOld->stage), pNew, mndTransStr(pNew->stage));
mTrace("trans:%d, perform update action, old row:%p stage:%s create:%" PRId64 ", new row:%p stage:%s create:%" PRId64,
pOld->id, pOld, mndTransStr(pOld->stage), pOld->createdTime, pNew, mndTransStr(pNew->stage),
pNew->createdTime);
if (pOld->createdTime != pNew->createdTime) {
mError("trans:%d, failed to perform update action since createTime not match, old row:%p stage:%s create:%" PRId64
", new row:%p stage:%s create:%" PRId64,
pOld->id, pOld, mndTransStr(pOld->stage), pOld->createdTime, pNew, mndTransStr(pNew->stage),
pNew->createdTime);
// only occured while sync timeout
terrno = TSDB_CODE_MND_TRNAS_SYNC_TIMEOUT;
return -1;
}
mndTransUpdateActions(pOld->redoActions, pNew->redoActions);
mndTransUpdateActions(pOld->undoActions, pNew->undoActions);
mndTransUpdateActions(pOld->commitActions, pNew->commitActions);
@ -779,16 +791,18 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) {
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("trans:%d, sync to other mnodes, stage:%s", pTrans->id, mndTransStr(pTrans->stage));
mInfo("trans:%d, sync to other mnodes, stage:%s createTime:%" PRId64, pTrans->id, mndTransStr(pTrans->stage),
pTrans->createdTime);
int32_t code = mndSyncPropose(pMnode, pRaw, pTrans->id);
if (code != 0) {
mError("trans:%d, failed to sync, errno:%s code:%s", pTrans->id, terrstr(), tstrerror(code));
mError("trans:%d, failed to sync, errno:%s code:%s createTime:%" PRId64 " saved trans:%d", pTrans->id, terrstr(),
tstrerror(code), pTrans->createdTime, pMnode->syncMgmt.transId);
sdbFreeRaw(pRaw);
return -1;
}
sdbFreeRaw(pRaw);
mInfo("trans:%d, sync finished", pTrans->id);
mInfo("trans:%d, sync finished, createTime:%" PRId64, pTrans->id, pTrans->createdTime);
return 0;
}
@ -891,7 +905,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
pTrans->rpcRsp = NULL;
pTrans->rpcRspLen = 0;
mndTransExecute(pMnode, pNew);
mndTransExecute(pMnode, pNew, true);
mndReleaseTrans(pMnode, pNew);
return 0;
}
@ -1054,7 +1068,7 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) {
mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, mndTransStr(pAction->stage),
action, pRsp->code, pAction->acceptableCode, pAction->retryCode);
mndTransExecute(pMnode, pTrans);
mndTransExecute(pMnode, pTrans, true);
_OVER:
mndReleaseTrans(pMnode, pTrans);
@ -1483,15 +1497,17 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr());
}
mInfo("trans:%d, execute finished, code:0x%x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes);
mInfo("trans:%d, execute finished, code:0x%x, failedTimes:%d createTime:%" PRId64, pTrans->id, pTrans->code,
pTrans->failedTimes, pTrans->createdTime);
return continueExec;
}
void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
void mndTransExecute(SMnode *pMnode, STrans *pTrans, bool isLeader) {
bool continueExec = true;
while (continueExec) {
mInfo("trans:%d, continue to execute, stage:%s", pTrans->id, mndTransStr(pTrans->stage));
mInfo("trans:%d, continue to execute, stage:%s createTime:%" PRId64 " leader:%d", pTrans->id,
mndTransStr(pTrans->stage), pTrans->createdTime, isLeader);
pTrans->lastExecTime = taosGetTimestampMs();
switch (pTrans->stage) {
case TRN_STAGE_PREPARE:
@ -1501,13 +1517,23 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
continueExec = mndTransPerformRedoActionStage(pMnode, pTrans);
break;
case TRN_STAGE_COMMIT:
if (isLeader) {
continueExec = mndTransPerformCommitStage(pMnode, pTrans);
} else {
mInfo("trans:%d, can not commit since not leader", pTrans->id);
continueExec = false;
}
break;
case TRN_STAGE_COMMIT_ACTION:
continueExec = mndTransPerformCommitActionStage(pMnode, pTrans);
break;
case TRN_STAGE_ROLLBACK:
if (isLeader) {
continueExec = mndTransPerformRollbackStage(pMnode, pTrans);
} else {
mInfo("trans:%d, can not rollback since not leader", pTrans->id);
continueExec = false;
}
break;
case TRN_STAGE_UNDO_ACTION:
continueExec = mndTransPerformUndoActionStage(pMnode, pTrans);
@ -1550,7 +1576,7 @@ int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) {
pAction->errCode = 0;
}
mndTransExecute(pMnode, pTrans);
mndTransExecute(pMnode, pTrans, true);
return 0;
}
@ -1608,7 +1634,7 @@ void mndTransPullup(SMnode *pMnode) {
int32_t *pTransId = taosArrayGet(pArray, i);
STrans *pTrans = mndAcquireTrans(pMnode, *pTransId);
if (pTrans != NULL) {
mndTransExecute(pMnode, pTrans);
mndTransExecute(pMnode, pTrans, true);
}
mndReleaseTrans(pMnode, pTrans);
}

94
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -428,7 +428,7 @@ static void *mndBuildAlterVnodeHashRangeReq(SMnode *pMnode, SVgObj *pVgroup, int
.hashEnd = pVgroup->hashEnd,
};
mInfo("vgId:%d, build alter vnode hashrange req, dstVgId:%d, begin:%u, end:%u", pVgroup->vgId, dstVgId,
mInfo("vgId:%d, build alter vnode hashrange req, dstVgId:%d, hashrange:[%u, %u]", pVgroup->vgId, dstVgId,
pVgroup->hashBegin, pVgroup->hashEnd);
int32_t contLen = tSerializeSAlterVnodeHashRangeReq(NULL, 0, &alterReq);
if (contLen < 0) {
@ -528,7 +528,12 @@ SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) {
return pArray;
}
static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) { return *dnode1Id >= *dnode2Id ? 1 : 0; }
static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) {
if (*dnode1Id == *dnode2Id) {
return 0;
}
return *dnode1Id > *dnode2Id ? 1 : -1;
}
static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float ratio) {
float totalDnodes = pDnode->numOfVnodes + (float)pDnode->numOfOtherNodes * ratio + additionDnodes;
@ -538,7 +543,10 @@ static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float r
static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) {
float d1Score = mndGetDnodeScore(pDnode1, 0, 0.9);
float d2Score = mndGetDnodeScore(pDnode2, 0, 0.9);
return d1Score >= d2Score ? 1 : 0;
if (d1Score == d2Score) {
return 0;
}
return d1Score > d2Score ? 1 : -1;
}
void mndSortVnodeGid(SVgObj *pVgroup) {
@ -796,6 +804,13 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false);
// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
// if (pDb == NULL || pDb->compactStartTime <= 0) {
// colDataAppendNULL(pColInfo, numOfRows);
// } else {
// colDataAppend(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false);
// }
numOfRows++;
sdbRelease(pSdb, pVgroup);
}
@ -1168,7 +1183,8 @@ int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD
return 0;
}
static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) {
static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
int32_t dnodeId) {
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId);
if (pDnode == NULL) return -1;
@ -1894,12 +1910,6 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
}
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER;
mInfo("vgId:%d, vgroup info after adjust replica, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
for (int32_t i = 0; i < newVg1.replica; ++i) {
mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId);
}
SVgObj newVg2 = {0};
memcpy(&newVg2, &newVg1, sizeof(SVgObj));
newVg1.replica = 1;
@ -1911,13 +1921,13 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj
memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid));
memset(&newVg2.vnodeGid[1], 0, sizeof(SVnodeGid));
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId,
newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
mInfo("vgId:%d, vgroup info after split, replica:%d hashrange:[%u, %u] vnode:0 dnode:%d", newVg1.vgId, newVg1.replica,
newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId);
for (int32_t i = 0; i < newVg1.replica; ++i) {
mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId);
}
mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg2.vgId,
newVg2.replica, newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId);
mInfo("vgId:%d, vgroup info after split, replica:%d hashrange:[%u, %u] vnode:0 dnode:%d", newVg2.vgId, newVg2.replica,
newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId);
for (int32_t i = 0; i < newVg1.replica; ++i) {
mInfo("vgId:%d, vnode:%d dnode:%d", newVg2.vgId, i, newVg2.vnodeGid[i].dnodeId);
}
@ -2198,3 +2208,59 @@ _OVER:
}
bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; }
static void *mndBuildCompactVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen,
int64_t compactTs) {
SCompactVnodeReq compactReq = {0};
compactReq.dbUid = pDb->uid;
compactReq.compactStartTime = compactTs;
tstrncpy(compactReq.db, pDb->name, TSDB_DB_FNAME_LEN);
mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId);
int32_t contLen = tSerializeSCompactVnodeReq(NULL, 0, &compactReq);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
contLen += sizeof(SMsgHead);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
SMsgHead *pHead = pReq;
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSCompactVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &compactReq);
*pContLen = contLen;
return pReq;
}
static int32_t mndAddCompactVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
int64_t compactTs) {
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t contLen = 0;
void *pReq = mndBuildCompactVnodeReq(pMnode, pDb, pVgroup, &contLen, compactTs);
if (pReq == NULL) return -1;
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_VND_COMPACT;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
return 0;
}
int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs) {
if (mndAddCompactVnodeAction(pMnode, pTrans, pDb, pVgroup, compactTs) != 0) return -1;
return 0;
}

2
source/dnode/vnode/CMakeLists.txt
View File

@ -14,6 +14,7 @@ target_sources(
"src/vnd/vnodeSvr.c"
"src/vnd/vnodeSync.c"
"src/vnd/vnodeSnapshot.c"
"src/vnd/vnodeCompact.c"
# meta
"src/meta/metaOpen.c"
@ -53,6 +54,7 @@ target_sources(
"src/tsdb/tsdbDiskData.c"
"src/tsdb/tsdbCompact.c"
"src/tsdb/tsdbMergeTree.c"
"src/tsdb/tsdbDataIter.c"
# tq
"src/tq/tq.c"

1
source/dnode/vnode/inc/vnode.h
View File

@ -181,7 +181,6 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL
void tsdbReaderClose(STsdbReader *pReader);
bool tsdbNextDataBlock(STsdbReader *pReader);
void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
void tsdbReleaseDataBlock(STsdbReader *pReader);
SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);

2
source/dnode/vnode/src/inc/sma.h
View File

@ -209,6 +209,8 @@ static FORCE_INLINE void tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
smaDebug("vgId:%d, unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref);
}
int32_t smaPreClose(SSma *pSma);
// rsma
void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree);
int32_t tdRSmaFSOpen(SSma *pSma, int64_t version, int8_t rollback);

90
source/dnode/vnode/src/inc/tsdb.h
View File

@ -69,6 +69,8 @@ typedef struct SDiskCol SDiskCol;
typedef struct SDiskData SDiskData;
typedef struct SDiskDataBuilder SDiskDataBuilder;
typedef struct SBlkInfo SBlkInfo;
typedef struct STsdbDataIter2 STsdbDataIter2;
typedef struct STsdbFilterInfo STsdbFilterInfo;
#define TSDBROW_ROW_FMT ((int8_t)0x0)
#define TSDBROW_COL_FMT ((int8_t)0x1)
@ -163,6 +165,8 @@ void tBlockDataDestroy(SBlockData *pBlockData);
int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid);
void tBlockDataReset(SBlockData *pBlockData);
int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid);
int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
void tBlockDataClear(SBlockData *pBlockData);
void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[],
@ -277,6 +281,7 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk);
int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg);
int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData);
int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData);
int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
// SDelFWriter
@ -308,6 +313,26 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB
int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder);
int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId);
int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo);
// tsdbDataIter.c ==============================================================================================
#define TSDB_MEM_TABLE_DATA_ITER 0
#define TSDB_DATA_FILE_DATA_ITER 1
#define TSDB_STT_FILE_DATA_ITER 2
#define TSDB_TOMB_FILE_DATA_ITER 3
#define TSDB_FILTER_FLAG_BY_VERSION 0x1
#define TSDB_FILTER_FLAG_BY_TABLEID 0x2
#define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2 *)(((char *)pNode) - offsetof(STsdbDataIter2, rbtn)))
/* open */
int32_t tsdbOpenDataFileDataIter(SDataFReader *pReader, STsdbDataIter2 **ppIter);
int32_t tsdbOpenSttFileDataIter(SDataFReader *pReader, int32_t iStt, STsdbDataIter2 **ppIter);
int32_t tsdbOpenTombFileDataIter(SDelFReader *pReader, STsdbDataIter2 **ppIter);
/* close */
void tsdbCloseDataIter2(STsdbDataIter2 *pIter);
/* cmpr */
int32_t tsdbDataIterCmprFn(const SRBTreeNode *pNode1, const SRBTreeNode *pNode2);
/* next */
int32_t tsdbDataIterNext2(STsdbDataIter2 *pIter, STsdbFilterInfo *pFilterInfo);
// structs =======================
struct STsdbFS {
@ -325,6 +350,8 @@ struct STsdb {
STsdbFS fs;
SLRUCache *lruCache;
TdThreadMutex lruMutex;
SLRUCache *biCache;
TdThreadMutex biMutex;
};
struct TSDBKEY {
@ -642,6 +669,7 @@ struct SDataFReader {
uint8_t *aBuf[3];
};
// NOTE: do NOT change the order of the fields
typedef struct {
int64_t suid;
int64_t uid;
@ -764,6 +792,9 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr,
int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h);
int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h);
int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHandle **handle);
int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h);
int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
@ -825,6 +856,65 @@ static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
return pIter->pRow;
}
int32_t tRowInfoCmprFn(const void *p1, const void *p2);
typedef struct {
int64_t suid;
int64_t uid;
SDelData delData;
} SDelInfo;
struct STsdbDataIter2 {
STsdbDataIter2 *next;
SRBTreeNode rbtn;
int32_t type;
SRowInfo rowInfo;
SDelInfo delInfo;
union {
// TSDB_MEM_TABLE_DATA_ITER
struct {
SMemTable *pMemTable;
} mIter;
// TSDB_DATA_FILE_DATA_ITER
struct {
SDataFReader *pReader;
SArray *aBlockIdx; // SArray<SBlockIdx>
SMapData mDataBlk;
SBlockData bData;
int32_t iBlockIdx;
int32_t iDataBlk;
int32_t iRow;
} dIter;
// TSDB_STT_FILE_DATA_ITER
struct {
SDataFReader *pReader;
int32_t iStt;
SArray *aSttBlk;
SBlockData bData;
int32_t iSttBlk;
int32_t iRow;
} sIter;
// TSDB_TOMB_FILE_DATA_ITER
struct {
SDelFReader *pReader;
SArray *aDelIdx;
SArray *aDelData;
int32_t iDelIdx;
int32_t iDelData;
} tIter;
};
};
struct STsdbFilterInfo {
int32_t flag;
int64_t sver;
int64_t ever;
TABLEID tbid;
};
#ifdef __cplusplus
}
#endif

6
source/dnode/vnode/src/inc/vnd.h
View File

@ -100,12 +100,16 @@ int32_t vnodeShouldCommit(SVnode* pVnode);
void vnodeUpdCommitSched(SVnode* pVnode);
void vnodeRollback(SVnode* pVnode);
int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg);
int32_t vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo);
int32_t vnodeCommitInfo(const char* dir);
int32_t vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo);
int32_t vnodeSyncCommit(SVnode* pVnode);
int32_t vnodeAsyncCommit(SVnode* pVnode);
bool vnodeShouldRollback(SVnode* pVnode);
// vnodeCompact.c
int32_t vnodeAsyncCompact(SVnode* pVnode);
int32_t vnodeSyncCompact(SVnode* pVnode);
// vnodeSync.c
int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
int32_t vnodeSyncStart(SVnode* pVnode);

8
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -76,6 +76,7 @@ typedef struct SRSmaSnapReader SRSmaSnapReader;
typedef struct SRSmaSnapWriter SRSmaSnapWriter;
typedef struct SSnapDataHdr SSnapDataHdr;
typedef struct SCommitInfo SCommitInfo;
typedef struct SCompactInfo SCompactInfo;
typedef struct SQueryNode SQueryNode;
#define VNODE_META_DIR "meta"
@ -173,6 +174,7 @@ int tsdbClose(STsdb** pTsdb);
int32_t tsdbBegin(STsdb* pTsdb);
int32_t tsdbPrepareCommit(STsdb* pTsdb);
int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo);
int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
int32_t tsdbFinishCommit(STsdb* pTsdb);
int32_t tsdbRollbackCommit(STsdb* pTsdb);
int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now);
@ -452,6 +454,12 @@ struct SCommitInfo {
TXN* txn;
};
struct SCompactInfo {
SVnode* pVnode;
int32_t flag;
int64_t commitID;
};
#ifdef __cplusplus
}
#endif

3
source/dnode/vnode/src/meta/metaTable.c
View File

@ -1409,6 +1409,9 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
int32_t ret = 0;
// get super table
if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) {
metaError("vgId:%d, failed to get stable suid for update. version:%" PRId64, TD_VID(pMeta->pVnode),
pCtbEntry->version);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
ret = -1;
goto end;
}

34
source/dnode/vnode/src/sma/smaCommit.c
View File

@ -17,18 +17,26 @@
extern SSmaMgmt smaMgmt;
static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma);
static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit);
static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo);
static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma);
static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat);
/**
* @brief only applicable to Rollup SMA
*
* @param pSma
* @return int32_t
*/
int32_t smaPreClose(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma, false); }
/**
* @brief async commit, only applicable to Rollup SMA
*
* @param pSma
* @return int32_t
*/
int32_t smaPrepareAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); }
int32_t smaPrepareAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma, true); }
/**
* @brief async commit, only applicable to Rollup SMA
@ -122,9 +130,10 @@ _exit:
* 2) Wait all running fetch task finish to fetch and put submitMsg into level 2/3 wQueue(blocking level 1 write)
*
* @param pSma
* @param isCommit
* @return int32_t
*/
static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) {
int32_t code = 0;
int32_t lino = 0;
@ -139,6 +148,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
// step 1: set rsma stat
atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
if (isCommit) {
while (atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 1) != 0) {
++nLoops;
if (nLoops > 1000) {
@ -146,21 +156,22 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
nLoops = 0;
}
}
pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied;
if (ASSERTS(pRSmaStat->commitAppliedVer >= 0, "commit applied version %" PRIi64 " < 0",
if (ASSERTS(pRSmaStat->commitAppliedVer >= -1, "commit applied version %" PRIi64 " < -1",
pRSmaStat->commitAppliedVer)) {
code = TSDB_CODE_APP_ERROR;
TSDB_CHECK_CODE(code, lino, _exit);
}
}
// step 2: wait for all triggered fetch tasks to finish
nLoops = 0;
while (1) {
if (T_REF_VAL_GET(pStat) == 0) {
smaDebug("vgId:%d, rsma commit, fetch tasks are all finished", SMA_VID(pSma));
if (atomic_load_32(&pRSmaStat->nFetchAll) <= 0) {
smaDebug("vgId:%d, rsma commit:%d, fetch tasks are all finished", SMA_VID(pSma), isCommit);
break;
} else {
smaDebug("vgId:%d, rsma commit, fetch tasks are not all finished yet", SMA_VID(pSma));
smaDebug("vgId:%d, rsma commit%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit);
}
++nLoops;
if (nLoops > 1000) {
@ -174,7 +185,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
* 1) This is high cost task and should not put in asyncPreCommit originally.
* 2) But, if put in asyncCommit, would trigger taskInfo cloning frequently.
*/
smaInfo("vgId:%d, rsma commit, wait for all items to be consumed, TID:%p", SMA_VID(pSma),
smaInfo("vgId:%d, rsma commit:%d, wait for all items to be consumed, TID:%p", SMA_VID(pSma), isCommit,
(void *)taosGetSelfPthreadId());
nLoops = 0;
while (atomic_load_64(&pRSmaStat->nBufItems) > 0) {
@ -184,6 +195,9 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
nLoops = 0;
}
}
if (!isCommit) goto _exit;
smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat));
TSDB_CHECK_CODE(code, lino, _exit);
@ -223,7 +237,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
_exit:
if (code) {
smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code));
smaError("vgId:%d, %s failed at line %d since %s(%d)", SMA_VID(pSma), __func__, lino, tstrerror(code), isCommit);
}
return code;
}

1
source/dnode/vnode/src/sma/smaOpen.c
View File

@ -167,6 +167,7 @@ _exit:
int32_t smaClose(SSma *pSma) {
if (pSma) {
smaPreClose(pSma);
taosThreadMutexDestroy(&pSma->mutex);
SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));

Some files were not shown because too many files have changed in this diff Show More