xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/ref

This commit is contained in:
Haojun Liao 2024-11-05 10:50:28 +08:00 committed by GitHub
parent a685c1394a 9588581b5d
commit 8148f26d30
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
318 changed files with 29948 additions and 8305 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
.github/workflows/taoskeeper-ci.yml vendored Normal file
View File

@ -0,0 +1,58 @@
name: TaosKeeper CI
on:
push:
paths:
- tools/keeper/**
jobs:
build:
runs-on: ubuntu-latest
name: Run unit tests
steps:
- name: Checkout the repository
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.18
- name: Install system dependencies
run: |
sudo apt update -y
sudo apt install -y build-essential cmake libgeos-dev
- name: Install TDengine
run: |
mkdir debug
cd debug
cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false
make -j 4
sudo make install
which taosd
which taosadapter
which taoskeeper
- name: Start taosd
run: |
cp /etc/taos/taos.cfg ./
sudo echo "supportVnodes 256" >> taos.cfg
nohup sudo taosd -c taos.cfg &
- name: Start taosadapter
run: nohup sudo taosadapter &
- name: Run tests with coverage
working-directory: tools/keeper
run: |
go mod tidy
sudo go test -v -ldflags="-X 'github.com/taosdata/taoskeeper/version.IsEnterprise=true'" -coverpkg=./... -coverprofile=coverage.out ./...
go tool cover -func=coverage.out
- name: Clean up
if: always()
run: |
if pgrep taosd; then sudo pkill taosd; fi
if pgrep taosadapter; then sudo pkill taosadapter; fi

2
README-CN.md
View File

@ -348,7 +348,7 @@ TDengine 提供了丰富的应用程序开发接口，其中包括 C/C++、Java
# 成为社区贡献者
点击 [这里](https://www.taosdata.com/cn/contributor/)，了解如何成为 TDengine 的贡献者。
点击 [这里](https://www.taosdata.com/contributor)，了解如何成为 TDengine 的贡献者。
# 加入技术交流群

55
cmake/cmake.define
View File

@ -1,6 +1,7 @@
cmake_minimum_required(VERSION 3.0)
set(CMAKE_VERBOSE_MAKEFILE FALSE)
set(TD_BUILD_TAOSA_INTERNAL FALSE)
set(TD_BUILD_KEEPER_INTERNAL FALSE)
# set output directory
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
@ -57,6 +58,19 @@ IF(TD_BUILD_HTTP)
ADD_DEFINITIONS(-DHTTP_EMBEDDED)
ENDIF()
IF("${BUILD_KEEPER}" STREQUAL "")
SET(TD_BUILD_KEEPER FALSE)
ELSEIF(${BUILD_KEEPER} MATCHES "false")
SET(TD_BUILD_KEEPER FALSE)
ELSEIF(${BUILD_KEEPER} MATCHES "true")
SET(TD_BUILD_KEEPER TRUE)
ELSEIF(${BUILD_KEEPER} MATCHES "internal")
SET(TD_BUILD_KEEPER FALSE)
SET(TD_BUILD_KEEPER_INTERNAL TRUE)
ELSE()
SET(TD_BUILD_KEEPER FALSE)
ENDIF()
IF("${BUILD_TOOLS}" STREQUAL "")
IF(TD_LINUX)
IF(TD_ARM_32)
@ -177,48 +191,11 @@ ELSE()
SET(COMPILER_SUPPORT_AVX512VL false)
ELSE()
CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA)
CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX)
CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2)
CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F)
CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512BMI)
CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL)
INCLUDE(CheckCSourceRuns)
SET(CMAKE_REQUIRED_FLAGS "-mavx")
check_c_source_runs("
#include <immintrin.h>
int main() {
__m256d a, b, c;
double buf[4] = {0};
a = _mm256_loadu_pd(buf);
b = _mm256_loadu_pd(buf);
c = _mm256_add_pd(a, b);
_mm256_storeu_pd(buf, c);
for (int i = 0; i < sizeof(buf) / sizeof(buf[0]); ++i) {
IF (buf[i] != 0) {
return 1;
}
}
return 0;
}
" COMPILER_SUPPORT_AVX)
SET(CMAKE_REQUIRED_FLAGS "-mavx2")
check_c_source_runs("
#include <immintrin.h>
int main() {
__m256i a, b, c;
int buf[8] = {0};
a = _mm256_loadu_si256((__m256i *)buf);
b = _mm256_loadu_si256((__m256i *)buf);
c = _mm256_and_si256(a, b);
_mm256_storeu_si256((__m256i *)buf, c);
for (int i = 0; i < sizeof(buf) / sizeof(buf[0]); ++i) {
IF (buf[i] != 0) {
return 1;
}
}
return 0;
}
" COMPILER_SUPPORT_AVX2)
ENDIF()
IF(COMPILER_SUPPORT_SSE42)

8
contrib/CMakeLists.txt
View File

@ -261,11 +261,19 @@ if(${TD_DARWIN})
endif(${TD_DARWIN})
add_subdirectory(zlib EXCLUDE_FROM_ALL)
if(${TD_DARWIN})
target_compile_options(zlibstatic PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories(
zlibstatic
PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib
)
if(${TD_DARWIN})
target_compile_options(zlib PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories(
zlib
PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib

2
docs/en/03-intro.md
View File

@ -46,7 +46,7 @@ For more details on features, please read through the entire documentation.
By making full use of [characteristics of time series data](https://tdengine.com/characteristics-of-time-series-data/), TDengine differentiates itself from other time series databases with the following advantages.
- **[High-Performance](https://tdengine.com/high-performance/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression.
- **[High-Performance](https://tdengine.com/high-performance/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while outperforming other time-series databases for data ingestion, querying and data compression.
- **[Simplified Solution](https://tdengine.com/comprehensive-industrial-data-solution/)**: Through built-in caching, stream processing and data subscription features, TDengine provides a simplified solution for time-series data processing. It reduces system design complexity and operation costs significantly.

4
docs/en/04-get-started/01-docker.md
View File

@ -75,9 +75,9 @@ taos>
## TDegnine Graphic User Interface
From TDengine 3.3.0.0, there is a new componenet called `taos-explorer` added in the TDengine docker image. You can use it to manage the databases, super tables, child tables, and data in your TDengine system. There are also some features only available in TDengine Enterprise Edition, please contact TDengine sales team in case you need these features.
From TDengine 3.3.0.0, there is a new component called `taos-explorer` added in the TDengine docker image. You can use it to manage the databases, super tables, child tables, and data in your TDengine system. There are also some features only available in TDengine Enterprise Edition, please contact TDengine sales team in case you need these features.
To use taos-explorer in the container, you need to access the host port mapped from container port 6060. Assuming the host name is abc.com, and the port used on host is 6060, you need to access `http://abc.com:6060`. taos-explorer uses port 6060 by default in the container. When you use it the first time, you need to register with your enterprise email, then can logon using your user name and password in the TDengine database management system.
To use taos-explorer in the container, you need to access the host port mapped from container port 6060. Assuming the host name is abc.com, and the port used on host is 6060, you need to access `http://abc.com:6060`. taos-explorer uses port 6060 by default in the container. The default username and password to log in to the TDengine Database Management System is "root/taosdata".
## Test data insert performance

3
docs/en/14-reference/01-components/10-taosbenchmark
View File

@ -364,6 +364,9 @@ The configuration parameters for specifying super table tag columns and data col
- **min**: The minimum value of the column/label of the data type. The generated value will equal or large than the minimum value.
- **max**: The maximum value of the column/label of the data type. The generated value will less than the maximum value.
- **scalingFactor**: Floating-point precision enhancement factor, which takes effect only when the data type is float/double. It has a valid range of positive integers from 1 to 1,000,000. It is used to enhance the precision of generated floating-point numbers, particularly when the min or max values are small. This property enhances the precision after the decimal point by powers of 10: scalingFactor of 10 indicates an enhancement of 1 decimal precision, 100 indicates an enhancement of 2 decimal precision, and so on.
- **fun**: This column of data is filled with functions. Currently, only the sin and cos functions are supported. The input parameter is the timestamp and converted to an angle value. The conversion formula is: angle x=input time column ts value % 360. At the same time, it supports coefficient adjustment and random fluctuation factor adjustment, presented in a fixed format expression, such as fun="10\*sin(x)+100\*random(5)", where x represents the angle, ranging from 0 to 360 degrees, and the growth step size is consistent with the time column step size. 10 represents the coefficient of multiplication, 100 represents the coefficient of addition or subtraction, and 5 represents the fluctuation range within a random range of 5%. The currently supported data types are int, bigint, float, and double. Note: The expression is fixed and cannot be reversed.
- **values**: The value field of the nchar/binary column/label, which will be chosen randomly from the values.

2
docs/en/14-reference/03-taos-sql/10-function.md
View File

@ -1187,7 +1187,7 @@ CSUM(expr)
### DERIVATIVE
```sql
DERIVATIVE(expr, time_inerval, ignore_negative)
DERIVATIVE(expr, time_interval, ignore_negative)
ignore_negative: {
0

6
docs/en/14-reference/03-taos-sql/21-node.md
View File

@ -27,11 +27,15 @@ The preceding SQL command shows all dnodes in the cluster with the ID, endpoint,
## Delete a DNODE
```sql
DROP DNODE dnode_id
DROP DNODE dnode_id [force] [unsafe]
```
Note that deleting a dnode does not stop its process. You must stop the process after the dnode is deleted.
Only online node is allowed to be deleted. Drop is executed with force option if the offline node need to be deleted.
Drop is executed with unsafe option if the node with single replica is offline, and the data on it is not able to be restored.
## Modify Dnode Configuration
```sql

4
docs/en/14-reference/03-taos-sql/31-compress.md
View File

@ -41,7 +41,7 @@ In this article, it specifically refers to the level within the secondary compre
### Create Table with Compression
```sql
CREATE [dbname.]tabname (colName colType [ENCODE 'encode_type'] [COMPRESS 'compress_type' [LEVEL 'level'], [, other cerate_definition]...])
CREATE [dbname.]tabname (colName colType [ENCODE 'encode_type'] [COMPRESS 'compress_type' [LEVEL 'level'], [, other create_definition]...])
```
**Parameter Description**
@ -58,7 +58,7 @@ CREATE [dbname.]tabname (colName colType [ENCODE 'encode_type'] [COMPRESS 'compr
### Change Compression Method
```sql
ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'ecode_type'] [COMPRESS 'compress_type'] [LEVEL "high"]
ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL "high"]
```
**Parameter Description**

2
docs/en/14-reference/05-connectors/60-rest-api.mdx
View File

@ -125,7 +125,7 @@ where `TOKEN` is the string after Base64 encoding of `{username}:{password}`, e.
Starting from `TDengine 3.0.3.0`, `taosAdapter` provides a configuration parameter `httpCodeServerError` to set whether to return a non-200 http status code when the C interface returns an error
| **Description** | **httpCodeServerError false** | **httpCodeServerError true** |
|--------------------|---------------------------- ------|---------------------------------------|
|--------------------|----------------------------------|---------------------------------------|
| taos_errno() returns 0 | 200 | 200 |
| taos_errno() returns non-0 | 200 (except authentication error) | 500 (except authentication error and 400/502 error) |
| Parameter error | 400 (only handle HTTP request URL parameter error) | 400 (handle HTTP request URL parameter error and taosd return error) |

19
docs/en/14-reference/12-config/index.md
View File

@ -701,15 +701,6 @@ The charset that takes effect is UTF-8.
| Type | String |
| Default Value | _tag_null |
### smlDataFormat
| Attribute | Description |
| ----------- | ----------------------------------------------------------------------------------- |
| Applicable | Client only |
| Meaning | Whether schemaless columns are consistently ordered, depat, discarded since 3.0.3.0 |
| Value Range | 0: not consistent; 1: consistent. |
| Default | 0 |
### smlTsDefaultName
| Attribute | Description |
@ -719,6 +710,16 @@ The charset that takes effect is UTF-8.
| Type | String |
| Default Value | _ts |
### smlDot2Underline
| Attribute | Description |
| -------- | -------------------------------------------------------- |
| Applicable | Client only |
| Meaning | Convert the dot in the supertable name to an underscore |
| Type | Bool |
| Default Value | true |
## Compress Parameters
### compressMsgSize

8
docs/en/26-tdinternal/04-load-balance.md
View File

@ -4,7 +4,7 @@ sidebar_label: Load Balance
description: This document describes how TDengine implements load balancing.
---
The load balance in TDengine is mainly about processing data series data. TDengine employes builtin hash algorithm to distribute all the tables, sub-tables and their data of a database across all the vgroups that belongs to the database. Each table or sub-table can only be handled by a single vgroup, while each vgroup can process multiple table or sub-table.
The load balance in TDengine is mainly about processing data series data. TDengine employs builtin hash algorithm to distribute all the tables, sub-tables and their data of a database across all the vgroups that belongs to the database. Each table or sub-table can only be handled by a single vgroup, while each vgroup can process multiple table or sub-table.
The number of vgroup can be specified when creating a database, using the parameter `vgroups`.
@ -12,10 +12,10 @@ The number of vgroup can be specified when creating a database, using the parame
create database db0 vgroups 100;
```
The proper value of `vgroups` depends on available system resources. Assuming there is only one database to be created in the system, then the number of `vgroups` is determined by the available resources from all dnodes. In principle more vgroups can be created if you have more CPU and memory. Disk I/O is another important factor to consider. Once the bottleneck shows on disk I/O, more vgroups may downgrad the system performance significantly. If multiple databases are to be created in the system, then the total number of `vroups` of all the databases are dependent on the available system resources. It needs to be careful to distribute vgroups among these databases, you need to consider the number of tables, data writing frequency, size of each data row for all these databases. A recommended practice is to firstly choose a starting number for `vgroups`, for example double of the number of CPU cores, then try to adjust and optimize system configurations to find the best setting for `vgroups`, then distribute these vgroups among databases.
The proper value of `vgroups` depends on available system resources. Assuming there is only one database to be created in the system, then the number of `vgroups` is determined by the available resources from all dnodes. In principle more vgroups can be created if you have more CPU and memory. Disk I/O is another important factor to consider. Once the bottleneck shows on disk I/O, more vgroups may degrade the system performance significantly. If multiple databases are to be created in the system, then the total number of `vgroups` of all the databases are dependent on the available system resources. It needs to be careful to distribute vgroups among these databases, you need to consider the number of tables, data writing frequency, size of each data row for all these databases. A recommended practice is to firstly choose a starting number for `vgroups`, for example double of the number of CPU cores, then try to adjust and optimize system configurations to find the best setting for `vgroups`, then distribute these vgroups among databases.
Furthermode, TDengine distributes the vgroups of each database equally among all dnodes. In case of replica 3, the distribution is even more complex, TDengine tries its best to prevent any dnode from becoming a bottleneck.
Furthermore, TDengine distributes the vgroups of each database equally among all dnodes. In case of replica 3, the distribution is even more complex, TDengine tries its best to prevent any dnode from becoming a bottleneck.
TDegnine utilizes the above ways to achieve load balance in a cluster, and finally achieve higher throughput.
TDengine utilizes the above ways to achieve load balance in a cluster, and finally achieve higher throughput.
Once the load balance is achieved, after some operations like deleting tables or dropping databases, the load across all dnodes may become imbalanced, the method of rebalance will be provided in later versions. However, even without explicit rebalancing, TDengine will try its best to achieve new balance without manual interfering when a new database is created.

15
docs/examples/java/src/main/java/com/taos/example/ConsumerLoopFull.java
View File

@ -1,8 +1,9 @@
package com.taos.example;
import com.alibaba.fastjson.JSON;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.taosdata.jdbc.TSDBDriver;
import com.taosdata.jdbc.tmq.*;
import com.taosdata.jdbc.utils.JsonUtil;
import java.sql.*;
import java.time.Duration;
@ -60,7 +61,7 @@ public class ConsumerLoopFull {
// ANCHOR_END: create_consumer
}
public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException {
public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
// ANCHOR: poll_data_code_piece
List<String> topics = Collections.singletonList("topic_meters");
try {
@ -73,7 +74,7 @@ public class ConsumerLoopFull {
for (ConsumerRecord<ResultBean> record : records) {
ResultBean bean = record.value();
// Add your data processing logic here
System.out.println("data: " + JSON.toJSONString(bean));
System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
}
}
} catch (Exception ex) {
@ -91,7 +92,7 @@ public class ConsumerLoopFull {
// ANCHOR_END: poll_data_code_piece
}
public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException {
public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
// ANCHOR: consumer_seek
List<String> topics = Collections.singletonList("topic_meters");
try {
@ -99,7 +100,7 @@ public class ConsumerLoopFull {
consumer.subscribe(topics);
System.out.println("Subscribe topics successfully.");
Set<TopicPartition> assignment = consumer.assignment();
System.out.println("Now assignment: " + JSON.toJSONString(assignment));
System.out.println("Now assignment: " + JsonUtil.getObjectMapper().writeValueAsString(assignment));
ConsumerRecords<ResultBean> records = ConsumerRecords.emptyRecord();
// make sure we have got some data
@ -125,7 +126,7 @@ public class ConsumerLoopFull {
}
public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException {
public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
// ANCHOR: commit_code_piece
List<String> topics = Collections.singletonList("topic_meters");
try {
@ -135,7 +136,7 @@ public class ConsumerLoopFull {
for (ConsumerRecord<ResultBean> record : records) {
ResultBean bean = record.value();
// Add your data processing logic here
System.out.println("data: " + JSON.toJSONString(bean));
System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
}
if (!records.isEmpty()) {
// after processing the data, commit the offset manually

8
docs/examples/java/src/main/java/com/taos/example/ConsumerLoopImp.java
View File

@ -1,7 +1,7 @@
package com.taos.example;
import com.alibaba.fastjson.JSON;
import com.taosdata.jdbc.TSDBDriver;
import com.taosdata.jdbc.utils.JsonUtil;
import java.sql.Connection;
import java.sql.DriverManager;
@ -31,7 +31,11 @@ public class ConsumerLoopImp {
final AbsConsumerLoop consumerLoop = new AbsConsumerLoop() {
@Override
public void process(ResultBean result) {
System.out.println("data: " + JSON.toJSONString(result));
try{
System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(result));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};

15
docs/examples/java/src/main/java/com/taos/example/WsConsumerLoopFull.java
View File

@ -1,8 +1,9 @@
package com.taos.example;
import com.alibaba.fastjson.JSON;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.taosdata.jdbc.TSDBDriver;
import com.taosdata.jdbc.tmq.*;
import com.taosdata.jdbc.utils.JsonUtil;
import java.sql.*;
import java.time.Duration;
@ -60,7 +61,7 @@ public class WsConsumerLoopFull {
// ANCHOR_END: create_consumer
}
public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException {
public static void pollExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
// ANCHOR: poll_data_code_piece
List<String> topics = Collections.singletonList("topic_meters");
try {
@ -73,7 +74,7 @@ public class WsConsumerLoopFull {
for (ConsumerRecord<ResultBean> record : records) {
ResultBean bean = record.value();
// Add your data processing logic here
System.out.println("data: " + JSON.toJSONString(bean));
System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
}
}
} catch (Exception ex) {
@ -91,7 +92,7 @@ public class WsConsumerLoopFull {
// ANCHOR_END: poll_data_code_piece
}
public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException {
public static void seekExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
// ANCHOR: consumer_seek
List<String> topics = Collections.singletonList("topic_meters");
try {
@ -99,7 +100,7 @@ public class WsConsumerLoopFull {
consumer.subscribe(topics);
System.out.println("Subscribe topics successfully.");
Set<TopicPartition> assignment = consumer.assignment();
System.out.println("Now assignment: " + JSON.toJSONString(assignment));
System.out.println("Now assignment: " + JsonUtil.getObjectMapper().writeValueAsString(assignment));
ConsumerRecords<ResultBean> records = ConsumerRecords.emptyRecord();
// make sure we have got some data
@ -125,7 +126,7 @@ public class WsConsumerLoopFull {
}
public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException {
public static void commitExample(TaosConsumer<ResultBean> consumer) throws SQLException, JsonProcessingException {
// ANCHOR: commit_code_piece
List<String> topics = Collections.singletonList("topic_meters");
try {
@ -135,7 +136,7 @@ public class WsConsumerLoopFull {
for (ConsumerRecord<ResultBean> record : records) {
ResultBean bean = record.value();
// Add your data processing logic here
System.out.println("data: " + JSON.toJSONString(bean));
System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(bean));
}
if (!records.isEmpty()) {
// after processing the data, commit the offset manually

8
docs/examples/java/src/main/java/com/taos/example/WsConsumerLoopImp.java
View File

@ -1,7 +1,7 @@
package com.taos.example;
import com.alibaba.fastjson.JSON;
import com.taosdata.jdbc.TSDBDriver;
import com.taosdata.jdbc.utils.JsonUtil;
import java.sql.Connection;
import java.sql.DriverManager;
@ -28,7 +28,11 @@ public abstract class WsConsumerLoopImp {
final AbsConsumerLoop consumerLoop = new AbsConsumerLoop() {
@Override
public void process(ResultBean result) {
System.out.println("data: " + JSON.toJSONString(result));
try{
System.out.println("data: " + JsonUtil.getObjectMapper().writeValueAsString(result));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};

3
docs/examples/java/src/main/java/com/taos/example/highvolume/DataBaseMonitor.java
View File

@ -13,6 +13,9 @@ public class DataBaseMonitor {
public DataBaseMonitor init() throws SQLException {
if (conn == null) {
String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
if (jdbcURL == null || jdbcURL == ""){
jdbcURL = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
}
conn = DriverManager.getConnection(jdbcURL);
stmt = conn.createStatement();
}

3
docs/examples/java/src/main/java/com/taos/example/highvolume/SQLWriter.java
View File

@ -69,6 +69,9 @@ public class SQLWriter {
*/
private static Connection getConnection() throws SQLException {
String jdbcURL = System.getenv("TDENGINE_JDBC_URL");
if (jdbcURL == null || jdbcURL == ""){
jdbcURL = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
}
return DriverManager.getConnection(jdbcURL);
}

57
docs/examples/java/src/test/java/com/taos/test/TestAll.java
View File

@ -17,6 +17,37 @@ public class TestAll {
stmt.execute("drop database if exists " + dbName);
}
}
waitTransaction();
}
public void dropTopic(String topicName) throws SQLException {
String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
try (Statement stmt = conn.createStatement()) {
stmt.execute("drop topic if exists " + topicName);
}
}
waitTransaction();
}
public void waitTransaction() throws SQLException {
String jdbcUrl = "jdbc:TAOS://localhost:6030?user=root&password=taosdata";
try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
try (Statement stmt = conn.createStatement()) {
for (int i = 0; i < 10; i++) {
stmt.execute("show transactions");
try (ResultSet resultSet = stmt.getResultSet()) {
if (resultSet.next()) {
int count = resultSet.getInt(1);
if (count == 0) {
break;
}
}
}
}
}
}
}
public void insertData() throws SQLException {
@ -104,14 +135,20 @@ public class TestAll {
SubscribeDemo.main(args);
}
// @Test
// public void testSubscribeJni() throws SQLException, InterruptedException {
// dropDB("power");
// ConsumerLoopFull.main(args);
// }
// @Test
// public void testSubscribeWs() throws SQLException, InterruptedException {
// dropDB("power");
// WsConsumerLoopFull.main(args);
// }
@Test
public void testSubscribeJni() throws SQLException, InterruptedException {
dropTopic("topic_meters");
dropDB("power");
ConsumerLoopFull.main(args);
dropTopic("topic_meters");
dropDB("power");
}
@Test
public void testSubscribeWs() throws SQLException, InterruptedException {
dropTopic("topic_meters");
dropDB("power");
WsConsumerLoopFull.main(args);
dropTopic("topic_meters");
dropDB("power");
}
}

2
docs/examples/node/package.json
View File

@ -4,6 +4,6 @@
"main": "index.js",
"license": "MIT",
"dependencies": {
"@tdengine/websocket": "^3.1.0"
"@tdengine/websocket": "^3.1.1"
}
}

1
docs/examples/node/websocketexample/sql_example.js
View File

@ -3,7 +3,6 @@ const taos = require("@tdengine/websocket");
let dsn = 'ws://localhost:6041';
async function createConnect() {
try {
let conf = new taos.WSConfig(dsn);
conf.setUser('root');

1
docs/examples/node/websocketexample/tmq_seek_example.js
View File

@ -10,7 +10,6 @@ const groupId = "group1";
const clientId = "client1";
async function createConsumer() {
let groupId = "group1";
let clientId = "client1";
let configMap = new Map([

6
docs/zh/01-index.md
View File

@ -4,9 +4,9 @@ sidebar_label: 文档首页
slug: /
---
TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的<a href="https://www.taosdata.com/" data-internallinksmanager029f6b8e52c="2" title="时序数据库" target="_blank" rel="noopener">时序数据库</a>（<a href="https://www.taosdata.com/time-series-database" data-internallinksmanager029f6b8e52c="9" title="Time Series DataBase" target="_blank" rel="noopener">Time Series Database</a>, <a href="https://www.taosdata.com/tsdb" data-internallinksmanager029f6b8e52c="8" title="TSDB" target="_blank" rel="noopener">TSDB</a>）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept)
TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的<a href="https://www.taosdata.com/" data-internallinksmanager029f6b8e52c="2" title="时序数据库" target="_blank" rel="noopener">时序数据库</a>（<a href="https://www.taosdata.com/time-series-database" data-internallinksmanager029f6b8e52c="9" title="Time Series DataBase" target="_blank" rel="noopener">Time Series Database</a>, <a href="https://www.taosdata.com/tsdb" data-internallinksmanager029f6b8e52c="8" title="TSDB" target="_blank" rel="noopener">TSDB</a>）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。如果你对时序数据的基本概念、价值以及其所能带来的业务价值尚不了解，请参考[时序数据基础](./concept)。
TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[数据模型](./basic/model)一章。
TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论你在工作中是什么角色，请您仔细阅读[数据模型](./basic/model)一章。
如果你是开发工程师，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、插入数据、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，你只要复制粘贴示例代码，针对自己的应用稍作改动，就能跑起来。对 REST API、各种编程语言的连接器（Connector）想做更多详细了解的话，请看[连接器](./reference/connector)一章。
@ -16,6 +16,8 @@ TDengine 采用 SQL 作为查询语言，大大降低学习成本、降低迁移
如果你是系统管理员，关心安装、升级、容错灾备、关心数据导入、导出、配置参数，如何监测 TDengine 是否健康运行，如何提升系统运行的性能，请仔细参考[运维指南](./operation)一章。
如果你对数据库内核设计感兴趣，或是开源爱好者，建议仔细阅读[技术内幕](./tdinterna)一章。该章从分布式架构到存储引擎、查询引擎、数据订阅，再到流计算引擎都做了详细阐述。建议对照文档，查看TDengine在GitHub的源代码，对TDengine的设计和编码做深入了解，更欢迎加入开源社区，贡献代码。
最后，作为一个开源软件，欢迎大家的参与。如果发现文档有任何错误、描述不清晰的地方，请在每个页面的最下方，点击“编辑本文档”直接进行修改。
Together, we make a difference!

4
docs/zh/04-get-started/01-docker.md
View File

@ -17,7 +17,7 @@ docker pull tdengine/tdengine:latest
或者指定版本的容器镜像：
```shell
docker pull tdengine/tdengine:3.0.1.4
docker pull tdengine/tdengine:3.3.3.0
```
然后只需执行下面的命令：
@ -121,4 +121,4 @@ SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 1
SELECT _wstart, AVG(current), MAX(voltage), MIN(phase) FROM test.d1001 INTERVAL(10s);
```
在上面的查询中，使用系统提供的伪列_wstart 来给出每个窗口的开始时间。
在上面的查询中，使用系统提供的伪列 _wstart 来给出每个窗口的开始时间。

6
docs/zh/04-get-started/03-package.md
View File

@ -14,7 +14,9 @@ TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）
为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。
在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTools 包获得。TDengine 也提供 Windows x64 平台和 macOS x64/m1 平台的安装包。
在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统，用户可以根据自己的运行环境自行选择。同时我们也提供了 tar.gz 格式安装包，以及 `apt-get` 工具从线上进行安装。
此外，TDengine 也提供 macOS x64/m1 平台的 pkg 安装包。
## 运行环境要求
在linux系统中，运行环境最低要求如下:
@ -317,4 +319,4 @@ SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 1
SELECT _wstart, AVG(current), MAX(voltage), MIN(phase) FROM test.d1001 INTERVAL(10s);
```
在上面的查询中，使用系统提供的伪列_wstart 来给出每个窗口的开始时间。
在上面的查询中，使用系统提供的伪列 _wstart 来给出每个窗口的开始时间。

2
docs/zh/04-get-started/_07-use.md
View File

@ -54,4 +54,4 @@ SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 1
SELECT _wstart, AVG(current), MAX(voltage), MIN(phase) FROM test.d1001 INTERVAL(10s);
```
在上面的查询中，使用系统提供的伪列_wstart 来给出每个窗口的开始时间。
在上面的查询中，使用系统提供的伪列 _wstart 来给出每个窗口的开始时间。

311
docs/zh/05-basic/03-query.md
View File

@ -8,7 +8,7 @@ toc_max_heading_level: 4
## 基本查询
为了更好的介绍 TDengine 数据查询，使用 如下 taosBenchmark 命令，生成本章内容需要的时序数据。
为了更好的介绍 TDengine 数据查询，使用如下 taosBenchmark 命令，生成本章内容需要的时序数据。
```shell
taosBenchmark --start-timestamp=1600000000000 --tables=100 --records=10000000 --time-step=10000
@ -20,21 +20,22 @@ taosBenchmark --start-timestamp=1600000000000 --tables=100 --records=10000000 --
```sql
SELECT * FROM meters
WHERE voltage > 10
WHERE voltage > 230
ORDER BY ts DESC
LIMIT 5
LIMIT 5;
```
上面的 SQL，从超级表 `meters` 中查询出电压 `voltage` 大于 10 的记录，按时间降序排列，且仅输出前 5 行。查询结果如下：
上面的 SQL，从超级表 `meters` 中查询出电压 `voltage` 大于 230V 的记录，按时间降序排列，且仅输出前 5 行。查询结果如下：
```text
ts | current | voltage | phase | groupid | location |
==========================================================================================================
2023-11-14 22:13:10.000 | 1.1294620 | 18 | 0.3531540 | 8 | California.MountainView |
2023-11-14 22:13:10.000 | 1.0294620 | 12 | 0.3631540 | 2 | California.Campbell |
2023-11-14 22:13:10.000 | 1.0294620 | 16 | 0.3531540 | 1 | California.Campbell |
2023-11-14 22:13:10.000 | 1.1294620 | 18 | 0.3531540 | 2 | California.Campbell |
2023-11-14 22:13:10.000 | 1.1294620 | 16 | 0.3431540 | 7 | California.PaloAlto |
ts | current | voltage | phase | groupid | location |
===================================================================================================
2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 10 | California.Sunnyvale |
2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 1 | California.LosAngles |
2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 10 | California.Sunnyvale |
2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 5 | California.Cupertino |
2023-11-15 06:13:10.000 | 14.0601978 | 232 | 146.5000000 | 4 | California.SanFrancisco |
Query OK, 5 row(s) in set (0.145403s)
```
## 聚合查询
@ -48,28 +49,28 @@ TDengine 支持通过 GROUP BY 子句，对数据进行聚合查询。SQL 语句
group by 子句用于对数据进行分组，并为每个分组返回一行汇总信息。在 group by 子句中，可以使用表或视图中的任何列作为分组依据，这些列不需要出现在 select 列表中。此外，用户可以直接在超级表上执行聚合查询，无须预先创建子表。以智能电表的数据模型为例，使用 group by 子句的 SQL 如下：
```sql
SELECT groupid,avg(voltage)
SELECT groupid, avg(voltage)
FROM meters
WHERE ts >= "2022-01-01T00:00:00+08:00"
AND ts < "2023-01-01T00:00:00+08:00"
GROUP BY groupid
GROUP BY groupid;
```
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2023-01-01T00:00:00+08:00` 的数据，按照 `groupid` 进行分组，求每组的平均电压。查询结果如下：
```text
groupid | avg(voltage) |
==========================================
8 | 9.104040404040404 |
5 | 9.078333333333333 |
1 | 9.087037037037037 |
7 | 8.991414141414142 |
9 | 8.789814814814815 |
6 | 9.051010101010101 |
4 | 9.135353535353536 |
10 | 9.213131313131314 |
2 | 9.008888888888889 |
3 | 8.783888888888889 |
groupid | avg(voltage) |
======================================
8 | 243.961981544901079 |
5 | 243.961981544901079 |
1 | 243.961981544901079 |
7 | 243.961981544901079 |
9 | 243.961981544901079 |
6 | 243.961981544901079 |
4 | 243.961981544901079 |
10 | 243.961981544901079 |
2 | 243.961981544901079 |
3 | 243.961981544901079 |
Query OK, 10 row(s) in set (0.042446s)
```
@ -110,24 +111,24 @@ TDengine 按如下方式处理数据切分子句。
```sql
SELECT location, avg(voltage)
FROM meters
PARTITION BY location
PARTITION BY location;
```
上面的示例 SQL 查询超级表 `meters`，将数据按标签 `location` 进行分组，每个分组计算电压的平均值。查询结果如下：
```text
location | avg(voltage) |
=========================================================
California.SantaClara | 8.793334320000000 |
California.SanFrancisco | 9.017645882352941 |
California.SanJose | 9.156112940000000 |
California.LosAngles | 9.036753507692307 |
California.SanDiego | 8.967037053333334 |
California.Sunnyvale | 8.978572085714285 |
California.PaloAlto | 8.936665800000000 |
California.Cupertino | 8.987654066666666 |
California.MountainView | 9.046297266666667 |
California.Campbell | 9.149999028571429 |
location | avg(voltage) |
======================================================
California.SantaClara | 243.962050000000005 |
California.SanFrancisco | 243.962050000000005 |
California.SanJose | 243.962050000000005 |
California.LosAngles | 243.962050000000005 |
California.SanDiego | 243.962050000000005 |
California.Sunnyvale | 243.962050000000005 |
California.PaloAlto | 243.962050000000005 |
California.Cupertino | 243.962050000000005 |
California.MountainView | 243.962050000000005 |
California.Campbell | 243.962050000000005 |
Query OK, 10 row(s) in set (2.415961s)
```
@ -200,20 +201,20 @@ SLIMIT 2;
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据；数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，且每个时间窗口向后偏移 5 秒；最后，仅取前 2 个分片的数据作为结果。查询结果如下：
```text
tbname | _wstart | _wend | avg(voltage) |
==========================================================================================
d40 | 2021-12-31 15:59:05.000 | 2021-12-31 16:00:05.000 | 4.000000000000000 |
d40 | 2021-12-31 16:00:05.000 | 2021-12-31 16:01:05.000 | 5.000000000000000 |
d40 | 2021-12-31 16:01:05.000 | 2021-12-31 16:02:05.000 | 8.000000000000000 |
d40 | 2021-12-31 16:02:05.000 | 2021-12-31 16:03:05.000 | 7.666666666666667 |
d40 | 2021-12-31 16:03:05.000 | 2021-12-31 16:04:05.000 | 9.666666666666666 |
d40 | 2021-12-31 16:04:05.000 | 2021-12-31 16:05:05.000 | 15.199999999999999 |
d41 | 2021-12-31 15:59:05.000 | 2021-12-31 16:00:05.000 | 4.000000000000000 |
d41 | 2021-12-31 16:00:05.000 | 2021-12-31 16:01:05.000 | 7.000000000000000 |
d41 | 2021-12-31 16:01:05.000 | 2021-12-31 16:02:05.000 | 9.000000000000000 |
d41 | 2021-12-31 16:02:05.000 | 2021-12-31 16:03:05.000 | 10.666666666666666 |
d41 | 2021-12-31 16:03:05.000 | 2021-12-31 16:04:05.000 | 8.333333333333334 |
d41 | 2021-12-31 16:04:05.000 | 2021-12-31 16:05:05.000 | 9.600000000000000 |
tbname | _wstart | _wend | avg(voltage) |
======================================================================================
d2 | 2021-12-31 23:59:05.000 | 2022-01-01 00:00:05.000 | 253.000000000000000 |
d2 | 2022-01-01 00:00:05.000 | 2022-01-01 00:01:05.000 | 244.166666666666657 |
d2 | 2022-01-01 00:01:05.000 | 2022-01-01 00:02:05.000 | 241.833333333333343 |
d2 | 2022-01-01 00:02:05.000 | 2022-01-01 00:03:05.000 | 243.166666666666657 |
d2 | 2022-01-01 00:03:05.000 | 2022-01-01 00:04:05.000 | 240.833333333333343 |
d2 | 2022-01-01 00:04:05.000 | 2022-01-01 00:05:05.000 | 244.800000000000011 |
d26 | 2021-12-31 23:59:05.000 | 2022-01-01 00:00:05.000 | 253.000000000000000 |
d26 | 2022-01-01 00:00:05.000 | 2022-01-01 00:01:05.000 | 244.166666666666657 |
d26 | 2022-01-01 00:01:05.000 | 2022-01-01 00:02:05.000 | 241.833333333333343 |
d26 | 2022-01-01 00:02:05.000 | 2022-01-01 00:03:05.000 | 243.166666666666657 |
d26 | 2022-01-01 00:03:05.000 | 2022-01-01 00:04:05.000 | 240.833333333333343 |
d26 | 2022-01-01 00:04:05.000 | 2022-01-01 00:05:05.000 | 244.800000000000011 |
Query OK, 12 row(s) in set (0.021265s)
```
@ -255,19 +256,19 @@ SLIMIT 1;
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据，数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，且时间窗口按照 30 秒进行滑动；最后，仅取前 1 个分片的数据作为结果。查询结果如下：
```text
tbname | _wstart | avg(voltage) |
================================================================
d40 | 2021-12-31 15:59:30.000 | 4.000000000000000 |
d40 | 2021-12-31 16:00:00.000 | 5.666666666666667 |
d40 | 2021-12-31 16:00:30.000 | 4.333333333333333 |
d40 | 2021-12-31 16:01:00.000 | 5.000000000000000 |
d40 | 2021-12-31 16:01:30.000 | 9.333333333333334 |
d40 | 2021-12-31 16:02:00.000 | 9.666666666666666 |
d40 | 2021-12-31 16:02:30.000 | 10.000000000000000 |
d40 | 2021-12-31 16:03:00.000 | 10.333333333333334 |
d40 | 2021-12-31 16:03:30.000 | 10.333333333333334 |
d40 | 2021-12-31 16:04:00.000 | 13.000000000000000 |
d40 | 2021-12-31 16:04:30.000 | 15.333333333333334 |
tbname | _wstart | avg(voltage) |
=============================================================
d2 | 2021-12-31 23:59:30.000 | 248.333333333333343 |
d2 | 2022-01-01 00:00:00.000 | 246.000000000000000 |
d2 | 2022-01-01 00:00:30.000 | 244.666666666666657 |
d2 | 2022-01-01 00:01:00.000 | 240.833333333333343 |
d2 | 2022-01-01 00:01:30.000 | 239.500000000000000 |
d2 | 2022-01-01 00:02:00.000 | 243.833333333333343 |
d2 | 2022-01-01 00:02:30.000 | 243.833333333333343 |
d2 | 2022-01-01 00:03:00.000 | 241.333333333333343 |
d2 | 2022-01-01 00:03:30.000 | 241.666666666666657 |
d2 | 2022-01-01 00:04:00.000 | 244.166666666666657 |
d2 | 2022-01-01 00:04:30.000 | 244.666666666666657 |
Query OK, 11 row(s) in set (0.013153s)
```
@ -290,13 +291,13 @@ SLIMIT 1;
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据，数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，且时间窗口按照 1 分钟进行切分；最后，仅取前 1 个分片的数据作为结果。查询结果如下：
```text
tbname | _wstart | _wend | avg(voltage) |
=================================================================================================================
d28 | 2021-12-31 16:00:00.000 | 2021-12-31 16:01:00.000 | 7.333333333333333 |
d28 | 2021-12-31 16:01:00.000 | 2021-12-31 16:02:00.000 | 8.000000000000000 |
d28 | 2021-12-31 16:02:00.000 | 2021-12-31 16:03:00.000 | 11.000000000000000 |
d28 | 2021-12-31 16:03:00.000 | 2021-12-31 16:04:00.000 | 6.666666666666667 |
d28 | 2021-12-31 16:04:00.000 | 2021-12-31 16:05:00.000 | 10.000000000000000 |
tbname | _wstart | _wend | avg(voltage) |
======================================================================================
d2 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:00.000 | 246.000000000000000 |
d2 | 2022-01-01 00:01:00.000 | 2022-01-01 00:02:00.000 | 240.833333333333343 |
d2 | 2022-01-01 00:02:00.000 | 2022-01-01 00:03:00.000 | 243.833333333333343 |
d2 | 2022-01-01 00:03:00.000 | 2022-01-01 00:04:00.000 | 241.333333333333343 |
d2 | 2022-01-01 00:04:00.000 | 2022-01-01 00:05:00.000 | 244.166666666666657 |
Query OK, 5 row(s) in set (0.016812s)
```
@ -342,53 +343,65 @@ SLIMIT 2;
上面的 SQL，查询超级表 `meters` 中，时间戳大于等于 `2022-01-01T00:00:00+08:00`，且时间戳小于 `2022-01-01T00:05:00+08:00` 的数据；数据首先按照子表名 `tbname` 进行数据切分，再按照每 1 分钟的时间窗口进行切分，如果窗口内的数据出现缺失，则使用使用前一个非 NULL 值填充数据；最后，仅取前 2 个分片的数据作为结果。查询结果如下：
```text
tbname | _wstart | _wend | avg(voltage) |
=================================================================================================================
d40 | 2021-12-31 16:00:00.000 | 2021-12-31 16:01:00.000 | 5.666666666666667 |
d40 | 2021-12-31 16:01:00.000 | 2021-12-31 16:02:00.000 | 5.000000000000000 |
d40 | 2021-12-31 16:02:00.000 | 2021-12-31 16:03:00.000 | 9.666666666666666 |
d40 | 2021-12-31 16:03:00.000 | 2021-12-31 16:04:00.000 | 10.333333333333334 |
d40 | 2021-12-31 16:04:00.000 | 2021-12-31 16:05:00.000 | 13.000000000000000 |
d41 | 2021-12-31 16:00:00.000 | 2021-12-31 16:01:00.000 | 5.666666666666667 |
d41 | 2021-12-31 16:01:00.000 | 2021-12-31 16:02:00.000 | 9.333333333333334 |
d41 | 2021-12-31 16:02:00.000 | 2021-12-31 16:03:00.000 | 11.000000000000000 |
d41 | 2021-12-31 16:03:00.000 | 2021-12-31 16:04:00.000 | 7.666666666666667 |
d41 | 2021-12-31 16:04:00.000 | 2021-12-31 16:05:00.000 | 10.000000000000000 |
tbname | _wstart | _wend | avg(voltage) |
=======================================================================================
d2 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:00.000 | 246.000000000000000 |
d2 | 2022-01-01 00:01:00.000 | 2022-01-01 00:02:00.000 | 240.833333333333343 |
d2 | 2022-01-01 00:02:00.000 | 2022-01-01 00:03:00.000 | 243.833333333333343 |
d2 | 2022-01-01 00:03:00.000 | 2022-01-01 00:04:00.000 | 241.333333333333343 |
d2 | 2022-01-01 00:04:00.000 | 2022-01-01 00:05:00.000 | 244.166666666666657 |
d26 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:00.000 | 246.000000000000000 |
d26 | 2022-01-01 00:01:00.000 | 2022-01-01 00:02:00.000 | 240.833333333333343 |
d26 | 2022-01-01 00:02:00.000 | 2022-01-01 00:03:00.000 | 243.833333333333343 |
d26 | 2022-01-01 00:03:00.000 | 2022-01-01 00:04:00.000 | 241.333333333333343 |
d26 | 2022-01-01 00:04:00.000 | 2022-01-01 00:05:00.000 | 244.166666666666657 |
Query OK, 10 row(s) in set (0.022866s)
```
### 状态窗口
使用整数（布尔值）或字符串来标识产生记录时候设备的状态量。产生的记录如果具有相同的状态量数值则归属于同一个状态窗口，数值改变后该窗口关闭。TDengine 还支持将 CASE 表达式用在状态量，可以表达某个状态的开始是由满足某个条件而触发，这个状态的结束是由另外一个条件满足而触发的语义。以智能电表为例，电压正常范围是 205V 到 235V，那么可以通过监控电压来判断电路是否正常。
使用整数（布尔值）或字符串来标识产生记录时候设备的状态量。产生的记录如果具有相同的状态量数值则归属于同一个状态窗口，数值改变后该窗口关闭。TDengine 还支持将 CASE 表达式用在状态量，可以表达某个状态的开始是由满足某个条件而触发，这个状态的结束是由另外一个条件满足而触发的语义。以智能电表为例，电压正常范围是 225V 到 235V，那么可以通过监控电压来判断电路是否正常。
```sql
SELECT tbname, _wstart, _wend,_wduration, CASE WHEN voltage >= 205 and voltage <= 235 THEN 1 ELSE 0 END status
SELECT tbname, _wstart, _wend,_wduration, CASE WHEN voltage >= 225 and voltage <= 235 THEN 1 ELSE 0 END status
FROM meters
WHERE ts >= "2022-01-01T00:00:00+08:00"
AND ts < "2022-01-01T00:05:00+08:00"
PARTITION BY tbname
STATE_WINDOW(
CASE WHEN voltage >= 205 and voltage <= 235 THEN 1 ELSE 0 END
CASE WHEN voltage >= 225 and voltage <= 235 THEN 1 ELSE 0 END
)
SLIMIT 10;
SLIMIT 2;
```
以上 SQL，查询超级表 meters 中，时间戳大于等于 2022-01-01T00:00:00+08:00，且时间戳小于 2022-01-01T00:05:00+08:00的数据；数据首先按照子表名 tbname 进行数据切分；根据电压是否在正常范围内进行状态窗口的划分；最后，取前 10 个分片的数据作为结果。查询结果如下：
以上 SQL，查询超级表 meters 中，时间戳大于等于 2022-01-01T00:00:00+08:00，且时间戳小于 2022-01-01T00:05:00+08:00的数据；数据首先按照子表名 tbname 进行数据切分；根据电压是否在正常范围内进行状态窗口的划分；最后，取前 2 个分片的数据作为结果。查询结果如下：（由于数据是随机生成，结果集包含的数据条数会有不同）
```text
tbname | _wstart | _wend | _wduration | status |
=====================================================================================================================================
d76 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
d47 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
d37 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
d87 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
d64 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
d35 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
d83 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
d51 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
d63 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
d0 | 2021-12-31 16:00:00.000 | 2021-12-31 16:04:50.000 | 290000 | 0 |
Query OK, 10 row(s) in set (0.040495s)
tbname | _wstart | _wend | _wduration | status |
===============================================================================================
d2 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:20.000 | 80000 | 0 |
d2 | 2022-01-01 00:01:30.000 | 2022-01-01 00:01:30.000 | 0 | 1 |
d2 | 2022-01-01 00:01:40.000 | 2022-01-01 00:01:40.000 | 0 | 0 |
d2 | 2022-01-01 00:01:50.000 | 2022-01-01 00:01:50.000 | 0 | 1 |
d2 | 2022-01-01 00:02:00.000 | 2022-01-01 00:02:20.000 | 20000 | 0 |
d2 | 2022-01-01 00:02:30.000 | 2022-01-01 00:02:30.000 | 0 | 1 |
d2 | 2022-01-01 00:02:40.000 | 2022-01-01 00:03:00.000 | 20000 | 0 |
d2 | 2022-01-01 00:03:10.000 | 2022-01-01 00:03:10.000 | 0 | 1 |
d2 | 2022-01-01 00:03:20.000 | 2022-01-01 00:03:40.000 | 20000 | 0 |
d2 | 2022-01-01 00:03:50.000 | 2022-01-01 00:03:50.000 | 0 | 1 |
d2 | 2022-01-01 00:04:00.000 | 2022-01-01 00:04:50.000 | 50000 | 0 |
d26 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:20.000 | 80000 | 0 |
d26 | 2022-01-01 00:01:30.000 | 2022-01-01 00:01:30.000 | 0 | 1 |
d26 | 2022-01-01 00:01:40.000 | 2022-01-01 00:01:40.000 | 0 | 0 |
d26 | 2022-01-01 00:01:50.000 | 2022-01-01 00:01:50.000 | 0 | 1 |
d26 | 2022-01-01 00:02:00.000 | 2022-01-01 00:02:20.000 | 20000 | 0 |
d26 | 2022-01-01 00:02:30.000 | 2022-01-01 00:02:30.000 | 0 | 1 |
d26 | 2022-01-01 00:02:40.000 | 2022-01-01 00:03:00.000 | 20000 | 0 |
d26 | 2022-01-01 00:03:10.000 | 2022-01-01 00:03:10.000 | 0 | 1 |
d26 | 2022-01-01 00:03:20.000 | 2022-01-01 00:03:40.000 | 20000 | 0 |
d26 | 2022-01-01 00:03:50.000 | 2022-01-01 00:03:50.000 | 0 | 1 |
d26 | 2022-01-01 00:04:00.000 | 2022-01-01 00:04:50.000 | 50000 | 0 |
Query OK, 22 row(s) in set (0.153403s)
```
### 会话窗口
@ -417,18 +430,18 @@ SLIMIT 10;
上面的 SQL，查询超级表 meters 中，时间戳大于等于 2022-01-01T00:00:00+08:00，且时间戳小于 2022-01-01T00:10:00+08:00的数据；数据先按照子表名 tbname 进行数据切分，再根据 10 分钟的会话窗口进行切分；最后，取前 10 个分片的数据作为结果，返回子表名、窗口开始时间、窗口结束时间、窗口宽度、窗口内数据条数。查询结果如下：
```text
tbname | _wstart | _wend | _wduration | count(*) |
=====================================================================================================================================
d76 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
d47 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
d37 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
d87 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
d64 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
d35 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
d83 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
d51 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
d63 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
d0 | 2021-12-31 16:00:00.000 | 2021-12-31 16:09:50.000 | 590000 | 60 |
tbname | _wstart | _wend | _wduration | count(*) |
===============================================================================================
d2 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d26 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d52 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d64 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d76 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d28 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d4 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d88 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d77 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
d54 | 2022-01-01 00:00:00.000 | 2022-01-01 00:09:50.000 | 590000 | 60 |
Query OK, 10 row(s) in set (0.043489s)
```
@ -458,26 +471,26 @@ FROM meters
WHERE ts >= "2022-01-01T00:00:00+08:00"
AND ts < "2022-01-01T00:10:00+08:00"
PARTITION BY tbname
EVENT_WINDOW START WITH voltage >= 10 END WITH voltage < 20
LIMIT 10;
EVENT_WINDOW START WITH voltage >= 225 END WITH voltage < 235
LIMIT 5;
```
上面的 SQL，查询超级表meters中，时间戳大于等于2022-01-01T00:00:00+08:00，且时间戳小于2022-01-01T00:10:00+08:00的数据；数据先按照子表名tbname进行数据切分，再根据事件窗口条件：电压大于等于 10V，且小于 20V 进行切分；最后，取前 10 行的数据作为结果，返回子表名、窗口开始时间、窗口结束时间、窗口宽度、窗口内数据条数。查询结果如下：
上面的 SQL，查询超级表meters中，时间戳大于等于2022-01-01T00:00:00+08:00，且时间戳小于2022-01-01T00:10:00+08:00的数据；数据先按照子表名tbname进行数据切分，再根据事件窗口条件：电压大于等于 225V，且小于 235V 进行切分；最后，取每个分片的前 5 行的数据作为结果，返回子表名、窗口开始时间、窗口结束时间、窗口宽度、窗口内数据条数。查询结果如下：
```text
tbname | _wstart | _wend | _wduration | count(*) |
=====================================================================================================================================
d0 | 2021-12-31 16:00:00.000 | 2021-12-31 16:00:00.000 | 0 | 1 |
d0 | 2021-12-31 16:00:30.000 | 2021-12-31 16:00:30.000 | 0 | 1 |
d0 | 2021-12-31 16:00:40.000 | 2021-12-31 16:00:40.000 | 0 | 1 |
d0 | 2021-12-31 16:01:20.000 | 2021-12-31 16:01:20.000 | 0 | 1 |
d0 | 2021-12-31 16:02:20.000 | 2021-12-31 16:02:20.000 | 0 | 1 |
d0 | 2021-12-31 16:02:30.000 | 2021-12-31 16:02:30.000 | 0 | 1 |
d0 | 2021-12-31 16:03:10.000 | 2021-12-31 16:03:10.000 | 0 | 1 |
d0 | 2021-12-31 16:03:30.000 | 2021-12-31 16:03:30.000 | 0 | 1 |
d0 | 2021-12-31 16:03:40.000 | 2021-12-31 16:03:40.000 | 0 | 1 |
d0 | 2021-12-31 16:03:50.000 | 2021-12-31 16:03:50.000 | 0 | 1 |
Query OK, 10 row(s) in set (0.034127s)
tbname | _wstart | _wend | _wduration | count(*) |
==============================================================================================
d0 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:30.000 | 90000 | 10 |
d0 | 2022-01-01 00:01:40.000 | 2022-01-01 00:02:30.000 | 50000 | 6 |
d0 | 2022-01-01 00:02:40.000 | 2022-01-01 00:03:10.000 | 30000 | 4 |
d0 | 2022-01-01 00:03:20.000 | 2022-01-01 00:07:10.000 | 230000 | 24 |
d0 | 2022-01-01 00:07:20.000 | 2022-01-01 00:07:50.000 | 30000 | 4 |
d1 | 2022-01-01 00:00:00.000 | 2022-01-01 00:01:30.000 | 90000 | 10 |
d1 | 2022-01-01 00:01:40.000 | 2022-01-01 00:02:30.000 | 50000 | 6 |
d1 | 2022-01-01 00:02:40.000 | 2022-01-01 00:03:10.000 | 30000 | 4 |
d1 | 2022-01-01 00:03:20.000 | 2022-01-01 00:07:10.000 | 230000 | 24 |
……
Query OK, 500 row(s) in set (0.328557s)
```
### 计数窗口
@ -492,17 +505,25 @@ sliding_val 是一个常量，表示窗口滑动的数量，类似于 interval
select _wstart, _wend, count(*)
from meters
where ts >= "2022-01-01T00:00:00+08:00" and ts < "2022-01-01T00:30:00+08:00"
count_window(10);
count_window(1000);
```
上面的 SQL 查询超级表 meters 中时间戳大于等于 2022-01-01T00:00:00+08:00 且时间戳小于 2022-01-01T00:10:00+08:00 的数据。以每 10 条数据为一组，返回每组的开始时间、结束时间和分组条数。查询结果如下。
上面的 SQL 查询超级表 meters 中时间戳大于等于 2022-01-01T00:00:00+08:00 且时间戳小于 2022-01-01T00:10:00+08:00 的数据。以每 1000 条数据为一组，返回每组的开始时间、结束时间和分组条数。查询结果如下：
```text
_wstart | _wend |count(*)|
===========================================================
2021-12-31 16:00:00.000 | 2021-12-31 16:10:00.000 | 10 |
2021-12-31 16:10:00.000 | 2021-12-31 16:20:00.000 | 10 |
2021-12-31 16:20:00.000 | 2021-12-31 16:30:00.000 | 10 |
_wstart | _wend | count(*) |
=====================================================================
2022-01-01 00:00:00.000 | 2022-01-01 00:01:30.000 | 1000 |
2022-01-01 00:01:40.000 | 2022-01-01 00:03:10.000 | 1000 |
2022-01-01 00:03:20.000 | 2022-01-01 00:04:50.000 | 1000 |
2022-01-01 00:05:00.000 | 2022-01-01 00:06:30.000 | 1000 |
2022-01-01 00:06:40.000 | 2022-01-01 00:08:10.000 | 1000 |
2022-01-01 00:08:20.000 | 2022-01-01 00:09:50.000 | 1000 |
2022-01-01 00:10:00.000 | 2022-01-01 00:11:30.000 | 1000 |
2022-01-01 00:11:40.000 | 2022-01-01 00:13:10.000 | 1000 |
2022-01-01 00:13:20.000 | 2022-01-01 00:14:50.000 | 1000 |
2022-01-01 00:15:00.000 | 2022-01-01 00:16:30.000 | 1000 |
Query OK, 10 row(s) in set (0.062794s)
```
## 时序数据特有函数
@ -563,14 +584,14 @@ UNION ALL
上面的 SQL，分别查询：子表 d1 的 1 条数据，子表 d11 的 2 条数据，子表 d21 的 3 条数据，并将结果合并。返回的结果如下：
```text
tbname | ts | current | voltage | phase |
=================================================================================================
d11 | 2020-09-13 12:26:40.000 | 1.0260611 | 6 | 0.3620200 |
d11 | 2020-09-13 12:26:50.000 | 2.9544230 | 8 | 1.0048079 |
d21 | 2020-09-13 12:26:40.000 | 1.0260611 | 2 | 0.3520200 |
d21 | 2020-09-13 12:26:50.000 | 2.9544230 | 2 | 0.9948080 |
d21 | 2020-09-13 12:27:00.000 | -0.0000430 | 12 | 0.0099860 |
d1 | 2020-09-13 12:26:40.000 | 1.0260611 | 10 | 0.3520200 |
tbname | ts | current | voltage | phase |
====================================================================================
d11 | 2020-09-13 20:26:40.000 | 11.5680809 | 247 | 146.5000000 |
d11 | 2020-09-13 20:26:50.000 | 14.2392311 | 234 | 148.0000000 |
d1 | 2020-09-13 20:26:40.000 | 11.5680809 | 247 | 146.5000000 |
d21 | 2020-09-13 20:26:40.000 | 11.5680809 | 247 | 146.5000000 |
d21 | 2020-09-13 20:26:50.000 | 14.2392311 | 234 | 148.0000000 |
d21 | 2020-09-13 20:27:00.000 | 10.0999422 | 251 | 146.0000000 |
Query OK, 6 row(s) in set (0.006438s)
```

4
docs/zh/06-advanced/02-cache.md
View File

@ -57,7 +57,7 @@ TDengine 利用这些日志文件实现故障前的状态恢复。在写入 WAL
- wal_fsync_period：当 wal_level 设置为 2 时，这个参数控制执行 fsync 的频率。设置为 0 表示每次写入后立即执行 fsync，这可以确保数据的安全性，但可能会牺牲一些性能。当设置为大于 0 的数值时，表示 fsync 周期，默认为 3000，范围是[1， 180000]，单位毫秒。
```sql
CREATE DATABASE POWER WAL_LEVEL 1 WAL_FSYNC_PERIOD 3000;
CREATE DATABASE POWER WAL_LEVEL 2 WAL_FSYNC_PERIOD 3000;
```
在创建数据库时可以选择不同的参数类型，来选择性能优先或者可靠性优先。
@ -119,4 +119,4 @@ taos> select last_row(ts,current) from meters;
Query OK, 1 row(s) in set (0.046682s)
```
可以看到查询的时延从 353/344ms 缩短到了 44ms，提升约 8 倍。
可以看到查询的时延从 353/344ms 缩短到了 44ms，提升约 8 倍。

30
docs/zh/06-advanced/05-data-in/index.md
View File

@ -14,18 +14,26 @@ TDengine Enterprise 配备了一个强大的可视化数据管理工具—taosEx
## 支持的数据源
目前 TDengine 支持的数据源如下：
目前 TDengine 支持的数据源如下表：
1. Aveva PI System：一个工业数据管理和分析平台，前身为 OSIsoft PI System，它能够实时采集、整合、分析和可视化工业数据，助力企业实现智能化决策和精细化管理
2. Aveva Historian：一个工业大数据分析软件，前身为 Wonderware Historian，专为工业环境设计，用于存储、管理和分析来自各种工业设备、传感器的实时和历史数据。
3. OPC DA/UA：OPC 是 Open Platform Communications 的缩写，是一种开放式、标准化的通信协议，用于不同厂商的自动化设备之间进行数据交换。它最初由微软公司开发，旨在解决工业控制领域中不同设备之间互操作性差的问题。OPC 协议最初于 1996 年发布，当时称为 OPC DA （Data Access），主要用于实时数据采集和控制；2006 年，OPC 基金会发布了 OPC UA （Unified Architecture） 标准，它是一种基于服务的面向对象的协议，具有更高的灵活性和可扩展性，已成为 OPC 协议的主流版本。
4. MQTT：Message Queuing Telemetry Transport 的缩写，一种基于发布/订阅模式的轻量级通讯协议，专为低开销、低带宽占用的即时通讯设计，广泛适用于物联网、小型设备、移动应用等领域。
5. Kafka：由 Apache 软件基金会开发的一个开源流处理平台，主要用于处理实时数据，并提供一个统一、高通量、低延迟的消息系统。它具备高速度、可伸缩性、持久性和分布式设计等特点，使得它能够在每秒处理数十万次的读写操作，支持上千个客户端，同时保持数据的可靠性和可用性。
6. OpenTSDB：基于 HBase 的分布式、可伸缩的时序数据库。它主要用于存储、索引和提供从大规模集群（包括网络设备、操作系统、应用程序等）中收集的指标数据，使这些数据更易于访问和图形化展示。
7. CSV：Comma Separated Values 的缩写，是一种以逗号分隔的纯文本文件格式，通常用于电子表格或数据库软件。
8. TDengine 2：泛指运行 TDengine 2.x 版本的 TDengine 实例。
9. TDengine 3：泛指运行 TDengine 3.x 版本的 TDengine 实例。
10. MySQL， PostgreSQL， Oracle 等关系型数据库。
| 数据源 | 支持版本 | 描述 |
| --- | --- | --- |
| Aveva PI System | PI AF Server Version 2.10.9.593 或以上 | 工业数据管理和分析平台，前身为 OSIsoft PI System，它能够实时采集、整合、分析和可视化工业数据，助力企业实现智能化决策和精细化管理 |
| Aveva Historian | AVEVA Historian 2020 RS SP1 | 工业大数据分析软件，前身为 Wonderware Historian，专为工业环境设计，用于存储、管理和分析来自各种工业设备、传感器的实时和历史数据 |
| OPC DA | Matrikon OPC version: 1.7.2.7433 | OPC 是 Open Platform Communications 的缩写，是一种开放式、标准化的通信协议，用于不同厂商的自动化设备之间进行数据交换。它最初由微软公司开发，旨在解决工业控制领域中不同设备之间互操作性差的问题；OPC 协议最初于 1996 年发布，当时称为 OPC DA （Data Access），主要用于实时数据采集和控制。 |
| OPC UA | KeepWare KEPServerEx 6.5 | 2006 年，OPC 基金会发布了 OPC UA （Unified Architecture） 标准，它是一种基于服务的面向对象的协议，具有更高的灵活性和可扩展性，已成为 OPC 协议的主流版本 |
| MQTT | emqx: 3.0.0 到 5.7.1<br/> hivemq: 4.0.0 到 4.31.0<br/> mosquitto: 1.4.4 到 2.0.18 | Message Queuing Telemetry Transport 的缩写，一种基于发布/订阅模式的轻量级通讯协议，专为低开销、低带宽占用的即时通讯设计，广泛适用于物联网、小型设备、移动应用等领域。 |
| Kafka | 2.11 ~ 3.8.0 | 由 Apache 软件基金会开发的一个开源流处理平台，主要用于处理实时数据，并提供一个统一、高通量、低延迟的消息系统。它具备高速度、可伸缩性、持久性和分布式设计等特点，使得它能够在每秒处理数十万次的读写操作，支持上千个客户端，同时保持数据的可靠性和可用性。 |
| InfluxDB | 1.7、1.8、2.0-2.7 | InfluxDB 是一种流行的开源时间序列数据库，它针对处理大量时间序列数据进行了优化。|
| OpenTSDB | 2.4.1 | 基于 HBase 的分布式、可伸缩的时序数据库。它主要用于存储、索引和提供从大规模集群（包括网络设备、操作系统、应用程序等）中收集的指标数据，使这些数据更易于访问和图形化展示。 |
| MySQL | 5.6,5.7,8.0+ | MySQL是最流行的关系型数据库管理系统之一，由于其体积小、速度快、总体拥有成本低，尤其是开放源码这一特点，一般中小型和大型网站的开发都选择 MySQL 作为网站数据库。 |
| Oracle | 11G/12c/19c | Oracle 数据库系统是世界上流行的关系数据库管理系统，系统可移植性好、使用方便、功能强，适用于各类大、中、小微机环境。它是一种高效率的、可靠性好的、适应高吞吐量的数据库方案。 |
| PostgreSQL | v15.0+ | PostgreSQL 是一个功能非常强大的、源代码开放的客户/服务器关系型数据库管理系统， 有很多在大型商业RDBMS中所具有的特性，包括事务、子选择、触发器、视图、外键引用完整性和复杂锁定功能。|
| SQL Server | 2012/2022 | Microsoft SQL Server 是一种关系型数据库管理系统，由 Microsoft 公司开发，具有使用方便可伸缩性好与相关软件集成程度高等优点。 |
| MongoDB | 3.6+ | MongoDB 是一个介于关系型数据库与非关系型数据库之间的产品，被广泛应用于内容管理系统、移动应用与物联网等众多领域。 |
| CSV | - | Comma Separated Values 的缩写，是一种以逗号分隔的纯文本文件格式，通常用于电子表格或数据库软件。 |
| TDengine 2.x | 2.4 或 2.6+ | TDengine 旧版本，已不再维护，推荐升级到 3.0 最新版本。 |
| TDengine 3.x | 源端版本+ | 使用 TMQ 进行 TDengine 指定从数据库或超级表的订阅。 |
## 数据提取、过滤和转换

48
docs/zh/06-advanced/06-data-analysis/01-arima.md
View File

@ -3,8 +3,52 @@ title: "ARIMA"
sidebar_label: "ARIMA"
---
本节讲述如何 ARIMA 算法的使用方法。
本节讲述 ARIMA 算法模型的使用方法。
## 功能概述
……
ARIMA 即自回归移动平均模型(Autoregressive Integrated Moving Average, ARIMA)，也记作 ARIMA(p,d,q)，是统计模型中最常见的一种用来进行时间序列预测的模型。
ARIMA模型是一种自回归模型，只需要自变量即可预测后续的值。ARIMA模型要求时序数据是**平稳**，或经过差分处理后平稳，如果是不平稳的数据，**无法**获得正确的结果。
>平稳的时间序列：其性质不随观测时间的变化而变化。具有趋势或季节性的时间序列不是平稳时间序列——趋势和季节性使得时间序列在不同时段呈现不同性质。
以下参数可以动态输入控制预测过程中生成 合适的 ARIMA 的模型。
- p= 自回归模型阶数
- d= 差分阶数
- q= 移动平均模型阶数
### 参数
分析平台中使用自动化的 ARIMA 模型进行计算，因此每次计算的时候会根据输入的数据自动拟合最合适的模型，然后根据该模型进行预测输出结果。
|参数名称|说明|必填项|
|---|---|---|
|period|输入时间序列数据每个周期包含的数据点个数。如果不设置该参数或则该参数设置为 0， 将使用非季节性/周期性的 ARIMA 模型预测。|选填|
|start_p| 自回归模型阶数的 起始值，0 开始的整数，不推荐大于 10 |选填|
|max_p| 自回归模型阶数的 结束值，0 开始的整数，不推荐大于 10 |选填|
|start_q| 移动平均模型阶数的起始值， 0 开始的整数，不推荐大于 10 |选填|
|max_q| 移动平均模型阶数的结束值， 0 开始的整数，不推荐大于 10 |选填|
|d| 差分阶数|选填|
`start_p`、`max_p` `start_q` `max_q` 四个参数约束了模型在多大的范围内去搜寻合适的最优解。相同输入数据的条件下，参数范围越大，消耗的资源越多，系统响应的时间越长。
### 示例及结果
针对 i32 列进行数据预测，输入列 i32 每 10 个点是一个周期，start_p 起始是 1， 最大拟合是 5，start_q是1，最大值是5，预测结果中返回 95% 置信区间范围边界。
```
FORECAST(i32, "algo=arima,alpha=95,period=10, start_p=1, max_p=5, start_q=1, max_q=5")
```
```json5
{
"rows": fc_rows, // 预测结果的行数
"period": period, // 返回结果的周期性，同输入
"alpha": alpha, // 返回结果的置信区间，同输入
"algo": "arima", // 返回结果使用的算法
"mse":mse, // 拟合输入时序数据时候生成模型的最小均方误差(MSE)
"res": res // 列模式的结果
}
```
### 参考文献
- https://en.wikipedia.org/wiki/Autoregressive_moving-average_model
- https://baike.baidu.com/item/%E8%87%AA%E5%9B%9E%E5%BD%92%E6%BB%91%E5%8A%A8%E5%B9%B3%E5%9D%87%E6%A8%A1%E5%9E%8B/5023931?fromtitle=ARMA%E6%A8%A1%E5%9E%8B&fromid=8048415

43
docs/zh/06-advanced/06-data-analysis/02-holtwinters.md Normal file
View File

@ -0,0 +1,43 @@
---
title: "HoltWinters"
sidebar_label: "HoltWinters"
---
本节讲述 HoltWinters 算法模型的使用方法。
## 功能概述
HoltWinters模型又称为多次指数平滑模型（EMA）。对含有线性趋势和周期波动的非平稳序列适用，利用指数平滑法让模型参数不断适应非平稳序列的变化，并对未来趋势进行**短期**预测。
HoltWinters有两种不同的季节性组成部分，当季节变化在该时间序列中大致保持不变时，通常选择**加法模型**；而当季节变化与时间序列的水平成比例变化时，通常选择**乘法模型**。
该模型对于返回数据也不提供计算的置信区间范围结果。在 95% 置信区间的上下界结果与预测结果相同。
### 参数
分析平台中使用自动化的 ARIMA 模型进行计算，因此每次计算的时候会根据输入的数据自动拟合最合适的模型，然后根据该模型进行预测输出结果。
|参数名称|说明|必填项|
|---|---|---|
|period| 输入时间序列数据每个周期包含的数据点个数。如果不设置该参数或则该参数设置为 0， 将使用一次（简单）指数平滑方式进行数据拟合，并据此进行未来数据的预测|选填|
|trend| 趋势模型使用加法模型还是乘法模型|选填|
|seasonal| 季节性采用加法模型还是乘法模型|选填|
参数 `trend` 和 `seasonal`的均可以选择 `add` （加法模型）或 `mul`（乘法模型）。
### 示例及结果
针对 i32 列进行数据预测，输入列 i32 每 10 个点是一个周期，趋势采用乘法模型，季节采用乘法模型
```
FORECAST(i32, "algo=holtwinters,period=10,trend=mul,seasonal=mul")
```
```json5
{
"rows": rows, // 结果的行数
"period": period, // 返回结果的周期性， 该结果与输入的周期性相同，如果没有周期性，该值为 0
"algo": 'holtwinters' // 返回结果使用的计算模型
"mse":mse, // 最小均方误差（minmum square error）
"res": res // 具体的结果，按照列形式返回的结果。一般意义上包含了 两列[timestamp][fc_results]。
}
```
### 参考文献
- https://en.wikipedia.org/wiki/Exponential_smoothing
- https://orangematter.solarwinds.com/2019/12/15/holt-winters-forecasting-simplified/

46
docs/zh/06-advanced/06-data-analysis/03-anomaly-detection.md Normal file
View File

@ -0,0 +1,46 @@
---
title: "Anomaly-detection"
sidebar_label: "Anomaly-detection"
---
本节讲述 异常检测 算法模型的使用方法。
## 概述
分析平台提供了 6 种异常检查模型，6 种异常检查模型分为 3 个类别，分别属于基于统计的异常检测模型、基于数据密度的检测模型、基于深度学习的异常检测模型。在不指定异常检测使用的方法的情况下，默认调用 iqr 的方法进行计算。
### 统计学异常检测方法
- k-sigma<sup>[1]</sup>: 即 ***68–95–99.7 rule*** 。***k***值默认为3， 即序列均值的 3 倍标准差范围为边界，超过边界的是异常值。KSigma 要求数据整体上服从正态分布，如果一个点偏离均值K倍标准差，则该点被视为异常点.
|参数名称|说明|是否必选|默认值|
|---|---|---|---|
|k|标准差倍数|选填|3|
- IQR<sup>[2]</sup>：四分位距 (Interquartile range, IQR) 是一种衡量变异性的方法. 四分位数将一个按等级排序的数据集划分为四个相等的部分。即 Q1（第 1 个四分位数）、Q2（第 2 个四分位数）和 Q3（第 3 个四分位数）。IQR 定义为 Q3–Q1，位于 Q3+1.5 。无输入参数。
- Grubbs<sup>[3]</sup>: 又称为 Grubbs' test，即最大标准残差测试。Grubbs 通常用作检验最大值、最小值偏离均值的程度是否为异常，该单变量数据集遵循近似标准正态分布。非正态分布数据集不能使用该方法。无输入参数。
- SHESD<sup>[4]</sup>： 带有季节性的 ESD 检测算法。ESD 可以检测时间序列数据的多异常点。需要指定异常点比例的上界***k***，最差的情况是至多49.9%。数据集的异常比例一般不超过5%
|参数名称|说明|是否必选|默认值|
|---|---|---|---|
|k|异常点在输入数据集中占比，范围是$`1\le K \le 49.9`$ |选填|5|
### 基于数据密度的检测方法
LOF<sup>[5]</sup>: 局部离群因子(LOF，又叫局部异常因子)算法是Breunig于2000年提出的一种基于密度的局部离群点检测算法，该方法适用于不同类簇密度分散情况迥异的数据。根据数据点周围的数据密集情况，首先计算每个数据点的一个局部可达密度，然后通过局部可达密度进一步计算得到每个数据点的一个离群因子，该离群因子即标识了一个数据点的离群程度，因子值越大，表示离群程度越高，因子值越小，表示离群程度越低。最后，输出离群程度最大的top(n)个点。
### 基于深度学习的检测方法
使用自动编码器的异常检测模型。可以对具有周期性的数据具有较好的检测结果。但是使用该模型需要针对输入的时序数据进行训练，同时将训练完成的模型部署到服务目录中，才能够运行与使用。
### 参考文献
1. https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule
2. https://en.wikipedia.org/wiki/Interquartile_range
3. Adikaram, K. K. L. B.; Hussein, M. A.; Effenberger, M.; Becker, T. (2015-01-14). "Data Transformation Technique to Improve the Outlier Detection Power of Grubbs's Test for Data Expected to Follow Linear Relation". Journal of Applied Mathematics. 2015: 1–9. doi:10.1155/2015/708948.
4. Hochenbaum, O. S. Vallis, and A. Kejariwal. 2017. Automatic Anomaly Detection in the Cloud Via Statistical Learning. arXiv preprint arXiv:1704.07706 (2017).
5. Breunig, M. M.; Kriegel, H.-P.; Ng, R. T.; Sander, J. (2000). LOF: Identifying Density-based Local Outliers (PDF). Proceedings of the 2000 ACM SIGMOD International Conference on Management of Data. SIGMOD. pp. 93–104. doi:10.1145/335191.335388. ISBN 1-58113-217-4.

167
docs/zh/06-advanced/06-data-analysis/addins.md Normal file
View File

@ -0,0 +1,167 @@
---
title: "addins"
sidebar_label: "addins"
---
本节说明如何将自己开发的新预测算法和异常检测算法整合到 TDengine 分析平台， 并能够通过 SQL 语句进行调用。
## 目录结构
![数据分析功能架构图](./pic/dir.png)
|目录|说明|
|---|---|
|taos|Python 源代码目录，其下包含了算法具体保存目录 algo，放置杂项目录 misc， 单元测试和集成测试目录 test。 algo目录下 ad 放置异常检测算法代码， fc 放置预测算法代码|
|script|是安装脚本和发布脚本放置目录|
|model|放置针对数据集完成的训练模型|
|cfg| 配置文件目录|
## 约定与限制
定义异常检测算法的 Python 代码文件 需放在 /taos/algo/ad 目录中，预测算法 Python 代码文件需要放在 /taos/algo/fc 目录中，以确保系统启动的时候能够正常加载对应目录下的 Python 文件。
### 类命名规范
算法类的名称需要 以下划线开始，以 Service 结尾。例如：_KsigmaService 是 KSigma 异常检测算法的实现类。
### 类继承约定
异常检测算法需要从 `AbstractAnomalyDetectionService` 继承，并实现其核心抽象方法 `execute`.
预测算法需要从 `AbstractForecastService` 继承，同样需要实现其核心抽象方法 `execute`。
### 类属性初始化
每个算法实现的类需要静态初始化两个类属性，分别是
`name`: 的触发调用关键词，全小写英文字母。
`desc`：该算法的描述信息。
### 核心方法输入与输出约定
`execute` 是算法处理的核心方法。调用该方法的时候， `self.list` 已经设置好输入数组。
异常检测输出结果
`execute` 的返回值是长度与 `self.list` 相同的数组，数组位置为 -1 的即为异常值点。例如：输入数组是 [2, 2, 2, 2, 100]， 如果 100 是异常点，那么返回值是 [1, 1, 1, 1, -1]。
预测输出结果
对于预测算法， `AbstractForecastService` 的对象属性说明如下：
|属性名称|说明|默认值|
|---|---|---|
|period|输入时序数据的周期性，多少个数据点表示一个完整的周期。如果没有周期性，那么设置为 0 即可。| 0|
|start_ts|预测数据的开始时间| 0|
|time_step|预测结果的两个数据点之间时间间隔|0 |
|fc_rows|预测结果数量| 0 |
|return_conf|返回结果中是否包含执行区间范围，如果算法计算结果不包含置信区间，那么上界和下界与自身相同| 1|
|conf|执行区间分位数 0.05|
预测返回结果如下：
```python
return {
"rows": self.fc_rows, # 预测数据行数
"period": self.period, # 数据周期性，同输入
"algo": "holtwinters", # 预测使用的算法
"mse": mse, # 预测算法的 mse
"res": res # 结果数组 [时间戳数组, 预测结果数组, 预测结果执行区间下界数组，预测结果执行区间上界数组]
}
```
## 示例代码
```python
import numpy as np
from service import AbstractAnomalyDetectionService
# 算法实现类名称 需要以下划线 "_" 开始，并以 Service 结束，如下 _IqrService 是 IQR 异常检测算法的实现类。
class _IqrService(AbstractAnomalyDetectionService):
""" IQR algorithm 定义类，从 AbstractAnomalyDetectionService 继承，并实现 AbstractAnomalyDetectionService类的抽象函数 """
# 定义算法调用关键词，全小写ASCII码(必须添加)
name = 'iqr'
# 该算法的描述信息(建议添加)
desc = """found the anomaly data according to the inter-quartile range"""
def __init__(self):
super().__init__()
def execute(self):
""" execute 是算法实现逻辑的核心实现，直接修改该实现即可 """
# self.list 是输入数值列，list 类型，例如：[1,2,3,4,5]。设置 self.list 的方法在父类中已经进行了定义。实现自己的算法，修改该文件即可，以下代码使用自己的实现替换即可。
#lower = np.quantile(self.list, 0.25)
#upper = np.quantile(self.list, 0.75)
#min_val = lower - 1.5 * (upper - lower)
#max_val = upper + 1.5 * (upper - lower)
#threshold = [min_val, max_val]
# 返回值是与输入数值列长度相同的数据列，异常值对应位置是 -1。例如上述输入数据列，返回数值列是 [1, 1, 1, 1, -1],表示 [5] 是异常值。
return [-1 if k < threshold[0] or k > threshold[1] else 1 for k in self.list]
def set_params(self, params):
"""该算法无需任何输入参数，直接重载父类该函数，不处理算法参数设置逻辑"""
pass
```
## 单元测试
在测试文件目录中的 anomaly_test.py 中增加单元测试用例。
```python
def test_iqr(self):
""" 测试 _IqrService 类 """
s = loader.get_service("iqr")
# 设置需要进行检测的输入数据
s.set_input_list(AnomalyDetectionTest.input_list)
# 测试 set_params 的处理逻辑
try:
s.set_params({"k": 2})
except ValueError as e:
self.assertEqual(1, 0)
r = s.execute()
# 绘制异常检测结果
draw_ad_results(AnomalyDetectionTest.input_list, r, "iqr")
# 检查结果
self.assertEqual(r[-1], -1)
self.assertEqual(len(r), len(AnomalyDetectionTest.input_list))
```
## 需要模型的算法
针对特定数据集，进行模型训练的算法，在训练完成后。需要将训练得到的模型保存在 model 目录中。需要注意的是，针对每个算法，需要建立独立的文件夹。例如 auto_encoder 的训练算法在 model 目录下建立了， autoencoder的目录，使用该算法针对不同数据集训练得到的模型，均需要放置在该目录下。
训练完成后的模型，使用 joblib 进行保存。
并在 model 目录下建立对应的文件夹存放该模型。
保存模型的调用，可参考 encoder.py 的方式，用户通过调用 set_params 方法，并指定参数 `{"model": "ad_encoder_keras"}` 的方式，可以调用该模型进行计算。
具体的调用方式如下：
```python
def test_autoencoder_ad(self):
# 获取特定的算法服务
s = loader.get_service("ac")
data = self.__load_remote_data_for_ad()
# 设置异常检查的输入数据
s.set_input_list(data)
# 指定调用的模型，该模型是之前针对该数据集进行训练获得
s.set_params({"model": "ad_encoder_keras"})
# 执行检查动作，并返回结果
r = s.execute()
num_of_error = -(sum(filter(lambda x: x == -1, r)))
self.assertEqual(num_of_error, 109)
```

311
docs/zh/06-advanced/06-data-analysis/index.md
View File

@ -5,9 +5,314 @@ title: 数据分析功能
## 概述
TDengine 提供数据分析功能的扩展组件，通过引入 ANode，TDengine 能够支持时间序列的机器学习分析
下图展示了数据分析的技术架构。
TDengine 通过 ANode(AnalysisNode) 是提供数据分析功能的扩展组件，通过 Restful 接口提供分析服务，从而拓展 TDengine 的功能，支持时间序列高级分析功能。
ANode 是无状态的数据分析节点，集群中可以存在多个 ANode节点，相互之间没有关联。将 ANode 注册到 TDengine 集群以后，通过 SQL 语句即可调用并完成时序分析任务。
下图是数据分析的技术架构示意图。
![数据分析功能架构图](./pic/data-analysis.png)
## 安装部署
### 环境准备
ANode 的要求节点上准备有 Python 3.10 及以上版本以及相应的Python包自动安装组件 Pip ，同时请确保能够正常连接互联网。
### 安装及卸载
使用专门的 ANode 安装包 TDengine-enterprise-anode-1.x.x.tar.gz 进行 ANode 的安装部署工作，安装过程与 TDengien 的安装流程一致。
```bash
tar -xzvf TDengine-enterprise-anode-1.0.0.tar.gz
cd TDengine-enterprise-anode-1.0.0
sudo ./install.sh
```
卸载 ANode，执行命令 `rmtaosanode` 即可。
### 其他
为了避免 ANode 安装后影响目标节点现有的 Python 库。 ANode 使用 Python 虚拟环境运行，安装后的默认 Python 目录处于 `/var/lib/taos/taosanode/venv/`。为了避免反复安装虚拟环境带来的开销，卸载 ANode 并不会自动删除该虚拟环境，如果您确认不需要 Python 的虚拟环境，可以手动删除。
## 启动及停止服务
安装 ANode 以后，可以使用`systemctl`来管理 ANode 的服务。使用如下命令可以启动/停止/检查状态。
```bash
systemctl start taosanoded
systemctl stop taosanoded
systemctl status taosanoded
```
## 目录及配置说明
|目录/文件|说明|
|---------------|------|
|/usr/local/taos/taosanode/bin|可执行文件目录|
|/usr/local/taos/taosanode/resource|资源文件目录，链接到文件夹 /var/lib/taos/taosanode/resource/|
|/usr/local/taos/taosanode/lib|库文件目录|
|/var/lib/taos/taosanode/model/|模型文件目录，链接到文件夹 /var/lib/taos/taosanode/model|
|/var/log/taos/taosanode/|日志文件目录|
|/etc/taos/taosanode.ini|配置文件|
### 配置说明
Anode 提供的 RestFul 服务使用 uWSGI 驱动，因此 ANode 的配置和 uWSGI 的配置在同一个配置文件中，具体如下：
```ini
[uwsgi]
# charset
env=LC_ALL=en_US.UTF-8
# ip:port
http = 127.0.0.1:6050
# the local unix socket file than communicate to Nginx
#socket = 127.0.0.1:8001
#socket-timeout=10
# base directory
chdir = /usr/local/taos/taosanode/lib
# initialize python file
wsgi-file = /usr/local/taos/taosanode/lib/app.py
# invoke app model
callable = app
# auto remove unix Socket and pid file when stopping
vacuum = true
# socket exec model
#chmod-socket = 664
# uWSGI pid
uid=root
# uWSGI gid
gid=root
# main process
master = true
# the number of worker processes
processes = 2
# pid file
pidfile = /usr/local/taos/taosanode/uwsgi.pid
# enable threads
enable-threads=true
# the number of threads for each process
threads=2
# memory useage report
memory-report = true
reload-mercy = 10
# conflict with systemctl, so do NOT uncomment this
# daemonize = /var/log/taos/taosanode/taosanode.log
# set log
logto = /var/log/taos/taosanode/taosanode.log
# monitor server
stats = 127.0.0.1:8387
# python virtual environment directory
virtualenv = /usr/local/taos/taosanode/venv/
[taosanode]
# default app log file
app-log = /var/log/taos/taosanode/taosanode.app.log
# model storage directory
model-dir=/usr/local/taos/taosanode/model/
# default log level
log-level = DEBUG
```
**提示**
请勿设置 `daemonize` 参数，该参数会导致 uWSGI 与 systemctl 冲突，从而无法正常启动。
## ANode 基本操作
### 管理 ANode
#### 创建 ANode
```sql
CREATE ANODE {node_url}
```
node_url 是提供服务的 ANode 的 IP 和 PORT, 例如：`create anode 'http://localhost:6050'`。启动 ANode 以后如果不注册到 TDengine 集群中，无法提供正常的服务。不建议 ANode 注册到两个或多个集群中。
#### 查看 ANode
列出集群中所有的数据分析节点，包括其 `FQDN`, `PORT`, `STATUS`。
```sql
SHOW ANODES;
```
#### 查看提供的时序数据分析服务
```SQL
SHOW ANODES FULL;
```
#### 强制刷新 TDengine 集群中分析算法缓存
```SQL
UPDATE ANODE {node_id}
UPDATE ALL ANODES
```
#### 删除 ANode
```sql
DROP ANODE {anode_id}
```
删除 ANode 只是将 ANode 从 TDengine 集群中删除，管理 ANode 的启停仍然需要使用`systemctl`命令。
### 时序数据分析功能
#### 白噪声检查
平台提供 Restful的服务检测输入时间序列是否是白噪声时间序列（White Noise Data, WND），白噪声时间序列及随机数序列。
此外，分析平台要求输入的数据不能是 , 因此针对的所有数据均默认进行 白噪声检查。当前白噪声检查采用通行的 `Ljung-Box`检验，`Ljung-Box` 统计量检查过程需要遍历整个输入序列并进行计算。
如果用户能够明确输入序列一定不是白噪声序列，那么可以通过输入参数，指定预测之前忽略该检查，从而节省分析过程的 CPU 计算资源。
同时支持独立地针对输入序列进行白噪声检测（该检测功能暂不独立对外开放）。
#### 数据重采样和时间戳对齐
数据分析平台支持将输入的数据进行重采样的预处理，从而确保输出结果按照用户指定的等间隔进行处理。处理过程分为两种类别：
- 数据时间戳对齐。由于真实数据时间可能并非严格按照查询指定的时间戳输入。此时数据平台将自动将数据的时间间隔按照指定的时间间隔进行对齐。例如有输入时间序列：[11, 22, 29, 41]，用户指定时间间隔为 10，那么该序列将被对齐重整为以下序列 [10, 20, 30, 40]。
- 数据时间重采样。用户输入的时间序列其采样频率超过了指定的查询需要获得结果的时间间隔，例如输入原始数据是 5， 但是输出结果的频率是 10. [0， 5， 10， 15， 20， 25， 30]，那么该输入数据列将重采用为间隔 为 10 的输入序列，其结果如下 [0, 10, 20，30]。[5, 15, 25] 处的数据将被丢弃。
需要注意的是，数据输入平台不支持缺失数据补齐后进行的预测分析，如果输入时间序列数据[11, 22, 29, 49]，并且用户要求的时间间隔为 10， 重整对齐后的序列是 [10, 20, 30, 50] 那么该序列进行预测分析将返回错误。
#### 时序数据异常检测
异常检测是针对输入的时序数据，使用预设或用户指定的算法确定时间序列中**可能**出现异常时间序列点，对于时间序列中若干个连续的异常点，将自动合并成为一个连续的（闭区间）异常窗口。对于只有单个点的场景，异常窗口窗口退化成为一个起始时间和结束时间相同的点。
异常检测生成的异常窗口受检测算法和算法参数的共同影响，对于异常窗口范围内的数据，可以应用 TDengine 提供的聚合和标量函数进行查询或变换处理。
对于输入时间序列 (1, 20), (2, 22), (3, 91), (4, 120), (5, 18), (6, 19)。系统检测到 (3, 91), (4, 120) 为异常点，那么返回的异常窗口是闭区间 [3, 4]。
##### 语法
```SQL
ANOMALY_WINDOW(column_name, option_expr)
option_expr: {"
algo=expr1
[,wncheck=1|0]
[,expr2]
"}
```
1. `column`：进行时序数据异常检测的输入数据列，当前只支持单列输入，且只能是数值类型，不能是字符类型（例如：`NCHAR` `VARCHAR` `VARBINARY`等类型），**不支持函数表达式**。
2. `options`：字符串。其中使用 K/V 调用异常检测的算法，及与算法相关的参数。采用 逗号分隔的K/V字符串表示，其中的字符串不需要使用单引号、双引号、或转意号等符号，不能使用中文及其他宽字符。例如：`algo=ksigma, k=2` 表示进行异常检测的算法是 ksigma，该算法接受的输入参数是 2。
3. 异常检测的结果可以作为外层查询的子查询输入，在 `SELECT` 子句中使用的聚合函数或标量函数与其他类型的窗口查询相同。
4. 输入数据默认进行白噪声检查，如果检查结果是输入数据是白噪声，将不会有任何（异常）窗口信息返回。
**参数说明**
|参数|含义|默认值|
|---|---|---|
|algo|异常检测调用的算法|iqr|
|wncheck|对输入数据列是否进行白噪声检查|取值为0或者1，默认值为 1，表示进行白噪声检查|
异常检测的返回结果以窗口的形式呈现，因此窗口查询相关的伪列在这种场景下仍然可用。可以使用的伪列如下：
1. `_WSTART`： 异常窗口开始时间戳
2. `_WEND`：异常窗口结束时间戳
3. `_WDURATION`：异常窗口持续时间
**示例**
```SQL
--- 使用 iqr 算法进行异常检测，检测列 i32 列。
SELECT _wstart, _wend, SUM(i32)
FROM ai.atb
ANOMALY_WINDOW(i32, "algo=iqr");
--- 使用 ksigma 算法进行异常检测，输入参数 k 值为 2，检测列 i32 列
SELECT _wstart, _wend, SUM(i32)
FROM ai.atb
ANOMALY_WINDOW(i32, "algo=ksigma,k=2");
```
```
taos> SELECT _wstart, _wend, count(*) FROM ai.atb ANOMAYL_WINDOW(i32);
_wstart | _wend | count(*) |
============================================================================
2020-01-01 00:00:16.000 | 2020-01-01 00:00:16.001 | 1 |
Query OK, 1 row(s) in set (0.028946s)
```
**可用异常检测算法**
- iqr
- ksigma
- grubbs
- lof
- shesd
- tac
#### 时序数据预测
数据预测以一段训练数据作为输入，预测接下来一个连续时间区间内，时序数据的趋势。
##### 语法
```SQL
FORECAST(column_expr, option_expr)
option_expr: {"
algo=expr1
[,wncheck=1|0]
[,conf=conf_val]
[,every=every_val]
[,rows=rows_val]
[,start=start_ts_val]
[,expr2]
"}
```
1. `column_expr`：预测的时序数据列。与异常检测相同，只支持数值类型输入。
2. `options`：异常检测函数的参数，使用规则与 anomaly_window 相同。预测还支持`conf`, `every`, `rows`, `start`, `rows` 几个参数，其含义如下：
**参数说明**
|参数|含义|默认值|
|---|---|---|
|algo|预测分析使用的算法|holtwinters|
|wncheck|白噪声（white noise data）检查|默认值为 1，0 表示不进行检查|
|conf|预测数据的置信区间范围 ，取值范围[0, 100]|95|
|every|预测数据的采样间隔|输入数据的采样间隔|
|start|预测结果的开始时间戳|输入数据最后一个时间戳加上一个采样时间段|
|rows|预测结果的记录数|10|
1. 预测查询结果新增了三个伪列，具体如下： `_FROWTS`：预测结果的时间戳、`_FLOW`：置信区间下界、`_FHIGH`：置信区间上界, 对于没有置信区间的预测算法，其置信区间同预测结果
2. 更改参数 `START`：返回预测结果的起始时间，改变这个起始时间不会影响返回的预测数值，只影响起始时间。
3. `EVERY`：可以与输入数据的采样频率不同。采样频率只能低于或等于输入数据采样频率，不能**高于**输入数据的采样频率。
4. 对于某些不需要计算置信区间的算法，即使指定了置信区间，返回的结果中其上下界退化成为一个点。
**示例**
```SQL
--- 使用 arima 算法进行预测，预测结果是 10 条记录（默认值），数据进行白噪声检查，默认置信区间 95%.
SELECT _flow, _fhigh, _frowts, FORECAST(i32, "algo=arima")
FROM ai.ftb;
--- 使用 arima 算法进行预测，输入数据的是周期数据，每10个采样点是一个周期。返回置信区间是 95%.
SELECT _flow, _fhigh, _frowts, FORECAST(i32, "algo=arima,alpha=95,period=10")
FROM ai.ftb;
```
```
taos> select _flow, _fhigh, _frowts, forecast(i32) from ai.ftb;
_flow | _fhigh | _frowts | forecast(i32) |
========================================================================================
10.5286684 | 41.8038254 | 2020-01-01 00:01:35.001 | 26 |
-21.9861946 | 83.3938904 | 2020-01-01 00:01:36.001 | 30 |
-78.5686035 | 144.6729126 | 2020-01-01 00:01:37.001 | 33 |
-154.9797363 | 230.3057709 | 2020-01-01 00:01:38.001 | 37 |
-253.9852905 | 337.6083984 | 2020-01-01 00:01:39.001 | 41 |
-375.7857971 | 466.4594727 | 2020-01-01 00:01:40.001 | 45 |
-514.8043823 | 622.4426270 | 2020-01-01 00:01:41.001 | 53 |
-680.6343994 | 796.2861328 | 2020-01-01 00:01:42.001 | 57 |
-868.4956665 | 992.8603516 | 2020-01-01 00:01:43.001 | 62 |
-1076.1566162 | 1214.4498291 | 2020-01-01 00:01:44.001 | 69 |
```
**可用预测算法**
- arima
- holtwinters

BIN
docs/zh/06-advanced/06-data-analysis/pic/data-analysis.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 62 KiB

After

Width:  |  Height:  |  Size: 49 KiB

BIN
docs/zh/06-advanced/06-data-analysis/pic/dir.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.1 KiB

18
docs/zh/08-operation/02-planning.md
View File

@ -53,7 +53,7 @@ M = (T × S × 3 + (N / 4096) + 100)
与 WebSocket 连接方式相比，RESTful 连接方式在内存占用上更大，除了缓冲区所需的内存以外，还需要考虑每个连接响应结果的内存开销。这种内存开销与响应结果的JSON 数据大小密切相关，特别是在查询数据量很大时，会占用大量内存。
由于 RESTful 连接方式不支持分批获取查询数据，这就导致在查询获取超大结果集时，可能会占用特别大的内存，从而导致内存溢出，因此，在大型项目中，建议打开batchfetch=true 选项，以启用 WebSocket 连接方式，实现流式结果集返回，从而避免内存溢出的风险
由于 RESTful 连接方式不支持分批获取查询数据，这就导致在查询获取超大结果集时，可能会占用特别大的内存，从而导致内存溢出，因此，在大型项目中，建议使用 WebSocket 连接方式，实现流式结果集返回，从而避免内存溢出的风险
**注意**
- 建议采用 RESTful/WebSocket 连接方式来访问 TDengine 集群，而不采用taosc 原生连接方式。
@ -146,11 +146,11 @@ TDengine 的多级存储功能在使用上还具备以下优点。
下表列出了 TDengine 的一些接口或组件的常用端口，这些端口均可以通过配置文件中的参数进行修改。
|接口或组件 | 端口 |
|:---------------------------:|:---------:|
|原生接口（taosc） | 6030 |
|RESTful 接口 | 6041 |
|WebSocket 接口 |6041 |
|taosKeeper | 6043 |
|taosX | 6050, 6055 |
|taosExplorer | 6060 |
| 接口或组件 | 端口 |
| :---------------: | :--------: |
| 原生接口（taosc） | 6030 |
| RESTful 接口 | 6041 |
| WebSocket 接口 | 6041 |
| taosKeeper | 6043 |
| taosX | 6050, 6055 |
| taosExplorer | 6060 |

14
docs/zh/08-operation/03-deployment.md
View File

@ -368,6 +368,18 @@ spec:
labels:
app: "tdengine"
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- tdengine
topologyKey: kubernetes.io/hostname
containers:
- name: "tdengine"
image: "tdengine/tdengine:3.2.3.0"
@ -837,4 +849,4 @@ Helm 管理下，清理操作也变得简单：
helm uninstall tdengine
```
但 Helm 也不会自动移除 PVC，需要手动获取 PVC 然后删除掉。
但 Helm 也不会自动移除 PVC，需要手动获取 PVC 然后删除掉。

30
docs/zh/08-operation/12-multi.md
View File

@ -109,9 +109,37 @@ s3migrate database <db_name>;
| # | 参数 | 默认值 | 最小值 | 最大值 | 描述 |
| :--- | :----------- | :----- | :----- | :------ | :----------------------------------------------------------- |
| 1 | s3_keeplocal | 365 | 1 | 365000 | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 |
| 2 | s3_chunksize | 262144 | 131072 | 1048576 | 上传对象的大小阈值，与 TSDB_PAGESIZE 参数一样，不可修改，单位为 TSDB 页 |
| 2 | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 |
| 3 | s3_compact | 1 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作。 |
### 对象存储读写次数估算
对象存储服务的使用成本与存储的数据量及请求次数相关，下面分别介绍数据的上传及下载过程。
#### 数据上传
当 TSDB 时序数据超过 `s3_keeplocal` 参数指定的时间，相关的数据文件会被切分成多个文件块，每个文件块的默认大小是 512M 字节 (`s3_chunkpages * tsdb_pagesize`)。除了最后一个文件块保留在本地文件系统外，其余的文件块会被上传到对象存储服务。
```math
上传次数 = 数据文件大小 / (s3_chunkpages * tsdb_pagesize) - 1
```
在创建数据库时，可以通过 `s3_chunkpages` 参数调整每个文件块的大小，从而控制每个数据文件的上传次数。
其它类型的文件如 head, stt, sma 等，保留在本地文件系统，以加速预计算相关查询。
#### 数据下载
在查询操作中，如果需要访问对象存储中的数据，TSDB 不会下载整个数据文件，而是计算所需数据在文件中的位置，只下载相应的数据到 TSDB 页缓存中，然后将数据返回给查询执行引擎。后续查询首先检查页缓存，查看数据是否已被缓存。如果数据已缓存，则直接使用缓存中的数据，而无需重复从对象存储下载，从而有效降低从对象存储下载数据的次数。
相邻的多个数据页会作为一个数据块从对象存储下载一次，以减少从对象存储下载的次数。每个数据页的大小，在创建数据库时，通过 `tsdb_pagesize` 参数指定，默认 4K 字节。
```math
下载次数 = 查询需要的数据块数量 - 已缓存的数据块数量
```
页缓存是内存缓存，节点重启后，再次查询需要重新下载数据。缓存采用 LRU (Least Recently Used) 策略，当缓存空间不足时，最近最少使用的数据将被淘汰。缓存的大小可以通过 `s3PageCacheSize` 参数进行调整，通常来说，缓存越大，下载次数越少。
## Azure Blob 存储
本节介绍在 TDengine Enterprise 如何使用微软 Azure Blob 对象存储。本功能是上一小节‘对象存储’功能的扩展，需额外依赖 Flexify 服务提供的 S3 网关。通过适当的参数配置，可以把大部分较冷的时序数据存储到 Azure Blob 服务中。

23
docs/zh/14-reference/01-components/01-taosd.md
View File

@ -33,7 +33,7 @@ taosd 命令行参数如下
| secondEp | taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，缺省值：无 |
| fqdn | 启动 taosd 后所监听的服务地址，缺省值：所在服务器上配置的第一个 hostname |
| serverPort | 启动 taosd 后所监听的端口，缺省值：6030 |
| numOfRpcSessions | 允许一个客户端能创建的最大连接数，取值范围 100-100000，缺省值：30000 |
| numOfRpcSessions | 允许一个 dnode 能发起的最大连接数，取值范围 100-100000，缺省值：30000 |
| timeToGetAvailableConn | 获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值：500000 |
### 监控相关
@ -156,14 +156,14 @@ charset 的有效值是 UTF-8。
### 内存相关
| 参数名称 | 参数说明 |
| :----------------: | :---------------------------------------------: |
| rpcQueueMemoryAllowed | 一个 dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10 |
| syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：10485760-INT64_MAX，缺省值：服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 |
| rpcQueueMemoryAllowed | 一个 dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围：104857600-INT64_MAX，缺省值：服务器内存的 1/10 |
| syncLogBufferMemoryAllowed | 一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围：104857600-INT64_MAX，缺省值：服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 |
### 性能调优
| 参数名称 | 参数说明 |
| :----------------: | :---------------------------------------------: |
| numOfCommitThreads | 写入线程的最大数量，取值范围 0-1024，缺省值为 4 |
| numOfCommitThreads | 落盘线程的最大数量，取值范围 0-1024，缺省值为 4 |
### 日志相关
@ -180,6 +180,7 @@ charset 的有效值是 UTF-8。
| tmrDebugFlag | 定时器模块的日志开关，取值范围同上 |
| uDebugFlag | 共用功能模块的日志开关，取值范围同上 |
| rpcDebugFlag | rpc 模块的日志开关，取值范围同上 |
| cDebugFlag | 客户端模块的日志开关，取值范围同上 |
| jniDebugFlag | jni 模块的日志开关，取值范围同上 |
| qDebugFlag | query 模块的日志开关，取值范围同上 |
| dDebugFlag | dnode 模块的日志开关，取值范围同上，缺省值 135 |
@ -223,16 +224,16 @@ lossyColumns float|double
| 参数名称 | 参数说明 |
| :--------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
| enableCoreFile | crash 时是否生成 core 文件；0: 不生成，1：生成；默认值 为 1; 不同的启动方式，生成 core 文件的目录如下：1、systemctl start taosd 启动：生成的 core 在根目录下 <br/> 2、手动启动，就在 taosd 执行目录下。 |
| udf | 是否启动 UDF 服务；0: 不启动，1：启动；默认值 为 0 |
| ttlChangeOnWrite | ttl 到期时间是否伴随表的修改操作改变; 0: 不改变，1：改变 ；默认值 为 |
| tmqMaxTopicNum | 订阅最多可建立的 topic 数量; 取值范围 1-10000；缺省值 为20 |
| maxTsmaNum | 集群内可创建的TSMA个数；取值范围：0-3；缺省值: 3 |
| enableCoreFile | crash 时是否生成 core 文件；0: 不生成，1：生成；默认值为 1; 不同的启动方式，生成 core 文件的目录如下：1、systemctl start taosd 启动：生成的 core 在根目录下 <br/> 2、手动启动，就在 taosd 执行目录下。 |
| udf | 是否启动 UDF 服务；0: 不启动，1：启动；默认值为 0 |
| ttlChangeOnWrite | ttl 到期时间是否伴随表的修改操作改变; 0: 不改变，1：改变；默认值为 0 |
| tmqMaxTopicNum | 订阅最多可建立的 topic 数量; 取值范围 1-10000；缺省值为20 |
| maxTsmaNum | 集群内可创建的TSMA个数；取值范围：0-3；缺省值为 3 |
## taosd 监控指标
taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeeper 写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。
taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeeper 写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。
### taosd\_cluster\_basic 表
@ -458,4 +459,4 @@ TDengine 的日志文件主要包括普通日志和慢日志两种类型。
3. 多个客户端的日志存储在相应日志路径下的同一个 taosSlowLog.yyyy.mm.dd 文件里。
4. 慢日志文件不自动删除，不压缩。
5. 使用和普通日志文件相同的三个参数 logDir, minimalLogDirGB, asyncLog。另外两个参数 numOfLogLines，logKeepDays 不适用于慢日志。

1
docs/zh/14-reference/01-components/02-taosc.md
View File

@ -35,6 +35,7 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在
|smlAutoChildTableNameDelimiter | schemaless tag之间的连接符，连起来作为子表名，无缺省值 |
|smlTagName | schemaless tag 为空时默认的 tag 名字, 缺省值 "_tag_null" |
|smlTsDefaultName | schemaless自动建表的时间列名字通过该配置设置, 缺省值 "_ts" |
|smlDot2Underline | schemaless 把超级表名中的 dot 转成下划线 |
|enableCoreFile | crash 时是否生成 core 文件，0: 不生成， 1： 生成；缺省值：1 |
|enableScience | 是否开启科学计数法显示浮点数; 0: 不开始, 1: 开启；缺省值：1 |
|compressMsgSize | 是否对 RPC 消息进行压缩; -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩; 缺省值 -1|

2
docs/zh/14-reference/01-components/07-explorer.md
View File

@ -8,7 +8,7 @@ taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工
## 安装
taosEexplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。如果按照 GitHub 里步骤自己编译 TDengine 源代码生成的安装包不包含 taosExplorer。
taosExplorer 无需单独安装，从 TDengine 3.3.0.0 版本开始，它随着 TDengine 安装包一起发布，安装完成后，就可以看到 `taos-explorer` 服务。如果按照 GitHub 里步骤自己编译 TDengine 源代码生成的安装包不包含 taosExplorer。
## 配置

2
docs/zh/14-reference/02-tools/10-taosbenchmark.md
View File

@ -364,6 +364,8 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
- **max** : 数据类型的 列/标签 的最大值。生成的值将小于最小值。
- **scalingFactor** : 浮点数精度增强因子，仅当数据类型是 float/double 时生效，有效值范围为 1 至 1000000 的正整数。用于增强生成浮点数的精度，特别是在 min 或 max 值较小的情况下。此属性按 10 的幂次增强小数点后的精度：scalingFactor 为 10 表示增强 1 位小数精度，100 表示增强 2 位，依此类推。
- **fun** : 此列数据以函数填充，目前只支持 sin 和 cos 两函数，输入参数为时间戳换算成角度值，换算公式： 角度 x = 输入的时间列ts值 % 360。同时支持系数调节，随机波动因子调节，以固定格式的表达式展现，如 fun=“10\*sin(x)+100\*random(5)” , x 表示角度，取值 0 ~ 360度，增长步长与时间列步长一致。10 表示乘的系数，100 表示加或减的系数，5 表示波动幅度在 5% 的随机范围内。目前支持的数据类型为 int, bigint, float, double 四种数据类型。注意：表达式为固定模式，不可前后颠倒。
- **values** : nchar/binary 列/标签的值域，将从值中随机选择。

8
docs/zh/14-reference/03-taos-sql/02-database.md
View File

@ -30,6 +30,7 @@ database_option: {
| SINGLE_STABLE {0 | 1}
| TABLE_PREFIX value
| TABLE_SUFFIX value
| DNODES value
| TSDB_PAGESIZE value
| WAL_LEVEL {1 | 2}
| WAL_FSYNC_PERIOD value
@ -63,19 +64,20 @@ database_option: {
- MAXROWS：文件块中记录的最大条数，默认为 4096 条。
- MINROWS：文件块中记录的最小条数，默认为 100 条。
- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。
- STT_TRIGGER：表示落盘文件触发文件合并的个数。默认为 1，范围 1 到 16。对于少表高频场景，此参数建议使用默认配置，或较小的值；而对于多表低频场景，此参数建议配置较大的值。
- STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。
- SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。
- 0：表示可以创建多张超级表。
- 1：表示只可以创建一张超级表。
- TABLE_PREFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的前缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的前缀；例如，假定表名为 "v30001"，当 TSDB_PREFIX = 2 时 使用 "0001" 来决定分配到哪个 vgroup ，当 TSDB_PREFIX = -2 时使用 "v3" 来决定分配到哪个 vgroup
- TABLE_SUFFIX：当其为正值时，在决定把一个表分配到哪个 vgroup 时要忽略表名中指定长度的后缀；当其为负值时，在决定把一个表分配到哪个 vgroup 时只使用表名中指定长度的后缀；例如，假定表名为 "v30001"，当 TSDB_SUFFIX = 2 时 使用 "v300" 来决定分配到哪个 vgroup ，当 TSDB_SUFFIX = -2 时使用 "01" 来决定分配到哪个 vgroup。
- TSDB_PAGESIZE：一个 VNODE 中时序数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB到 16 MB。
- DNODES：指定 VNODE 所在的 DNODE 列表，如 '1,2,3'，以逗号区分且字符间不能有空格，仅企业版支持。
- WAL_LEVEL：WAL 级别，默认为 1。
- 1：写 WAL，但不执行 fsync。
- 2：写 WAL，而且执行 fsync。
- WAL_FSYNC_PERIOD：当 WAL_LEVEL 参数设置为 2 时，用于设置落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。
- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要WAL日志文件额外保留的最大时长策略。WAL日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 3600，表示在 WAL 保留最近 3600 秒的数据，请根据数据订阅的需要修改这个参数为适当值。
- WAL_RETENTION_SIZE：为了数据订阅消费，需要WAL日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。
- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要 WAL 日志文件额外保留的最大时长策略。WAL 日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 3600，表示在 WAL 保留最近 3600 秒的数据，请根据数据订阅的需要修改这个参数为适当值。
- WAL_RETENTION_SIZE：为了数据订阅消费，需要 WAL 日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。
### 创建数据库示例
```sql

14
docs/zh/14-reference/03-taos-sql/03-table.md
View File

@ -42,12 +42,12 @@ table_option: {
**使用说明**
1. 表（列）名命名规则参见[名称命名规则](./19-limit.md#名称命名规则)。
1. 表名最大长度为 192。
1. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键。
1. 除时间戳主键列之外，还可以通过 PRIMARY KEY 关键字指定第二列为额外的主键列。被指定为主键列的第二列必须为整型或字符串类型（varchar）。
1. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置）。
1. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节。
1. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考[按列压缩](../compress)
2. 表名最大长度为 192。
3. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键。
4. 除时间戳主键列之外，还可以通过 PRIMARY KEY 关键字指定第二列为额外的主键列。被指定为主键列的第二列必须为整型或字符串类型（VARCHAR）。
5. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 VARCHAR/NCHAR/GEOMETRY 类型的列还会额外占用 2 个字节的存储位置）。
6. 使用数据类型 VARCHAR/NCHAR/GEOMETRY，需指定其最长的字节数，如 VARCHAR(20)，表示 20 字节。
7. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考[按列压缩](../compress)
**参数说明**
@ -87,7 +87,7 @@ CREATE TABLE [IF NOT EXISTS] USING [db_name.]stb_name (field1_name [, field2_nam
**参数说明**
1. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。CSV 文件中应仅包含 table name 与 tag 值。如需插入数据，请参考数据写入章节。
1. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。CSV 文件中应仅包含 table name 与 tag 值。如需插入数据，请参考'数据写入'章节。
2. 为指定的 stb_name 创建子表，该超级表必须已经存在。
3. field_name 列表顺序与 CSV 文件各列内容顺序一致。列表中不允许出现重复项，且必须包含 `tbname`，可包含零个或多个超级表中已定义的标签列。未包含在列表中的标签值将被设置为 NULL。

26
docs/zh/14-reference/03-taos-sql/05-insert.md
View File

@ -1,7 +1,7 @@
---
sidebar_label: 数据写入
title: 数据写入
description: 写入数据的详细语法
description: 写入数据的详细语法
---
## 写入语法
@ -25,9 +25,9 @@ INSERT INTO tb_name [(field1_name, ...)] subquery
### 超级表语法
```sql
INSERT INTO
stb1_name [(field1_name, ...)]
stb1_name [(field1_name, ...)]
VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
[stb2_name [(field1_name, ...)]
[stb2_name [(field1_name, ...)]
VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
...];
```
@ -47,7 +47,7 @@ INSERT INTO
2. VALUES 语法表示了要插入的一行或多行数据。
3. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。
3. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。如仅需创建子表，请参考'表'章节。
4. `INSERT ... VALUES` 语句和 `INSERT ... FILE` 语句均可以在一条 INSERT 语句中同时向多个表插入数据。
@ -154,12 +154,20 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c
INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/csvfile_21001.csv'
d21002 USING meters (groupId) TAGS (2) FILE '/tmp/csvfile_21002.csv';
```
## 超级表语法
## 向超级表插入数据并自动创建子表
自动建表, 表名通过tbname列指定
自动建表, 表名通过 tbname 列指定
```sql
INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase)
values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32)
INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase)
VALUES ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32)
('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)
('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)
```
## 通过 CSV 文件向超级表插入数据并自动创建子表
根据 csv 文件内容，为 超级表创建子表，并填充相应 column 与 tag
```sql
INSERT INTO meters(tbname, location, groupId, ts, current, voltage, phase)
FILE '/tmp/csvfile_21002.csv'
```

2
docs/zh/14-reference/03-taos-sql/07-tag-index.md
View File

@ -11,7 +11,7 @@ description: 使用标签索引提升查询性能
创建索引的语法如下
```sql
CREATE INDEX index_name ON tbl_name (tagColName）
CREATE INDEX index_name ON tbl_name (tagColName)
```
其中 `index_name` 为索引名称， `tbl_name` 为超级表名称，`tagColName` 为要在其上建立索引的 tag 列的名称。`tagColName` 的类型不受限制，即任何类型的 tag 列都可以建立索引。

7
docs/zh/14-reference/03-taos-sql/14-stream.md
View File

@ -212,11 +212,11 @@ TDengine 对于修改数据提供两种处理方式，由 IGNORE UPDATE 选项
```sql
[field1_name,...]
```
用来指定stb_name的列与subquery输出结果的对应关系。如果stb_name的列与subquery输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果stb_name的列与subquery输出结果的数据类型不匹配，会把subquery输出结果的类型转换成对应的stb_name的列的类型。
在本页文档顶部的 [field1_name,...] 是用来指定 stb_name 的列与 subquery 输出结果的对应关系的。如果 stb_name 的列与 subquery 输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果 stb_name 的列与 subquery 输出结果的数据类型不匹配，会把 subquery 输出结果的类型转换成对应的 stb_name 的列的类型。
对于已经存在的超级表，检查列的schema信息
1. 检查列的schema信息是否匹配，对于不匹配的，则自动进行类型转换，当前只有数据长度大于4096byte时才报错，其余场景都能进行类型转换。
2. 检查列的个数是否相同，如果不同，需要显示的指定超级表与subquery的列的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。
1. 检查列的 schema 信息是否匹配，对于不匹配的，则自动进行类型转换，当前只有数据长度大于 4096byte 时才报错，其余场景都能进行类型转换。
2. 检查列的个数是否相同，如果不同，需要显示的指定超级表与 subquery 的列的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。
## 自定义TAG
@ -291,3 +291,4 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name;
CREATE SNODE ON DNODE [id]
```
其中的 id 是集群中的 dnode 的序号。请注意选择的dnode，流计算的中间状态将自动在其上进行备份。
从 3.3.4.0 版本开始，在多副本环境中创建流会进行 snode 的**存在性检查**，要求首先创建 snode。如果 snode 不存在，无法创建流。

6
docs/zh/14-reference/03-taos-sql/21-node.md
View File

@ -27,11 +27,15 @@ SHOW DNODES;
## 删除数据节点
```sql
DROP DNODE dnode_id
DROP DNODE dnode_id [force] [unsafe]
```
注意删除 dnode 不等于停止相应的进程。实际中推荐先将一个 dnode 删除之后再停止其所对应的进程。
只有在线节点可以被删除。如果要强制删除离线节点，需要执行强制删除操作, 即指定force选项。
当节点上存在单副本，并且节点处于离线，如果要强制删除该节点，需要执行非安全删除，即制定unsafe，并且数据不可再恢复。
## 修改数据节点配置
```sql

1
docs/zh/14-reference/05-connector/35-node.mdx
View File

@ -26,6 +26,7 @@ Node.js 连接器目前仅支持 WebSocket 连接器， 其通过 taosAdapter
| Node.js 连接器 版本 | 主要变化 | TDengine 版本 |
| :------------------: | :----------------------: | :----------------: |
| 3.1.1 | 优化了数据传输性能 | 3.3.2.0 及更高版本 |
| 3.1.0 | 新版本发布，支持 WebSocket 连接 | 3.2.0.0 及更高版本 |
## 处理异常

23
docs/zh/14-reference/05-connector/50-odbc.mdx
View File

@ -3,11 +3,14 @@ sidebar_label: ODBC
title: TDengine ODBC
---
TDengine ODBC 是为 TDengine 实现的 ODBC 驱动程序，支持 Windows 系统的应用（如 [PowerBI](https://powerbi.microsoft.com/zh-cn/) 等）通过 ODBC 标准接口访问本地、远程和云服务的 TDengine 数据库的数据表/视图。
TDengine ODBC 是为 TDengine 实现的 ODBC 驱动程序，支持 Windows 系统的应用（如 [PowerBI](https://powerbi.microsoft.com/zh-cn/) 等）以及用户自定义开发的应用程序，通过 ODBC 标准接口访问本地、远程和云服务的 TDengine 数据库。
TDengine ODBC 提供基于 WebSocket（推荐）和 原生连接两种方式连接 TDengine 数据库，使用时可以为 TDengine 数据源设置不同的连接方式。访问云服务时必须使用 WebSocket 连接方式。
TDengine ODBC 提供基于 WebSocket（推荐）和 原生连接两种方式连接 TDengine 数据库，使用时可以为 TDengine 数据源设置不同的连接方式。访问云服务时必须使用 WebSocket 连接方式。
注意：TDengine ODBC 支持 32/64 位 Windows 系统，调用 TDengine ODBC 需要通过相应位数的 ODBC 驱动管理器进行。在 32 位 Windows 系统或者 64 位 Windows 系统的 32 位应用程序中，仅支持使用 WebSocket 连接方式访问 TDengine 数据库。
TDengine ODBC 提供 64 位 和 32 位 两种驱动程序。但 32 位仅企业版支持，且仅支持 WebSocket 连接方式。
**注意：**
- 驱动管理器：确保使用与应用程序架构匹配的 ODBC 驱动管理器。32 位应用程序需要使用 32 位 ODBC 驱动管理器，64 位应用程序需要使用 64 位 ODBC 驱动管理器。
- 数据源名称（DSN）：32 位和 64 位 ODBC 驱动管理器都可以看到所有 DSN，用户 DSN 标签页下的 DSN 如果名字相同会共用，因此需要在 DSN 名称上去区分。
想更多了解 TDengine 时序时序数据库的使用，可访问 [TDengine官方文档](https://docs.taosdata.com/intro/)。
@ -24,17 +27,17 @@ TDengine ODBC 提供基于 WebSocket（推荐）和 原生连接两种方式连
### 数据源连接类型与区别
TDengine ODBC 支持两种连接 TDengine 数据库方式：Websocket 连接与 Native 连接，其区别如下：
TDengine ODBC 支持两种连接 TDengine 数据库方式：WebSocket 连接与 Native 连接，其区别如下：
1. 访问云服务仅支持使用 Websocket 连接方式。
1. 访问云服务仅支持使用 WebSocket 连接方式。
2. 32 位应用程序仅支持使用 WebSocket 连接方式。
3. Websocket 连接的兼容性更好，一般不需要随着 TDengine 数据库服务端升级而升级客户端的库。
3. WebSocket 连接的兼容性更好，一般不需要随着 TDengine 数据库服务端升级而升级客户端的库。
4. Native 连接通常性能更好一点，但是必须与 TDengine 数据库服务端的版本保持一致。
5. 对于一般用户，建议使用 **Websocket** 连接方式，性能与 Native 差别不大，兼容性更好。
5. 对于一般用户，建议使用 **WebSocket** 连接方式，性能与 Native 差别不大，兼容性更好。
### WebSocket 连接
@ -46,11 +49,11 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式：Websocket 连接与
4. 点击完成，进入 TDengine ODBC 数据源配置页面，填写如下必要信息
![ODBC websocket connection config](./assets/odbc-ws-config-zh.webp)
![ODBC WebSocket connection config](./assets/odbc-ws-config-zh.webp)
4.1 【DSN】:Data Source Name 必填，为新添加的 ODBC 数据源命名
4.2【连接类型】 : 必选，选择连接类型，这里选择 【Websocket】
4.2【连接类型】 : 必选，选择连接类型，这里选择 【WebSocket】
4.3【URL】必填，ODBC 数据源 URL，示例: `http://localhost:6041`， 云服务的 url 示例: `https://gw.cloud.taosdata.com?token=your_token`
@ -111,7 +114,7 @@ WebSocket 连接方式除此之外还支持 Windows X64系统上运行的 32 位
| taos_odbc 版本 | 主要变化 | TDengine 版本 |
| :----------- | :-------------------------------------------------------------------------------------------------- | :---------------- |
| v1.1.0 | 1. 支持视图功能；<br/>2. 支持 VARBINARY/GEOMETRY 数据类型； | 3.3.3.0及更高版本 |
| v1.1.0 | 1. 支持视图功能；<br/>2. 支持 VARBINARY/GEOMETRY 数据类型；<br/>3. 支持 ODBC 32 位 WebSocket 连接方式（仅企业版支持）；<br/>4. 支持 ODBC 数据源配置对话框设置对工业软件 KingSCADA、Kepware 等的兼容性适配选项（仅企业版支持）； | 3.3.3.0及更高版本 |
| v1.0.2 | 支持 CP1252 字符编码； | 3.2.3.0及更高版本 |
| v1.0.1 | 1. 支持 DSN 设置 BI 模式，在 BI 模式下 TDengine 数据库不返回系统数据库和超级表子表信息；<br/>2. 重构字符集转换模块，提升读写性能；<br/>3. ODBC 数据源配置对话框中默认修改默认连接方式为“WebSocket”；<br/>4. ODBC 数据源配置对话框增加“测试连接”控件；<br/>5. ODBC 数据源配置支持中文/英文界面； | - |
| v1.0.0.0 | 发布初始版本，支持与Tdengine数据库交互以读写数据，具体请参考“API 参考”一节 | 3.2.2.0及更高版本 |

4
docs/zh/14-reference/05-connector/index.md
View File

@ -34,8 +34,8 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器
| **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.1.0 | 当前版本 | 与 TDengine 相同版本 |
| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 |
| **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | 与 TDengine 相同版本 |
| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | 与 TDengine 相同版本 |
| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | 与 TDengine 相同版本 |
| **2.2.x.x** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | 与 TDengine 相同版本 |
| **2.0.x.x** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | 与 TDengine 相同版本 |
## 功能特性

49
docs/zh/14-reference/07-supported.md
View File

@ -6,15 +6,28 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表"
## TDengine 服务端支持的平台列表
| | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 以上** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** |
| ------------ | ---------------------------- | ----------------- | ---------------- | ------------------ | ------------ | ----------------- | ---------------- | --------- |
| X64 | ●/E | ●/E | ● | ● | ●/E | ●/E | ●/E | ● |
| 树莓派 ARM64 | | | ● | | | | | |
| 华为云 ARM64 | | | | ● | | | | |
| M1 | | | | | | | | ● |
| | **版本** | **X64 64bit** | **ARM64** |
| ----------------------|----------------| ------------- | --------- |
| **CentOS** | **7.9 以上** | ● | ● |
| **Ubuntu** | **18 以上** | ● | ● |
| **RedHat** | **RHEL 7 以上** | ● | ● |
| **Debian** | **6.0 以上** | ● | ● |
| **FreeBSD** | **12 以上** | ● | ● |
| **OpenSUSE** | **全部版本** | ● | ● |
| **SUSE Linux** | **11 以上** | ● | ● |
| **Fedora** | **21 以上** | ● | ● |
| **Windows Server** | **2016 以上** | ●/E | |
| **Windows** | **10/11** | ●/E | |
| **银河麒麟** | **V10 以上** | ●/E | ●/E |
| **中标麒麟** | **V7.0 以上** | ●/E | ●/E |
| **统信 UOS** | **V20 以上** | ●/E | |
| **凝思磐石** | **V8.0 以上** | ●/E | |
| **华为欧拉 openEuler** | **V20.03 以上** | ●/E | |
| **龙蜥 Anolis OS** | **V8.6 以上** | ●/E | |
| **macOS** | **11.0 以上** | | ● |
注：1) ● 表示经过官方测试验证， ○ 表示非官方测试验证，E 表示仅企业版支持。
2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。
2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/CentOS Stream/RedHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。
## TDengine 客户端和连接器支持的平台列表
@ -22,16 +35,16 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表"
对照矩阵如下：
| **CPU** | **X64 64bit** | **X64 64bit** | **ARM64** | **X64 64bit** | **ARM64** |
| ----------- | ------------- | ------------- | --------- | ------------- | --------- |
| **OS** | **Linux** | **Win64** | **Linux** | **macOS** | **macOS** |
| **C/C++** | ● | ● | ● | ● | ● |
| **JDBC** | ● | ● | ● | ○ | ○ |
| **Python** | ● | ● | ● | ● | ● |
| **Go** | ● | ● | ● | ● | ● |
| **NodeJs** | ● | ● | ● | ○ | ○ |
| **C#** | ● | ● | ○ | ○ | ○ |
| **Rust** | ● | ● | ○ | ● | ● |
| **RESTful** | ● | ● | ● | ● | ● |
| **CPU** | **X64 64bit** | **X64 64bit** | **X64 64bit** | **ARM64** | **ARM64** |
| ----------- | ------------- | ------------- | ------------- | --------- | --------- |
| **OS** | **Linux** | **Win64** | **macOS** | **Linux** | **macOS** |
| **C/C++** | ● | ● | ● | ● | ● |
| **JDBC** | ● | ● | ● | ● | ● |
| **Python** | ● | ● | ● | ● | ● |
| **Go** | ● | ● | ● | ● | ● |
| **NodeJs** | ● | ● | ● | ● | ● |
| **C#** | ● | ● | ○ | ● | ○ |
| **Rust** | ● | ● | ● | ○ | ● |
| **RESTful** | ● | ● | ● | ● | ● |
注：● 表示官方测试验证通过，○ 表示非官方测试验证通过，-- 表示未经验证。

4
docs/zh/26-tdinternal/01-arch.md
View File

@ -178,7 +178,7 @@ TDengine 集群可以容纳单个、多个甚至几千个数据节点。应用
TDengine 存储的数据包括采集的时序数据以及库、表相关的元数据、标签数据等，这些数据具体分为三部分：
- 时序数据：TDengine 的核心存储对象，存放于 vnode 里，由 data、head 和 last 三个文件组成，数据量大，查询量取决于应用场景。允许乱序写入，但暂时不支持删除操作，并且仅在 update 参数设置为 1 时允许更新操作。通过采用一个采集点一张表的模型，一个时间段的数据是连续存储，对单张表的写入是简单的追加操作，一次读，可以读到多条记录，这样保证对单个采集点的插入和查询操作，性能达到最优。
- 时序数据：时序数据是 TDengine 的核心存储对象，它们被存储在 vnode 中。时序数据由 data、head、sma 和 stt 4 类文件组成，这些文件共同构成了时序数据的完整存储结构。由于时序数据的特点是数据量大且查询需求取决于具体应用场景，因此 TDengine 采用了“一个数据采集点一张表”的模型来优化存储和查询性能。在这种模型下，一个时间段内的数据是连续存储的，对单张表的写入是简单的追加操作，一次读取可以获取多条记录。这种设计确保了单个数据采集点的写入和查询操作都能达到最优性能。
- 数据表元数据：包含标签信息和 Table Schema 信息，存放于 vnode 里的 meta 文件，支持增删改查四个标准操作。数据量很大，有 N 张表，就有 N 条记录，因此采用 LRU 存储，支持标签数据的索引。TDengine 支持多核多线程并发查询。只要计算内存足够，元数据全内存存储，千万级别规模的标签数据过滤结果能毫秒级返回。在内存资源不足的情况下，仍然可以支持数千万张表的快速查询。
- 数据库元数据：存放于 mnode 里，包含系统节点、用户、DB、STable Schema 等信息，支持增删改查四个标准操作。这部分数据的量不大，可以全内存保存，而且由于客户端有缓存，查询量也不大。因此目前的设计虽是集中式存储管理，但不会构成性能瓶颈。
@ -321,4 +321,4 @@ TDengine 采用了一种数据驱动的策略来实现缓存数据的持久化
此外，TDengine 还提供了数据分级存储的功能，允许用户将不同时间段的数据存储在不同存储设备的目录中，以此实现将“热”数据和“冷”数据分开存储。这样做可以充分利用各种存储资源，同时节约成本。例如，对于最新采集且需要频繁访问的数据，由于其读取性能要求较高，用户可以配置将这些数据存储在高性能的固态硬盘上。而对于超过一定期限、查询需求较低的数据，则可以将其存储在成本相对较低的机械硬盘上。
为了进一步降低存储成本，TDengine 还支持将时序数据存储在对象存储系统中。通过其创新性的设计，在大多数情况下，从对象存储系统中查询时序数据的性能接近本地硬盘的一半，而在某些场景下，性能甚至可以与本地硬盘相媲美。同时，TDengine 还允许用户对存储在对象存储中的时序数据执行删除和更新操作。
为了进一步降低存储成本，TDengine 还支持将时序数据存储在对象存储系统中。通过其创新性的设计，在大多数情况下，从对象存储系统中查询时序数据的性能接近本地硬盘的一半，而在某些场景下，性能甚至可以与本地硬盘相媲美。同时，TDengine 还允许用户对存储在对象存储中的时序数据执行删除和更新操作。

47
include/client/taos.h
View File

@ -81,6 +81,13 @@ typedef enum {
TSDB_SML_TIMESTAMP_NANO_SECONDS,
} TSDB_SML_TIMESTAMP_TYPE;
typedef enum TAOS_FIELD_T {
TAOS_FIELD_COL = 1,
TAOS_FIELD_TAG,
TAOS_FIELD_QUERY,
TAOS_FIELD_TBNAME,
} TAOS_FIELD_T;
typedef struct taosField {
char name[65];
int8_t type;
@ -95,6 +102,15 @@ typedef struct TAOS_FIELD_E {
int32_t bytes;
} TAOS_FIELD_E;
typedef struct TAOS_FIELD_STB {
char name[65];
int8_t type;
uint8_t precision;
uint8_t scale;
int32_t bytes;
TAOS_FIELD_T field_type;
} TAOS_FIELD_STB;
#ifdef WINDOWS
#define DLL_EXPORT __declspec(dllexport)
#else
@ -195,13 +211,6 @@ DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt);
typedef void TAOS_STMT2;
typedef enum TAOS_FIELD_T {
TAOS_FIELD_COL = 1,
TAOS_FIELD_TAG,
TAOS_FIELD_QUERY,
TAOS_FIELD_TBNAME,
} TAOS_FIELD_T;
typedef struct TAOS_STMT2_OPTION {
int64_t reqid;
bool singleStbInsert;
@ -232,7 +241,9 @@ DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows);
DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt);
DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert);
DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields);
DLL_EXPORT int taos_stmt2_get_stb_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_STB **fields);
DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields);
DLL_EXPORT void taos_stmt2_free_stb_fields(TAOS_STMT2 *stmt, TAOS_FIELD_STB *fields);
DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt);
DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt);
@ -251,17 +262,17 @@ DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res);
DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res);
DLL_EXPORT int taos_select_db(TAOS *taos, const char *db);
DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
DLL_EXPORT void taos_stop_query(TAOS_RES *res);
DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col);
DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows);
DLL_EXPORT bool taos_is_update_query(TAOS_RES *res);
DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
DLL_EXPORT void taos_reset_current_db(TAOS *taos);
DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
DLL_EXPORT void taos_stop_query(TAOS_RES *res);
DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col);
DLL_EXPORT int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows);
DLL_EXPORT bool taos_is_update_query(TAOS_RES *res);
DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
DLL_EXPORT void taos_reset_current_db(TAOS *taos);
DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);

27
include/common/tmsg.h
View File

@ -421,7 +421,7 @@ typedef enum ENodeType {
// physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, // INACTIVE
QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN,
QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN,
@ -435,7 +435,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_SORT,
QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT,
QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, // INACTIVE
QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
@ -1337,6 +1337,7 @@ typedef struct {
char* sql;
int8_t withArbitrator;
int8_t encryptAlgorithm;
char dnodeListStr[TSDB_DNODE_LIST_LEN];
} SCreateDbReq;
int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq);
@ -4104,18 +4105,16 @@ void tDeleteMqMetaRsp(SMqMetaRsp* pRsp);
#define MQ_DATA_RSP_VERSION 100
typedef struct {
struct {
SMqRspHead head;
STqOffsetVal rspOffset;
STqOffsetVal reqOffset;
int32_t blockNum;
int8_t withTbName;
int8_t withSchema;
SArray* blockDataLen;
SArray* blockData;
SArray* blockTbName;
SArray* blockSchema;
};
SMqRspHead head;
STqOffsetVal rspOffset;
STqOffsetVal reqOffset;
int32_t blockNum;
int8_t withTbName;
int8_t withSchema;
SArray* blockDataLen;
SArray* blockData;
SArray* blockTbName;
SArray* blockSchema;
union{
struct{

776
include/common/ttokendef.h
View File

@ -16,394 +16,394 @@
#ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_IS_IMPORT 35
#define TK_NK_INTEGER 36
#define TK_CREATEDB 37
#define TK_USER 38
#define TK_ENABLE 39
#define TK_SYSINFO 40
#define TK_ADD 41
#define TK_DROP 42
#define TK_GRANT 43
#define TK_ON 44
#define TK_TO 45
#define TK_REVOKE 46
#define TK_FROM 47
#define TK_SUBSCRIBE 48
#define TK_READ 49
#define TK_WRITE 50
#define TK_NK_DOT 51
#define TK_WITH 52
#define TK_ENCRYPT_KEY 53
#define TK_ANODE 54
#define TK_UPDATE 55
#define TK_ANODES 56
#define TK_DNODE 57
#define TK_PORT 58
#define TK_DNODES 59
#define TK_RESTORE 60
#define TK_NK_IPTOKEN 61
#define TK_FORCE 62
#define TK_UNSAFE 63
#define TK_CLUSTER 64
#define TK_LOCAL 65
#define TK_QNODE 66
#define TK_BNODE 67
#define TK_SNODE 68
#define TK_MNODE 69
#define TK_VNODE 70
#define TK_DATABASE 71
#define TK_USE 72
#define TK_FLUSH 73
#define TK_TRIM 74
#define TK_S3MIGRATE 75
#define TK_COMPACT 76
#define TK_IF 77
#define TK_NOT 78
#define TK_EXISTS 79
#define TK_BUFFER 80
#define TK_CACHEMODEL 81
#define TK_CACHESIZE 82
#define TK_COMP 83
#define TK_DURATION 84
#define TK_NK_VARIABLE 85
#define TK_MAXROWS 86
#define TK_MINROWS 87
#define TK_KEEP 88
#define TK_PAGES 89
#define TK_PAGESIZE 90
#define TK_TSDB_PAGESIZE 91
#define TK_PRECISION 92
#define TK_REPLICA 93
#define TK_VGROUPS 94
#define TK_SINGLE_STABLE 95
#define TK_RETENTIONS 96
#define TK_SCHEMALESS 97
#define TK_WAL_LEVEL 98
#define TK_WAL_FSYNC_PERIOD 99
#define TK_WAL_RETENTION_PERIOD 100
#define TK_WAL_RETENTION_SIZE 101
#define TK_WAL_ROLL_PERIOD 102
#define TK_WAL_SEGMENT_SIZE 103
#define TK_STT_TRIGGER 104
#define TK_TABLE_PREFIX 105
#define TK_TABLE_SUFFIX 106
#define TK_S3_CHUNKSIZE 107
#define TK_S3_KEEPLOCAL 108
#define TK_S3_COMPACT 109
#define TK_KEEP_TIME_OFFSET 110
#define TK_ENCRYPT_ALGORITHM 111
#define TK_NK_COLON 112
#define TK_BWLIMIT 113
#define TK_START 114
#define TK_TIMESTAMP 115
#define TK_END 116
#define TK_TABLE 117
#define TK_NK_LP 118
#define TK_NK_RP 119
#define TK_USING 120
#define TK_FILE 121
#define TK_STABLE 122
#define TK_COLUMN 123
#define TK_MODIFY 124
#define TK_RENAME 125
#define TK_TAG 126
#define TK_SET 127
#define TK_NK_EQ 128
#define TK_TAGS 129
#define TK_BOOL 130
#define TK_TINYINT 131
#define TK_SMALLINT 132
#define TK_INT 133
#define TK_INTEGER 134
#define TK_BIGINT 135
#define TK_FLOAT 136
#define TK_DOUBLE 137
#define TK_BINARY 138
#define TK_NCHAR 139
#define TK_UNSIGNED 140
#define TK_JSON 141
#define TK_VARCHAR 142
#define TK_MEDIUMBLOB 143
#define TK_BLOB 144
#define TK_VARBINARY 145
#define TK_GEOMETRY 146
#define TK_DECIMAL 147
#define TK_COMMENT 148
#define TK_MAX_DELAY 149
#define TK_WATERMARK 150
#define TK_ROLLUP 151
#define TK_TTL 152
#define TK_SMA 153
#define TK_DELETE_MARK 154
#define TK_FIRST 155
#define TK_LAST 156
#define TK_SHOW 157
#define TK_FULL 158
#define TK_PRIVILEGES 159
#define TK_DATABASES 160
#define TK_TABLES 161
#define TK_STABLES 162
#define TK_MNODES 163
#define TK_QNODES 164
#define TK_ARBGROUPS 165
#define TK_FUNCTIONS 166
#define TK_INDEXES 167
#define TK_ACCOUNTS 168
#define TK_APPS 169
#define TK_CONNECTIONS 170
#define TK_LICENCES 171
#define TK_GRANTS 172
#define TK_LOGS 173
#define TK_MACHINES 174
#define TK_ENCRYPTIONS 175
#define TK_QUERIES 176
#define TK_SCORES 177
#define TK_TOPICS 178
#define TK_VARIABLES 179
#define TK_BNODES 180
#define TK_SNODES 181
#define TK_TRANSACTIONS 182
#define TK_DISTRIBUTED 183
#define TK_CONSUMERS 184
#define TK_SUBSCRIPTIONS 185
#define TK_VNODES 186
#define TK_ALIVE 187
#define TK_VIEWS 188
#define TK_VIEW 189
#define TK_COMPACTS 190
#define TK_NORMAL 191
#define TK_CHILD 192
#define TK_LIKE 193
#define TK_TBNAME 194
#define TK_QTAGS 195
#define TK_AS 196
#define TK_SYSTEM 197
#define TK_TSMA 198
#define TK_INTERVAL 199
#define TK_RECURSIVE 200
#define TK_TSMAS 201
#define TK_FUNCTION 202
#define TK_INDEX 203
#define TK_COUNT 204
#define TK_LAST_ROW 205
#define TK_META 206
#define TK_ONLY 207
#define TK_TOPIC 208
#define TK_CONSUMER 209
#define TK_GROUP 210
#define TK_DESC 211
#define TK_DESCRIBE 212
#define TK_RESET 213
#define TK_QUERY 214
#define TK_CACHE 215
#define TK_EXPLAIN 216
#define TK_ANALYZE 217
#define TK_VERBOSE 218
#define TK_NK_BOOL 219
#define TK_RATIO 220
#define TK_NK_FLOAT 221
#define TK_OUTPUTTYPE 222
#define TK_AGGREGATE 223
#define TK_BUFSIZE 224
#define TK_LANGUAGE 225
#define TK_REPLACE 226
#define TK_STREAM 227
#define TK_INTO 228
#define TK_PAUSE 229
#define TK_RESUME 230
#define TK_PRIMARY 231
#define TK_KEY 232
#define TK_TRIGGER 233
#define TK_AT_ONCE 234
#define TK_WINDOW_CLOSE 235
#define TK_IGNORE 236
#define TK_EXPIRED 237
#define TK_FILL_HISTORY 238
#define TK_SUBTABLE 239
#define TK_UNTREATED 240
#define TK_KILL 241
#define TK_CONNECTION 242
#define TK_TRANSACTION 243
#define TK_BALANCE 244
#define TK_VGROUP 245
#define TK_LEADER 246
#define TK_MERGE 247
#define TK_REDISTRIBUTE 248
#define TK_SPLIT 249
#define TK_DELETE 250
#define TK_INSERT 251
#define TK_NK_BIN 252
#define TK_NK_HEX 253
#define TK_NULL 254
#define TK_NK_QUESTION 255
#define TK_NK_ALIAS 256
#define TK_NK_ARROW 257
#define TK_ROWTS 258
#define TK_QSTART 259
#define TK_QEND 260
#define TK_QDURATION 261
#define TK_WSTART 262
#define TK_WEND 263
#define TK_WDURATION 264
#define TK_IROWTS 265
#define TK_ISFILLED 266
#define TK_FLOW 267
#define TK_FHIGH 268
#define TK_FROWTS 269
#define TK_CAST 270
#define TK_POSITION 271
#define TK_IN 272
#define TK_FOR 273
#define TK_NOW 274
#define TK_TODAY 275
#define TK_RAND 276
#define TK_SUBSTR 277
#define TK_SUBSTRING 278
#define TK_BOTH 279
#define TK_TRAILING 280
#define TK_LEADING 281
#define TK_TIMEZONE 282
#define TK_CLIENT_VERSION 283
#define TK_SERVER_VERSION 284
#define TK_SERVER_STATUS 285
#define TK_CURRENT_USER 286
#define TK_PI 287
#define TK_CASE 288
#define TK_WHEN 289
#define TK_THEN 290
#define TK_ELSE 291
#define TK_BETWEEN 292
#define TK_IS 293
#define TK_NK_LT 294
#define TK_NK_GT 295
#define TK_NK_LE 296
#define TK_NK_GE 297
#define TK_NK_NE 298
#define TK_MATCH 299
#define TK_NMATCH 300
#define TK_CONTAINS 301
#define TK_JOIN 302
#define TK_INNER 303
#define TK_LEFT 304
#define TK_RIGHT 305
#define TK_OUTER 306
#define TK_SEMI 307
#define TK_ANTI 308
#define TK_ASOF 309
#define TK_WINDOW 310
#define TK_WINDOW_OFFSET 311
#define TK_JLIMIT 312
#define TK_SELECT 313
#define TK_NK_HINT 314
#define TK_DISTINCT 315
#define TK_WHERE 316
#define TK_PARTITION 317
#define TK_BY 318
#define TK_SESSION 319
#define TK_STATE_WINDOW 320
#define TK_EVENT_WINDOW 321
#define TK_COUNT_WINDOW 322
#define TK_ANOMALY_WINDOW 323
#define TK_SLIDING 324
#define TK_FILL 325
#define TK_VALUE 326
#define TK_VALUE_F 327
#define TK_NONE 328
#define TK_PREV 329
#define TK_NULL_F 330
#define TK_LINEAR 331
#define TK_NEXT 332
#define TK_HAVING 333
#define TK_RANGE 334
#define TK_EVERY 335
#define TK_ORDER 336
#define TK_SLIMIT 337
#define TK_SOFFSET 338
#define TK_LIMIT 339
#define TK_OFFSET 340
#define TK_ASC 341
#define TK_NULLS 342
#define TK_ABORT 343
#define TK_AFTER 344
#define TK_ATTACH 345
#define TK_BEFORE 346
#define TK_BEGIN 347
#define TK_BITAND 348
#define TK_BITNOT 349
#define TK_BITOR 350
#define TK_BLOCKS 351
#define TK_CHANGE 352
#define TK_COMMA 353
#define TK_CONCAT 354
#define TK_CONFLICT 355
#define TK_COPY 356
#define TK_DEFERRED 357
#define TK_DELIMITERS 358
#define TK_DETACH 359
#define TK_DIVIDE 360
#define TK_DOT 361
#define TK_EACH 362
#define TK_FAIL 363
#define TK_GLOB 364
#define TK_ID 365
#define TK_IMMEDIATE 366
#define TK_IMPORT 367
#define TK_INITIALLY 368
#define TK_INSTEAD 369
#define TK_ISNULL 370
#define TK_MODULES 371
#define TK_NK_BITNOT 372
#define TK_NK_SEMI 373
#define TK_NOTNULL 374
#define TK_OF 375
#define TK_PLUS 376
#define TK_PRIVILEGE 377
#define TK_RAISE 378
#define TK_RESTRICT 379
#define TK_ROW 380
#define TK_STAR 381
#define TK_STATEMENT 382
#define TK_STRICT 383
#define TK_STRING 384
#define TK_TIMES 385
#define TK_VALUES 386
#define TK_VARIABLE 387
#define TK_WAL 388
#define TK_OR 1
#define TK_AND 2
#define TK_UNION 3
#define TK_ALL 4
#define TK_MINUS 5
#define TK_EXCEPT 6
#define TK_INTERSECT 7
#define TK_NK_BITAND 8
#define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12
#define TK_NK_MINUS 13
#define TK_NK_STAR 14
#define TK_NK_SLASH 15
#define TK_NK_REM 16
#define TK_NK_CONCAT 17
#define TK_CREATE 18
#define TK_ACCOUNT 19
#define TK_NK_ID 20
#define TK_PASS 21
#define TK_NK_STRING 22
#define TK_ALTER 23
#define TK_PPS 24
#define TK_TSERIES 25
#define TK_STORAGE 26
#define TK_STREAMS 27
#define TK_QTIME 28
#define TK_DBS 29
#define TK_USERS 30
#define TK_CONNS 31
#define TK_STATE 32
#define TK_NK_COMMA 33
#define TK_HOST 34
#define TK_IS_IMPORT 35
#define TK_NK_INTEGER 36
#define TK_CREATEDB 37
#define TK_USER 38
#define TK_ENABLE 39
#define TK_SYSINFO 40
#define TK_ADD 41
#define TK_DROP 42
#define TK_GRANT 43
#define TK_ON 44
#define TK_TO 45
#define TK_REVOKE 46
#define TK_FROM 47
#define TK_SUBSCRIBE 48
#define TK_READ 49
#define TK_WRITE 50
#define TK_NK_DOT 51
#define TK_WITH 52
#define TK_ENCRYPT_KEY 53
#define TK_ANODE 54
#define TK_UPDATE 55
#define TK_ANODES 56
#define TK_DNODE 57
#define TK_PORT 58
#define TK_DNODES 59
#define TK_RESTORE 60
#define TK_NK_IPTOKEN 61
#define TK_FORCE 62
#define TK_UNSAFE 63
#define TK_CLUSTER 64
#define TK_LOCAL 65
#define TK_QNODE 66
#define TK_BNODE 67
#define TK_SNODE 68
#define TK_MNODE 69
#define TK_VNODE 70
#define TK_DATABASE 71
#define TK_USE 72
#define TK_FLUSH 73
#define TK_TRIM 74
#define TK_S3MIGRATE 75
#define TK_COMPACT 76
#define TK_IF 77
#define TK_NOT 78
#define TK_EXISTS 79
#define TK_BUFFER 80
#define TK_CACHEMODEL 81
#define TK_CACHESIZE 82
#define TK_COMP 83
#define TK_DURATION 84
#define TK_NK_VARIABLE 85
#define TK_MAXROWS 86
#define TK_MINROWS 87
#define TK_KEEP 88
#define TK_PAGES 89
#define TK_PAGESIZE 90
#define TK_TSDB_PAGESIZE 91
#define TK_PRECISION 92
#define TK_REPLICA 93
#define TK_VGROUPS 94
#define TK_SINGLE_STABLE 95
#define TK_RETENTIONS 96
#define TK_SCHEMALESS 97
#define TK_WAL_LEVEL 98
#define TK_WAL_FSYNC_PERIOD 99
#define TK_WAL_RETENTION_PERIOD 100
#define TK_WAL_RETENTION_SIZE 101
#define TK_WAL_ROLL_PERIOD 102
#define TK_WAL_SEGMENT_SIZE 103
#define TK_STT_TRIGGER 104
#define TK_TABLE_PREFIX 105
#define TK_TABLE_SUFFIX 106
#define TK_S3_CHUNKPAGES 107
#define TK_S3_KEEPLOCAL 108
#define TK_S3_COMPACT 109
#define TK_KEEP_TIME_OFFSET 110
#define TK_ENCRYPT_ALGORITHM 111
#define TK_NK_COLON 112
#define TK_BWLIMIT 113
#define TK_START 114
#define TK_TIMESTAMP 115
#define TK_END 116
#define TK_TABLE 117
#define TK_NK_LP 118
#define TK_NK_RP 119
#define TK_USING 120
#define TK_FILE 121
#define TK_STABLE 122
#define TK_COLUMN 123
#define TK_MODIFY 124
#define TK_RENAME 125
#define TK_TAG 126
#define TK_SET 127
#define TK_NK_EQ 128
#define TK_TAGS 129
#define TK_BOOL 130
#define TK_TINYINT 131
#define TK_SMALLINT 132
#define TK_INT 133
#define TK_INTEGER 134
#define TK_BIGINT 135
#define TK_FLOAT 136
#define TK_DOUBLE 137
#define TK_BINARY 138
#define TK_NCHAR 139
#define TK_UNSIGNED 140
#define TK_JSON 141
#define TK_VARCHAR 142
#define TK_MEDIUMBLOB 143
#define TK_BLOB 144
#define TK_VARBINARY 145
#define TK_GEOMETRY 146
#define TK_DECIMAL 147
#define TK_COMMENT 148
#define TK_MAX_DELAY 149
#define TK_WATERMARK 150
#define TK_ROLLUP 151
#define TK_TTL 152
#define TK_SMA 153
#define TK_DELETE_MARK 154
#define TK_FIRST 155
#define TK_LAST 156
#define TK_SHOW 157
#define TK_FULL 158
#define TK_PRIVILEGES 159
#define TK_DATABASES 160
#define TK_TABLES 161
#define TK_STABLES 162
#define TK_MNODES 163
#define TK_QNODES 164
#define TK_ARBGROUPS 165
#define TK_FUNCTIONS 166
#define TK_INDEXES 167
#define TK_ACCOUNTS 168
#define TK_APPS 169
#define TK_CONNECTIONS 170
#define TK_LICENCES 171
#define TK_GRANTS 172
#define TK_LOGS 173
#define TK_MACHINES 174
#define TK_ENCRYPTIONS 175
#define TK_QUERIES 176
#define TK_SCORES 177
#define TK_TOPICS 178
#define TK_VARIABLES 179
#define TK_BNODES 180
#define TK_SNODES 181
#define TK_TRANSACTIONS 182
#define TK_DISTRIBUTED 183
#define TK_CONSUMERS 184
#define TK_SUBSCRIPTIONS 185
#define TK_VNODES 186
#define TK_ALIVE 187
#define TK_VIEWS 188
#define TK_VIEW 189
#define TK_COMPACTS 190
#define TK_NORMAL 191
#define TK_CHILD 192
#define TK_LIKE 193
#define TK_TBNAME 194
#define TK_QTAGS 195
#define TK_AS 196
#define TK_SYSTEM 197
#define TK_TSMA 198
#define TK_INTERVAL 199
#define TK_RECURSIVE 200
#define TK_TSMAS 201
#define TK_FUNCTION 202
#define TK_INDEX 203
#define TK_COUNT 204
#define TK_LAST_ROW 205
#define TK_META 206
#define TK_ONLY 207
#define TK_TOPIC 208
#define TK_CONSUMER 209
#define TK_GROUP 210
#define TK_DESC 211
#define TK_DESCRIBE 212
#define TK_RESET 213
#define TK_QUERY 214
#define TK_CACHE 215
#define TK_EXPLAIN 216
#define TK_ANALYZE 217
#define TK_VERBOSE 218
#define TK_NK_BOOL 219
#define TK_RATIO 220
#define TK_NK_FLOAT 221
#define TK_OUTPUTTYPE 222
#define TK_AGGREGATE 223
#define TK_BUFSIZE 224
#define TK_LANGUAGE 225
#define TK_REPLACE 226
#define TK_STREAM 227
#define TK_INTO 228
#define TK_PAUSE 229
#define TK_RESUME 230
#define TK_PRIMARY 231
#define TK_KEY 232
#define TK_TRIGGER 233
#define TK_AT_ONCE 234
#define TK_WINDOW_CLOSE 235
#define TK_IGNORE 236
#define TK_EXPIRED 237
#define TK_FILL_HISTORY 238
#define TK_SUBTABLE 239
#define TK_UNTREATED 240
#define TK_KILL 241
#define TK_CONNECTION 242
#define TK_TRANSACTION 243
#define TK_BALANCE 244
#define TK_VGROUP 245
#define TK_LEADER 246
#define TK_MERGE 247
#define TK_REDISTRIBUTE 248
#define TK_SPLIT 249
#define TK_DELETE 250
#define TK_INSERT 251
#define TK_NK_BIN 252
#define TK_NK_HEX 253
#define TK_NULL 254
#define TK_NK_QUESTION 255
#define TK_NK_ALIAS 256
#define TK_NK_ARROW 257
#define TK_ROWTS 258
#define TK_QSTART 259
#define TK_QEND 260
#define TK_QDURATION 261
#define TK_WSTART 262
#define TK_WEND 263
#define TK_WDURATION 264
#define TK_IROWTS 265
#define TK_ISFILLED 266
#define TK_FLOW 267
#define TK_FHIGH 268
#define TK_FROWTS 269
#define TK_CAST 270
#define TK_POSITION 271
#define TK_IN 272
#define TK_FOR 273
#define TK_NOW 274
#define TK_TODAY 275
#define TK_RAND 276
#define TK_SUBSTR 277
#define TK_SUBSTRING 278
#define TK_BOTH 279
#define TK_TRAILING 280
#define TK_LEADING 281
#define TK_TIMEZONE 282
#define TK_CLIENT_VERSION 283
#define TK_SERVER_VERSION 284
#define TK_SERVER_STATUS 285
#define TK_CURRENT_USER 286
#define TK_PI 287
#define TK_CASE 288
#define TK_WHEN 289
#define TK_THEN 290
#define TK_ELSE 291
#define TK_BETWEEN 292
#define TK_IS 293
#define TK_NK_LT 294
#define TK_NK_GT 295
#define TK_NK_LE 296
#define TK_NK_GE 297
#define TK_NK_NE 298
#define TK_MATCH 299
#define TK_NMATCH 300
#define TK_CONTAINS 301
#define TK_JOIN 302
#define TK_INNER 303
#define TK_LEFT 304
#define TK_RIGHT 305
#define TK_OUTER 306
#define TK_SEMI 307
#define TK_ANTI 308
#define TK_ASOF 309
#define TK_WINDOW 310
#define TK_WINDOW_OFFSET 311
#define TK_JLIMIT 312
#define TK_SELECT 313
#define TK_NK_HINT 314
#define TK_DISTINCT 315
#define TK_WHERE 316
#define TK_PARTITION 317
#define TK_BY 318
#define TK_SESSION 319
#define TK_STATE_WINDOW 320
#define TK_EVENT_WINDOW 321
#define TK_COUNT_WINDOW 322
#define TK_ANOMALY_WINDOW 323
#define TK_SLIDING 324
#define TK_FILL 325
#define TK_VALUE 326
#define TK_VALUE_F 327
#define TK_NONE 328
#define TK_PREV 329
#define TK_NULL_F 330
#define TK_LINEAR 331
#define TK_NEXT 332
#define TK_HAVING 333
#define TK_RANGE 334
#define TK_EVERY 335
#define TK_ORDER 336
#define TK_SLIMIT 337
#define TK_SOFFSET 338
#define TK_LIMIT 339
#define TK_OFFSET 340
#define TK_ASC 341
#define TK_NULLS 342
#define TK_ABORT 343
#define TK_AFTER 344
#define TK_ATTACH 345
#define TK_BEFORE 346
#define TK_BEGIN 347
#define TK_BITAND 348
#define TK_BITNOT 349
#define TK_BITOR 350
#define TK_BLOCKS 351
#define TK_CHANGE 352
#define TK_COMMA 353
#define TK_CONCAT 354
#define TK_CONFLICT 355
#define TK_COPY 356
#define TK_DEFERRED 357
#define TK_DELIMITERS 358
#define TK_DETACH 359
#define TK_DIVIDE 360
#define TK_DOT 361
#define TK_EACH 362
#define TK_FAIL 363
#define TK_GLOB 364
#define TK_ID 365
#define TK_IMMEDIATE 366
#define TK_IMPORT 367
#define TK_INITIALLY 368
#define TK_INSTEAD 369
#define TK_ISNULL 370
#define TK_MODULES 371
#define TK_NK_BITNOT 372
#define TK_NK_SEMI 373
#define TK_NOTNULL 374
#define TK_OF 375
#define TK_PLUS 376
#define TK_PRIVILEGE 377
#define TK_RAISE 378
#define TK_RESTRICT 379
#define TK_ROW 380
#define TK_STAR 381
#define TK_STATEMENT 382
#define TK_STRICT 383
#define TK_STRING 384
#define TK_TIMES 385
#define TK_VALUES 386
#define TK_VARIABLE 387
#define TK_WAL 388
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

22
include/common/ttypes.h
View File

@ -238,12 +238,26 @@ typedef struct {
case TSDB_DATA_TYPE_UBIGINT: \
snprintf(_output, (int32_t)(_outputBytes), "%" PRIu64, *(uint64_t *)(_input)); \
break; \
case TSDB_DATA_TYPE_FLOAT: \
snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input)); \
case TSDB_DATA_TYPE_FLOAT: { \
int32_t n = snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input)); \
if (n >= (_outputBytes)) { \
n = snprintf(_output, (int32_t)(_outputBytes), "%.7e", *(float *)(_input)); \
if (n >= (_outputBytes)) { \
snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input)); \
} \
} \
break; \
case TSDB_DATA_TYPE_DOUBLE: \
snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input)); \
} \
case TSDB_DATA_TYPE_DOUBLE: { \
int32_t n = snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input)); \
if (n >= (_outputBytes)) { \
snprintf(_output, (int32_t)(_outputBytes), "%.15e", *(double *)(_input)); \
if (n >= (_outputBytes)) { \
snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input)); \
} \
} \
break; \
} \
case TSDB_DATA_TYPE_UINT: \
snprintf(_output, (int32_t)(_outputBytes), "%u", *(uint32_t *)(_input)); \
break; \

7
include/libs/executor/executor.h
View File

@ -151,8 +151,9 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
* @param tversion
* @return
*/
int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
int32_t* tversion, int32_t idx, bool* tbGet);
int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, int32_t dbNameBuffLen, char* tableName,
int32_t tbaleNameBuffLen, int32_t* sversion, int32_t* tversion, int32_t idx,
bool* tbGet);
/**
* The main task execution function, including query on both table and multiple tables,
@ -209,7 +210,7 @@ SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
const char* qExtractTbnameFromTask(qTaskInfo_t tinfo);
const char* qExtractTbnameFromTask(qTaskInfo_t tinfo);
void* qExtractReaderFromStreamScanner(void* scanner);

2
include/libs/function/functionMgt.h
View File

@ -26,6 +26,8 @@ extern "C" {
#define FUNC_AGGREGATE_UDF_ID 5001
#define FUNC_SCALAR_UDF_ID 5002
extern const int32_t funcMgtBuiltinsNum;
typedef enum EFunctionType {
// aggregate function
FUNCTION_TYPE_APERCENTILE = 1,

1
include/libs/nodes/cmdnodes.h
View File

@ -72,6 +72,7 @@ typedef struct SDatabaseOptions {
int8_t compressionLevel;
int8_t encryptAlgorithm;
int32_t daysPerFile;
char dnodeListStr[TSDB_DNODE_LIST_LEN];
char encryptAlgorithmStr[TSDB_ENCRYPT_ALGO_STR_LEN];
SValueNode* pDaysPerFile;
int32_t fsyncPeriod;

6
include/libs/nodes/plannodes.h
View File

@ -788,9 +788,9 @@ typedef struct SDataDeleterNode {
char tableFName[TSDB_TABLE_NAME_LEN];
char tsColName[TSDB_COL_NAME_LEN];
STimeWindow deleteTimeRange;
SNode* pAffectedRows;
SNode* pStartTs;
SNode* pEndTs;
SNode* pAffectedRows; // usless
SNode* pStartTs; // usless
SNode* pEndTs; // usless
} SDataDeleterNode;
typedef struct SSubplan {

7
include/libs/parser/parser.h
View File

@ -65,7 +65,7 @@ typedef struct SParseCsvCxt {
const char* pLastSqlPos; // the location of the last parsed sql
} SParseCsvCxt;
typedef void(*setQueryFn)(int64_t);
typedef void (*setQueryFn)(int64_t);
typedef struct SParseContext {
uint64_t requestId;
@ -147,6 +147,7 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum);
int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBuildStmtStbColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_STB** fields);
int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
@ -176,8 +177,8 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen);
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);

2
include/libs/qcom/query.h
View File

@ -364,7 +364,7 @@ void* getTaskPoolWorkerCb();
#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) \
((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || (_code) == TSDB_CODE_MND_INVALID_SCHEMA_VER)
((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || (_code) == TSDB_CODE_MND_INVALID_SCHEMA_VER || (_code) == TSDB_CODE_SCH_DATA_SRC_EP_MISS)
#define NEED_CLIENT_HANDLE_ERROR(_code) \
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))

4
include/libs/scheduler/scheduler.h
View File

@ -76,8 +76,6 @@ int32_t schedulerExecJob(SSchedulerReq* pReq, int64_t* pJob);
int32_t schedulerFetchRows(int64_t jobId, SSchedulerReq* pReq);
void schedulerFetchRowsA(int64_t job, schedulerFetchFp fp, void* param);
int32_t schedulerGetTasksStatus(int64_t job, SArray* pSub);
void schedulerStopQueryHb(void* pTrans);
@ -100,6 +98,8 @@ void schedulerFreeJob(int64_t* job, int32_t errCode);
void schedulerDestroy(void);
int32_t schedulerValidatePlan(SQueryPlan* pPlan);
#ifdef __cplusplus
}
#endif

3
include/util/taoserror.h
View File

@ -352,6 +352,8 @@ int32_t taosGetErrSize();
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
#define TSDB_CODE_MND_INVALID_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x039C)
#define TSDB_CODE_MND_INVALID_DNODE_LIST_FMT TAOS_DEF_ERROR_CODE(0, 0x039D)
#define TSDB_CODE_MND_DNODE_LIST_REPEAT TAOS_DEF_ERROR_CODE(0, 0x039E)
// mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)
@ -771,6 +773,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_SCH_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x2504)
#define TSDB_CODE_SCH_JOB_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x2505)
#define TSDB_CODE_SCH_JOB_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2506)
#define TSDB_CODE_SCH_DATA_SRC_EP_MISS TAOS_DEF_ERROR_CODE(0, 0x2507)
//parser
#define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600)

9
include/util/tcompression.h
View File

@ -152,15 +152,12 @@ int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int
// for internal usage
int32_t getWordLength(char type);
#ifdef __AVX2__
int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type);
int32_t tsDecompressFloatImpAvx2(const char *input, int32_t nelements, char *output);
int32_t tsDecompressDoubleImpAvx2(const char *input, int32_t nelements, char *output);
#endif
#ifdef __AVX512VL__
void tsDecompressTimestampAvx2(const char *input, int32_t nelements, char *output, bool bigEndian);
void tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output, bool bigEndian);
#endif
int32_t tsDecompressTimestampAvx2(const char *input, int32_t nelements, char *output, bool bigEndian);
int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output,
bool bigEndian);
/*************************************************************************
* REGULAR COMPRESSION 2

45
include/util/tdef.h
View File

@ -41,6 +41,7 @@ extern const int32_t TYPE_BYTES[21];
#define FLOAT_BYTES sizeof(float)
#define DOUBLE_BYTES sizeof(double)
#define POINTER_BYTES sizeof(void *)
#define M256_BYTES 32
#define TSDB_KEYSIZE sizeof(TSKEY)
#define TSDB_NCHAR_SIZE sizeof(TdUcs4)
@ -188,6 +189,47 @@ typedef enum EOperatorType {
OP_TYPE_ASSIGN = 200
} EOperatorType;
static const EOperatorType OPERATOR_ARRAY[] = {
OP_TYPE_ADD,
OP_TYPE_SUB,
OP_TYPE_MULTI,
OP_TYPE_DIV,
OP_TYPE_REM,
OP_TYPE_MINUS,
OP_TYPE_BIT_AND,
OP_TYPE_BIT_OR,
OP_TYPE_GREATER_THAN,
OP_TYPE_GREATER_EQUAL,
OP_TYPE_LOWER_THAN,
OP_TYPE_LOWER_EQUAL,
OP_TYPE_EQUAL,
OP_TYPE_NOT_EQUAL,
OP_TYPE_IN,
OP_TYPE_NOT_IN,
OP_TYPE_LIKE,
OP_TYPE_NOT_LIKE,
OP_TYPE_MATCH,
OP_TYPE_NMATCH,
OP_TYPE_IS_NULL,
OP_TYPE_IS_NOT_NULL,
OP_TYPE_IS_TRUE,
OP_TYPE_IS_FALSE,
OP_TYPE_IS_UNKNOWN,
OP_TYPE_IS_NOT_TRUE,
OP_TYPE_IS_NOT_FALSE,
OP_TYPE_IS_NOT_UNKNOWN,
//OP_TYPE_COMPARE_MAX_VALUE,
OP_TYPE_JSON_GET_VALUE,
OP_TYPE_JSON_CONTAINS,
OP_TYPE_ASSIGN
};
#define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR
typedef enum ELogicConditionType {
@ -411,6 +453,7 @@ typedef enum ELogicConditionType {
#define TSDB_CACHE_MODEL_LAST_ROW 1
#define TSDB_CACHE_MODEL_LAST_VALUE 2
#define TSDB_CACHE_MODEL_BOTH 3
#define TSDB_DNODE_LIST_LEN 256
#define TSDB_ENCRYPT_ALGO_STR_LEN 16
#define TSDB_ENCRYPT_ALGO_NONE_STR "none"
#define TSDB_ENCRYPT_ALGO_SM4_STR "sm4"
@ -450,7 +493,7 @@ typedef enum ELogicConditionType {
#define TSDB_MIN_S3_CHUNK_SIZE (128 * 1024)
#define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024)
#define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024)
#define TSDB_DEFAULT_S3_CHUNK_SIZE (128 * 1024)
#define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute
#define TSDB_MAX_S3_KEEP_LOCAL (365000 * 1440)
#define TSDB_DEFAULT_S3_KEEP_LOCAL (365 * 1440)

2
include/util/tlog.h
View File

@ -70,7 +70,7 @@ extern int32_t tdbDebugFlag;
extern int32_t sndDebugFlag;
extern int32_t simDebugFlag;
extern int32_t tqClientDebug;
extern int32_t tqClientDebugFlag;
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc);
void taosCloseLog();

29
packaging/tools/make_install.bat
View File

@ -12,9 +12,18 @@ if exist C:\\TDengine\\data\\dnode\\dnodeCfg.json (
rem // stop and delete service
mshta vbscript:createobject("shell.application").shellexecute("%~s0",":stop_delete","","runas",1)(window.close)
echo This might take a few moment to accomplish deleting service taosd/taosadapter ...
if exist %binary_dir%\\build\\bin\\taosadapter.exe (
echo This might take a few moment to accomplish deleting service taosd/taosadapter ...
)
if exist %binary_dir%\\build\\bin\\taoskeeper.exe (
echo This might take a few moment to accomplish deleting service taosd/taoskeeper ...
)
call :check_svc taosd
call :check_svc taosadapter
call :check_svc taoskeeper
set source_dir=%2
set source_dir=%source_dir:/=\\%
@ -46,6 +55,11 @@ if exist %binary_dir%\\test\\cfg\\taosadapter.toml (
copy %binary_dir%\\test\\cfg\\taosadapter.toml %target_dir%\\cfg\\taosadapter.toml > nul
)
)
if exist %binary_dir%\\test\\cfg\\taoskeeper.toml (
if not exist %target_dir%\\cfg\\taoskeeper.toml (
copy %binary_dir%\\test\\cfg\\taoskeeper.toml %target_dir%\\cfg\\taoskeeper.toml > nul
)
)
copy %source_dir%\\include\\client\\taos.h %target_dir%\\include > nul
copy %source_dir%\\include\\util\\taoserror.h %target_dir%\\include > nul
copy %source_dir%\\include\\libs\\function\\taosudf.h %target_dir%\\include > nul
@ -98,12 +112,15 @@ if %Enterprise% == TRUE (
copy %binary_dir%\\build\\bin\\*explorer.exe %target_dir% > nul
)
)
copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul
copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul
if exist %binary_dir%\\build\\bin\\taosadapter.exe (
copy %binary_dir%\\build\\bin\\taosadapter.exe %target_dir% > nul
)
if exist %binary_dir%\\build\\bin\\taoskeeper.exe (
copy %binary_dir%\\build\\bin\\taoskeeper.exe %target_dir% > nul
)
mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)
@ -116,6 +133,10 @@ if exist %binary_dir%\\build\\bin\\taosadapter.exe (
echo To start/stop taosAdapter with administrator privileges: %ESC%[92msc start/stop taosadapter %ESC%[0m
)
if exist %binary_dir%\\build\\bin\\taoskeeper.exe (
echo To start/stop taosKeeper with administrator privileges: %ESC%[92msc start/stop taoskeeper %ESC%[0m
)
goto :eof
:hasAdmin
@ -123,6 +144,7 @@ goto :eof
call :stop_delete
call :check_svc taosd
call :check_svc taosadapter
call :check_svc taoskeeper
if exist c:\\windows\\sysnative (
echo x86
@ -141,6 +163,7 @@ if exist c:\\windows\\sysnative (
rem // create services
sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND
sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND
sc create "taoskeeper" binPath= "C:\\TDengine\\taoskeeper.exe" start= DEMAND
set "env=HKLM\System\CurrentControlSet\Control\Session Manager\Environment"
for /f "tokens=2*" %%I in ('reg query "%env%" /v Path ^| findstr /i "\<Path\>"') do (
@ -181,6 +204,8 @@ sc stop taosd
sc delete taosd
sc stop taosadapter
sc delete taosadapter
sc stop taoskeeper
sc delete taoskeeper
exit /B 0
:check_svc

80
packaging/tools/make_install.sh
View File

@ -129,6 +129,13 @@ function kill_taosadapter() {
fi
}
function kill_taoskeeper() {
pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function kill_taosd() {
pid=$(ps -ef | grep -w ${serverName} | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
@ -155,6 +162,7 @@ function install_bin() {
${csudo}rm -f ${bin_link_dir}/${clientName} || :
${csudo}rm -f ${bin_link_dir}/${serverName} || :
${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taosdump || :
@ -169,6 +177,7 @@ function install_bin() {
[ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
[ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taoskeeper ] && ${csudo}cp -r ${binary_dir}/build/bin/taoskeeper ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || :
${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || :
@ -183,6 +192,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taoskeeper ] && ${csudo}ln -s ${install_main_dir}/bin/taoskeeper ${bin_link_dir}/taoskeeper > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo > /dev/null 2>&1 || :
@ -197,6 +207,7 @@ function install_bin() {
[ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
[ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taoskeeper ] && ${csudo}cp -r ${binary_dir}/build/bin/taoskeeper ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/*explorer ] && ${csudo}cp -r ${binary_dir}/build/bin/*explorer ${install_main_dir}/bin || :
@ -208,6 +219,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taoskeeper ] && ${csudo}ln -s ${install_main_dir}/bin/taoskeeper ${bin_link_dir}/taoskeeper > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
[ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
@ -407,6 +419,29 @@ function install_taosadapter_config() {
fi
}
function install_taoskeeper_config() {
if [ ! -f "${cfg_install_dir}/taoskeeper.toml" ]; then
${csudo}mkdir -p ${cfg_install_dir} || :
[ -f ${binary_dir}/test/cfg/taoskeeper.toml ] &&
${csudo}cp ${binary_dir}/test/cfg/taoskeeper.toml ${cfg_install_dir} &&
${csudo}cp ${binary_dir}/test/cfg/taoskeeper.toml ${cfg_dir} || :
[ -f ${cfg_install_dir}/taoskeeper.toml ] &&
${csudo}chmod 644 ${cfg_install_dir}/taoskeeper.toml || :
[ -f ${binary_dir}/test/cfg/taoskeeper.toml ] &&
${csudo}cp -f ${binary_dir}/test/cfg/taoskeeper.toml \
${cfg_install_dir}/taoskeeper.toml.${verNumber} || :
[ -f ${cfg_install_dir}/taoskeeper.toml ] &&
${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml \
${install_main_dir}/cfg/taoskeeper.toml > /dev/null 2>&1 || :
else
if [ -f "${binary_dir}/test/cfg/taoskeeper.toml" ]; then
${csudo}cp -f ${binary_dir}/test/cfg/taoskeeper.toml \
${cfg_install_dir}/taoskeeper.toml.${verNumber} || :
${csudo}cp -f ${binary_dir}/test/cfg/taoskeeper.toml ${cfg_dir} || :
fi
fi
}
function install_log() {
${csudo}rm -rf ${log_dir} || :
${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
@ -526,6 +561,15 @@ function install_taosadapter_service() {
fi
}
function install_taoskeeper_service() {
if ((${service_mod} == 0)); then
[ -f ${binary_dir}/test/cfg/taoskeeper.service ] &&
${csudo}cp ${binary_dir}/test/cfg/taoskeeper.service \
${service_config_dir}/ || :
${csudo}systemctl daemon-reload
fi
}
function install_service_on_launchctl() {
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist
@ -534,6 +578,10 @@ function install_service_on_launchctl() {
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || :
${csudo}cp ${script_dir}/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || :
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist > /dev/null 2>&1 || :
${csudo}cp ${script_dir}/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist > /dev/null 2>&1 || :
}
function install_service() {
@ -549,6 +597,7 @@ function install_service() {
install_service_on_launchctl
fi
}
function install_app() {
if [ "$osType" = "Darwin" ]; then
${csudo}rm -rf /Applications/TDengine.app &&
@ -573,6 +622,7 @@ function update_TDengine() {
elif ((${service_mod} == 1)); then
${csudo}service ${serverName} stop || :
else
kill_taoskeeper
kill_taosadapter
kill_taosd
fi
@ -591,9 +641,11 @@ function update_TDengine() {
install_service
install_taosadapter_service
install_taoskeeper_service
install_config
install_taosadapter_config
install_taoskeeper_config
echo
echo -e "\033[44;32;1m${productName} is updated successfully!${NC}"
@ -602,22 +654,33 @@ function update_TDengine() {
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}"
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml"
[ -f ${configDir}/taoskeeper.toml ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To configure Keeper ${NC}: edit ${configDir}/taoskeeper.toml"
if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
[ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}systemctl start taoskeeper ${NC}"
elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}"
[ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}service taoskeeper start${NC}"
else
if [ "$osType" != "Darwin" ]; then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}"
[ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}"
[ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: taoskeeper &${NC}"
else
echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}"
echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}"
[ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}"
[ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: sudo launchctl start com.tdengine.taoskeeper${NC}"
fi
fi
@ -643,9 +706,11 @@ function install_TDengine() {
install_service
install_taosadapter_service
install_taoskeeper_service
install_config
install_taosadapter_config
install_taoskeeper_config
# Ask if to start the service
echo
@ -654,22 +719,33 @@ function install_TDengine() {
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}"
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml"
[ -f ${configDir}/taoskeeper.toml ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To configure Keeper ${NC}: edit ${configDir}/taoskeeper.toml"
if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
[ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}systemctl start taoskeeper ${NC}"
elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}"
[ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}service taoskeeper start${NC}"
else
if [ "$osType" != "Darwin" ]; then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}"
[ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}"
[ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: taoskeeper &${NC}"
else
echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}"
echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}"
[ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}"
[ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: sudo launchctl start com.tdengine.taoskeeper${NC}"
fi
fi

9
source/client/CMakeLists.txt
View File

@ -10,6 +10,10 @@ else()
add_library(taos SHARED ${CLIENT_SRC})
endif()
if(${TD_DARWIN})
target_compile_options(taos PRIVATE -Wno-error=deprecated-non-prototype)
endif()
INCLUDE_DIRECTORIES(jni)
target_include_directories(
@ -46,6 +50,11 @@ set_target_properties(
)
add_library(taos_static STATIC ${CLIENT_SRC})
if(${TD_DARWIN})
target_compile_options(taos_static PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories(
taos_static
PUBLIC "${TD_SOURCE_DIR}/include/client"

2
source/client/inc/clientInt.h
View File

@ -58,6 +58,8 @@ enum {
#define TD_RES_TMQ_METADATA(res) (*(int8_t*)(res) == RES_TYPE__TMQ_METADATA)
#define TD_RES_TMQ_BATCH_META(res) (*(int8_t*)(res) == RES_TYPE__TMQ_BATCH_META)
#define TSC_MAX_SUBPLAN_CAPACITY_NUM 1000
typedef struct SAppInstInfo SAppInstInfo;
typedef struct {

31
source/client/inc/clientSml.h
View File

@ -92,6 +92,26 @@ extern "C" {
} \
}
#define SML_CHECK_CODE(CMD) \
code = (CMD); \
if (TSDB_CODE_SUCCESS != code) { \
lino = __LINE__; \
goto END; \
}
#define SML_CHECK_NULL(CMD) \
if (NULL == (CMD)) { \
code = terrno; \
lino = __LINE__; \
goto END; \
}
#define RETURN \
if (code != 0){ \
uError("%s failed code:%d line:%d", __FUNCTION__ , code, lino); \
} \
return code;
typedef enum {
SCHEMA_ACTION_NULL,
SCHEMA_ACTION_CREATE_STABLE,
@ -191,7 +211,6 @@ typedef struct {
cJSON *root; // for parse json
int8_t offset[OTD_JSON_FIELDS_NUM];
SSmlLineInfo *lines; // element is SSmlLineInfo
bool parseJsonByLib;
SArray *tagJsonArray;
SArray *valueJsonArray;
@ -211,13 +230,8 @@ typedef struct {
extern int64_t smlFactorNS[];
extern int64_t smlFactorS[];
typedef int32_t (*_equal_fn_sml)(const void *, const void *);
int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle);
void smlDestroyInfo(SSmlHandle *info);
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
@ -237,7 +251,7 @@ void smlDestroyTableInfo(void *para);
void freeSSmlKv(void* data);
int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseJSON(SSmlHandle *info, char *payload);
int32_t smlParseJSONExt(SSmlHandle *info, char *payload);
int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta);
bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
@ -246,7 +260,8 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);
int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements);
int32_t smlJoinMeasureTag(SSmlLineInfo *elements);
void smlBuildTsKv(SSmlKv *kv, int64_t ts);
int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv);
int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv);
int32_t smlParseEndTelnetJsonUnFormat(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv);
int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs);
static inline bool smlDoubleToInt64OverFlow(double num) {

1
source/client/inc/clientStmt2.h
View File

@ -222,6 +222,7 @@ int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags);
int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx);
int stmtGetTagFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields);
int stmtGetColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_E **fields);
int stmtGetStbColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_STB **fields);
int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums);
int stmtGetParamTbName(TAOS_STMT2 *stmt, int *nums);
int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert);

2
source/client/src/clientImpl.c
View File

@ -1250,6 +1250,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) {
int32_t code = 0;
int32_t subplanNum = 0;
if (pQuery->pRoot) {
pRequest->stmtType = pQuery->pRoot->type;
@ -1405,6 +1406,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
if (TSDB_CODE_SUCCESS == code) {
code = schedulerExecJob(&req, &pRequest->body.queryJob);
}
taosArrayDestroy(pNodeList);
} else {
qDestroyQueryPlan(pDag);

25
source/client/src/clientMain.c
View File

@ -84,7 +84,7 @@ void taos_cleanup(void) {
taosCloseRef(id);
nodesDestroyAllocatorSet();
// cleanupAppInfo();
// cleanupAppInfo();
rpcCleanup();
tscDebug("rpc cleanup");
@ -388,7 +388,6 @@ void taos_free_result(TAOS_RES *res) {
tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp);
}
taosMemoryFree(pRsp);
}
void taos_kill_query(TAOS *taos) {
@ -484,7 +483,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
return taos_print_row_with_size(str, INT32_MAX, row, fields, num_fields);
}
int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields){
int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
int32_t len = 0;
for (int i = 0; i < num_fields; ++i) {
if (i > 0 && len < size - 1) {
@ -589,7 +588,7 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD
break;
}
}
if (len < size){
if (len < size) {
str[len] = 0;
}
@ -2082,7 +2081,7 @@ int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert) {
}
int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count, TAOS_FIELD_E **fields) {
if (stmt == NULL || NULL == count) {
if (stmt == NULL || count == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
@ -2103,12 +2102,28 @@ int taos_stmt2_get_fields(TAOS_STMT2 *stmt, TAOS_FIELD_T field_type, int *count,
}
}
int taos_stmt2_get_stb_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_STB **fields) {
if (stmt == NULL || count == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return stmtGetStbColFields2(stmt, count, fields);
}
void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_E *fields) {
(void)stmt;
if (!fields) return;
taosMemoryFree(fields);
}
DLL_EXPORT void taos_stmt2_free_stb_fields(TAOS_STMT2 *stmt, TAOS_FIELD_STB *fields) {
(void)stmt;
if (!fields) return;
taosMemoryFree(fields);
}
TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt) {
if (stmt == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);

677
source/client/src/clientRawBlockWrite.c
View File

@ -52,10 +52,8 @@
#define TMQ_META_VERSION "1.0"
static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t,
SColCmprWrapper* pColCmprRow, cJSON** pJson) {
int32_t code = TSDB_CODE_SUCCESS;
@ -457,7 +455,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
cJSON* tvalue = NULL;
if (IS_VAR_DATA_TYPE(pTagVal->type)) {
char* buf = NULL;
char* buf = NULL;
int64_t bufSize = 0;
if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) {
bufSize = pTagVal->nData * 2 + 2 + 3;
@ -890,9 +888,6 @@ end:
}
static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVCreateStbReq req = {0};
SDecoder coder;
SMCreateStbReq pReq = {0};
@ -1003,9 +998,6 @@ end:
}
static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVDropStbReq req = {0};
SDecoder coder = {0};
SMDropStbReq pReq = {0};
@ -1115,9 +1107,6 @@ static void destroyCreateTbReqBatch(void* data) {
}
static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVCreateTbBatchReq req = {0};
SDecoder coder = {0};
int32_t code = TSDB_CODE_SUCCESS;
@ -1304,9 +1293,6 @@ static void destroyDropTbReqBatch(void* data) {
}
static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVDropTbBatchReq req = {0};
SDecoder coder = {0};
int32_t code = TSDB_CODE_SUCCESS;
@ -1419,9 +1405,6 @@ end:
}
static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SDeleteRes req = {0};
SDecoder coder = {0};
char sql[256] = {0};
@ -1457,9 +1440,6 @@ end:
}
static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVAlterTbReq req = {0};
SDecoder dcoder = {0};
int32_t code = TSDB_CODE_SUCCESS;
@ -1622,7 +1602,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0, false));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
@ -1682,7 +1662,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0, false));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
@ -1708,116 +1688,6 @@ static void* getRawDataFromRes(void* pRetrieve) {
return rawData;
}
static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
if (taos == NULL || data == NULL) {
SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
return TSDB_CODE_INVALID_PARA;
}
int32_t code = TSDB_CODE_SUCCESS;
SHashObj* pVgHash = NULL;
SQuery* pQuery = NULL;
SMqRspObj rspObj = {0};
SDecoder decoder = {0};
STableMeta* pTableMeta = NULL;
SRequestObj* pRequest = NULL;
RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
pRequest->syncQuery = true;
rspObj.resIter = -1;
rspObj.resType = RES_TYPE__TMQ;
int8_t dataVersion = *(int8_t*)data;
if (dataVersion >= MQ_DATA_RSP_VERSION) {
data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
dataLen -= sizeof(int8_t) + sizeof(int32_t);
}
tDecoderInit(&decoder, data, dataLen);
code = tDecodeMqDataRsp(&decoder, &rspObj.dataRsp);
if (code != 0) {
SET_ERROR_MSG("decode mq data rsp failed");
code = TSDB_CODE_INVALID_MSG;
goto end;
}
if (!pRequest->pDb) {
code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
goto end;
}
struct SCatalog* pCatalog = NULL;
RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
SRequestConnInfo conn = {0};
conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter;
conn.requestId = pRequest->requestId;
conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash);
while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
RAW_NULL_CHECK(pRetrieve);
if (!rspObj.dataRsp.withSchema) {
goto end;
}
const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
RAW_NULL_CHECK(tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
SVgroupInfo vg = {0};
RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg));
void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
if (hData == NULL) {
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
}
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
RAW_NULL_CHECK(fields);
for (int i = 0; i < pSW->nCols; i++) {
fields[i].type = pSW->pSchema[i].type;
fields[i].bytes = pSW->pSchema[i].bytes;
tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
}
void* rawData = getRawDataFromRes(pRetrieve);
char err[ERR_MSG_LEN] = {0};
code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN);
taosMemoryFree(fields);
taosMemoryFreeClear(pTableMeta);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", tbName, err);
goto end;
}
}
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code;
end:
uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
tDeleteMqDataRsp(&rspObj.dataRsp);
tDecoderClear(&decoder);
qDestroyQuery(pQuery);
destroyRequest(pRequest);
taosHashCleanup(pVgHash);
taosMemoryFreeClear(pTableMeta);
return code;
}
static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
// find schema data info
int32_t code = 0;
@ -1855,152 +1725,368 @@ end:
return code;
}
static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
if (taos == NULL || data == NULL) {
SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
return TSDB_CODE_INVALID_PARA;
typedef enum {
WRITE_RAW_INIT_START = 0,
WRITE_RAW_INIT_OK,
WRITE_RAW_INIT_FAIL,
} WRITE_RAW_INIT_STATUS;
static SHashObj* writeRawCache = NULL;
static int8_t initFlag = 0;
static int8_t initedFlag = WRITE_RAW_INIT_START;
typedef struct {
SHashObj* pVgHash;
SHashObj* pNameHash;
SHashObj* pMetaHash;
} rawCacheInfo;
typedef struct {
SVgroupInfo vgInfo;
int64_t uid;
int64_t suid;
} tbInfo;
static void tmqFreeMeta(void* data) {
STableMeta* pTableMeta = *(STableMeta**)data;
taosMemoryFree(pTableMeta);
}
static void freeRawCache(void* data) {
rawCacheInfo* pRawCache = (rawCacheInfo*)data;
taosHashCleanup(pRawCache->pMetaHash);
taosHashCleanup(pRawCache->pNameHash);
taosHashCleanup(pRawCache->pVgHash);
}
static int32_t initRawCacheHash() {
if (writeRawCache == NULL) {
writeRawCache = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (writeRawCache == NULL) {
return terrno;
}
taosHashSetFreeFp(writeRawCache, freeRawCache);
}
int32_t code = TSDB_CODE_SUCCESS;
SHashObj* pVgHash = NULL;
SQuery* pQuery = NULL;
SMqRspObj rspObj = {0};
SDecoder decoder = {0};
STableMeta* pTableMeta = NULL;
SHashObj* pCreateTbHash = NULL;
return 0;
}
SRequestObj* pRequest = NULL;
RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW) {
char* p = (char*)rawData;
// | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
// column length |
p += sizeof(int32_t);
p += sizeof(int32_t);
p += sizeof(int32_t);
p += sizeof(int32_t);
p += sizeof(int32_t);
p += sizeof(uint64_t);
int8_t* fields = p;
uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
pRequest->syncQuery = true;
rspObj.resIter = -1;
rspObj.resType = RES_TYPE__TMQ_METADATA;
if (pSW->nCols != pTableMeta->tableInfo.numOfColumns) {
return true;
}
for (int i = 0; i < pSW->nCols; i++) {
int j = 0;
for (; j < pTableMeta->tableInfo.numOfColumns; j++) {
SSchema* pColSchema = &pTableMeta->schema[j];
char* fieldName = pSW->pSchema[i].name;
int8_t dataVersion = *(int8_t*)data;
if (dataVersion >= MQ_DATA_RSP_VERSION) {
data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
dataLen -= sizeof(int8_t) + sizeof(int32_t);
if (strcmp(pColSchema->name, fieldName) == 0) {
if (*fields != pColSchema->type || *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
return true;
}
break;
}
}
fields += sizeof(int8_t) + sizeof(int32_t);
if (j == pTableMeta->tableInfo.numOfColumns) return true;
}
return false;
}
static int32_t getRawCache(SHashObj** pVgHash, SHashObj** pNameHash, SHashObj** pMetaHash, void* key) {
int32_t code = 0;
void* cacheInfo = taosHashGet(writeRawCache, &key, POINTER_BYTES);
if (cacheInfo == NULL) {
*pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(*pVgHash);
*pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
RAW_NULL_CHECK(*pNameHash);
*pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(*pMetaHash);
taosHashSetFreeFp(*pMetaHash, tmqFreeMeta);
rawCacheInfo info = {*pVgHash, *pNameHash, *pMetaHash};
RAW_RETURN_CHECK(taosHashPut(writeRawCache, &key, POINTER_BYTES, &info, sizeof(rawCacheInfo)));
} else {
rawCacheInfo* info = (rawCacheInfo*)cacheInfo;
*pVgHash = info->pVgHash;
*pNameHash = info->pNameHash;
*pMetaHash = info->pMetaHash;
}
tDecoderInit(&decoder, data, dataLen);
code = tDecodeSTaosxRsp(&decoder, &rspObj.dataRsp);
if (code != 0) {
SET_ERROR_MSG("decode mq taosx data rsp failed");
code = TSDB_CODE_INVALID_MSG;
goto end;
}
return 0;
end:
taosHashCleanup(*pMetaHash);
taosHashCleanup(*pNameHash);
taosHashCleanup(*pVgHash);
return code;
}
if (!pRequest->pDb) {
static int32_t buildRawRequest(TAOS* taos, SRequestObj** pRequest, SCatalog** pCatalog, SRequestConnInfo* conn) {
int32_t code = 0;
RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, pRequest));
(*pRequest)->syncQuery = true;
if (!(*pRequest)->pDb) {
code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
goto end;
}
struct SCatalog* pCatalog = NULL;
RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
RAW_RETURN_CHECK(catalogGetHandle((*pRequest)->pTscObj->pAppInfo->clusterId, pCatalog));
conn->pTrans = (*pRequest)->pTscObj->pAppInfo->pTransporter;
conn->requestId = (*pRequest)->requestId;
conn->requestObjRefId = (*pRequest)->self;
conn->mgmtEps = getEpSet_s(&(*pRequest)->pTscObj->pAppInfo->mgmtEp);
end:
return code;
}
typedef int32_t _raw_decode_func_(SDecoder* pDecoder, SMqDataRsp* pRsp);
static int32_t decodeRawData(SDecoder* decoder, void* data, int32_t dataLen, _raw_decode_func_ func,
SMqRspObj* rspObj) {
int8_t dataVersion = *(int8_t*)data;
if (dataVersion >= MQ_DATA_RSP_VERSION) {
data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
dataLen -= sizeof(int8_t) + sizeof(int32_t);
}
rspObj->resIter = -1;
tDecoderInit(decoder, data, dataLen);
int32_t code = func(decoder, &rspObj->dataRsp);
if (code != 0) {
SET_ERROR_MSG("decode mq taosx data rsp failed");
}
return code;
}
static int32_t processCacheMeta(SHashObj* pVgHash, SHashObj* pNameHash, SHashObj* pMetaHash,
SVCreateTbReq* pCreateReqDst, SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName,
STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry) {
int32_t code = 0;
STableMeta* pTableMeta = NULL;
tbInfo* tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
if (tmpInfo == NULL || retry > 0) {
tbInfo info = {0};
RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, conn, pName, &info.vgInfo));
if (pCreateReqDst && tmpInfo == NULL) { // change stable name to get meta
tstrncpy(pName->tname, pCreateReqDst->ctb.stbName, TSDB_TABLE_NAME_LEN);
}
RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
info.uid = pTableMeta->uid;
if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
info.suid = pTableMeta->suid;
} else {
info.suid = pTableMeta->uid;
}
code = taosHashPut(pMetaHash, &info.suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
if (code != 0) {
taosMemoryFree(pTableMeta);
goto end;
}
if (pCreateReqDst) {
pTableMeta->vgId = info.vgInfo.vgId;
pTableMeta->uid = pCreateReqDst->uid;
pCreateReqDst->ctb.suid = pTableMeta->suid;
}
RAW_RETURN_CHECK(taosHashPut(pNameHash, pName->tname, strlen(pName->tname), &info, sizeof(tbInfo)));
tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
RAW_RETURN_CHECK(
taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo)));
}
if (pTableMeta == NULL || retry > 0) {
STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, &tmpInfo->suid, LONG_BYTES);
if (pTableMetaTmp == NULL || retry > 0 || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
code = taosHashPut(pMetaHash, &tmpInfo->suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
if (code != 0) {
taosMemoryFree(pTableMeta);
goto end;
}
} else {
pTableMeta = *pTableMetaTmp;
pTableMeta->uid = tmpInfo->uid;
pTableMeta->vgId = tmpInfo->vgInfo.vgId;
}
}
*pMeta = pTableMeta;
end:
return code;
}
static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
int32_t code = TSDB_CODE_SUCCESS;
SQuery* pQuery = NULL;
SMqRspObj rspObj = {0};
SDecoder decoder = {0};
SRequestObj* pRequest = NULL;
SCatalog* pCatalog = NULL;
SRequestConnInfo conn = {0};
conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter;
conn.requestId = pRequest->requestId;
conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeMqDataRsp, &rspObj));
SHashObj* pVgHash = NULL;
SHashObj* pNameHash = NULL;
SHashObj* pMetaHash = NULL;
RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
int retry = 0;
while (1) {
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
if (!rspObj.dataRsp.withSchema) {
goto end;
}
const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
RAW_NULL_CHECK(tbName);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
RAW_NULL_CHECK(pSW);
void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
RAW_NULL_CHECK(pRetrieve);
void* rawData = getRawDataFromRes(pRetrieve);
RAW_NULL_CHECK(rawData);
uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
STableMeta* pTableMeta = NULL;
RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn, &pName, &pTableMeta, pSW,
rawData, retry));
char err[ERR_MSG_LEN] = {0};
code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
goto end;
}
}
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code;
if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) {
uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
qDestroyQuery(pQuery);
pQuery = NULL;
rspObj.resIter = -1;
continue;
}
break;
}
end:
uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
tDeleteMqDataRsp(&rspObj.dataRsp);
tDecoderClear(&decoder);
qDestroyQuery(pQuery);
destroyRequest(pRequest);
return code;
}
static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
int32_t code = TSDB_CODE_SUCCESS;
SQuery* pQuery = NULL;
SMqRspObj rspObj = {0};
SDecoder decoder = {0};
SHashObj* pCreateTbHash = NULL;
SRequestObj* pRequest = NULL;
SCatalog* pCatalog = NULL;
SRequestConnInfo conn = {0};
RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeSTaosxRsp, &rspObj));
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash);
pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pCreateTbHash);
RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
RAW_NULL_CHECK(pRetrieve);
if (!rspObj.dataRsp.withSchema) {
goto end;
}
SHashObj* pVgHash = NULL;
SHashObj* pNameHash = NULL;
SHashObj* pMetaHash = NULL;
RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
int retry = 0;
while (1) {
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
if (!rspObj.dataRsp.withSchema) {
goto end;
}
const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
if (!tbName) {
SET_ERROR_MSG("block tbname is null");
code = terrno;
goto end;
}
const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
RAW_NULL_CHECK(tbName);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
RAW_NULL_CHECK(pSW);
void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
RAW_NULL_CHECK(pRetrieve);
void* rawData = getRawDataFromRes(pRetrieve);
RAW_NULL_CHECK(rawData);
uDebug(LOG_ID_TAG " write raw metadata block tbname:%s", LOG_ID_VALUE, tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
// find schema data info
SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName));
SVgroupInfo vg = {0};
RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg));
if (pCreateReqDst) { // change stable name to get meta
tstrncpy(pName.tname, pCreateReqDst->ctb.stbName, TSDB_TABLE_NAME_LEN);
// find schema data info
SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, pName.tname, strlen(pName.tname));
STableMeta* pTableMeta = NULL;
RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, pCreateReqDst, pCatalog, &conn, &pName,
&pTableMeta, pSW, rawData, retry));
char err[ERR_MSG_LEN] = {0};
code =
rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
goto end;
}
}
RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code;
if (pCreateReqDst) {
pTableMeta->vgId = vg.vgId;
pTableMeta->uid = pCreateReqDst->uid;
pCreateReqDst->ctb.suid = pTableMeta->suid;
}
void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
if (hData == NULL) {
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
}
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
if (fields == NULL) {
SET_ERROR_MSG("calloc fields failed");
code = terrno;
goto end;
}
for (int i = 0; i < pSW->nCols; i++) {
fields[i].type = pSW->pSchema[i].type;
fields[i].bytes = pSW->pSchema[i].bytes;
tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
}
void* rawData = getRawDataFromRes(pRetrieve);
char err[ERR_MSG_LEN] = {0};
SVCreateTbReq* pCreateReqTmp = NULL;
if (pCreateReqDst) {
RAW_RETURN_CHECK(cloneSVreateTbReq(pCreateReqDst, &pCreateReqTmp));
}
code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN);
if (pCreateReqTmp != NULL) {
tdDestroySVCreateTbReq(pCreateReqTmp);
taosMemoryFree(pCreateReqTmp);
}
taosMemoryFree(fields);
taosMemoryFreeClear(pTableMeta);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", tbName, err);
goto end;
if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) {
uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
qDestroyQuery(pQuery);
pQuery = NULL;
rspObj.resIter = -1;
continue;
}
break;
}
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code;
end:
uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
tDeleteSTaosxRsp(&rspObj.dataRsp);
void* pIter = taosHashIterate(pCreateTbHash, NULL);
while (pIter) {
tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
pIter = taosHashIterate(pCreateTbHash, pIter);
}
taosHashCleanup(pCreateTbHash);
tDeleteSTaosxRsp(&rspObj.dataRsp);
tDecoderClear(&decoder);
qDestroyQuery(pQuery);
destroyRequest(pRequest);
taosHashCleanup(pVgHash);
taosMemoryFreeClear(pTableMeta);
return code;
}
@ -2076,18 +2162,18 @@ char* tmq_get_json_meta(TAOS_RES* res) {
return NULL;
}
char* string = NULL;
char* string = NULL;
SMqRspObj* rspObj = (SMqRspObj*)res;
if (TD_RES_TMQ_METADATA(res)) {
processAutoCreateTable(&rspObj->dataRsp, &string);
} else if (TD_RES_TMQ_BATCH_META(res)) {
processBatchMetaToJson(&rspObj->batchMetaRsp, &string);
} else if (TD_RES_TMQ_META(res)) {
cJSON* pJson = NULL;
cJSON* pJson = NULL;
processSimpleMeta(&rspObj->metaRsp, &pJson);
string = cJSON_PrintUnformatted(pJson);
cJSON_Delete(pJson);
} else{
} else {
uError("tmq_get_json_meta res:%d, invalid type", *(int8_t*)res);
}
@ -2098,7 +2184,7 @@ char* tmq_get_json_meta(TAOS_RES* res) {
void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); }
static int32_t getOffSetLen(const SMqDataRsp* pRsp) {
SEncoder coder = {0};
SEncoder coder = {0};
tEncoderInit(&coder, NULL, 0);
if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1;
if (tEncodeSTqOffsetVal(&coder, &pRsp->rspOffset) < 0) return -1;
@ -2108,45 +2194,45 @@ static int32_t getOffSetLen(const SMqDataRsp* pRsp) {
}
typedef int32_t __encode_func__(SEncoder* pEncoder, const SMqDataRsp* pRsp);
static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, tmq_raw_data* raw) {
int32_t len = 0;
int32_t code = 0;
SEncoder encoder = {0};
void* buf = NULL;
tEncodeSize(encodeFunc, rspObj, len, code);
if (code < 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, tmq_raw_data* raw) {
int32_t len = 0;
int32_t code = 0;
SEncoder encoder = {0};
void* buf = NULL;
tEncodeSize(encodeFunc, rspObj, len, code);
if (code < 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
len += sizeof(int8_t) + sizeof(int32_t);
buf = taosMemoryCalloc(1, len);
if (buf == NULL) {
code = terrno;
goto FAILED;
len += sizeof(int8_t) + sizeof(int32_t);
buf = taosMemoryCalloc(1, len);
if (buf == NULL) {
code = terrno;
goto FAILED;
}
tEncoderInit(&encoder, buf, len);
if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
tEncoderInit(&encoder, buf, len);
if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
int32_t offsetLen = getOffSetLen(rspObj);
if (offsetLen <= 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
int32_t offsetLen = getOffSetLen(rspObj);
if (offsetLen <= 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
if (tEncodeI32(&encoder, offsetLen) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
if (tEncodeI32(&encoder, offsetLen) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
if (encodeFunc(&encoder, rspObj) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
if (encodeFunc(&encoder, rspObj) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
tEncoderClear(&encoder);
tEncoderClear(&encoder);
raw->raw = buf;
raw->raw_len = len;
return code;
raw->raw = buf;
raw->raw_len = len;
return code;
FAILED:
tEncoderClear(&encoder);
taosMemoryFree(buf);
@ -2164,7 +2250,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
raw->raw_type = rspObj->metaRsp.resMsgType;
uDebug("tmq get raw type meta:%p", raw);
} else if (TD_RES_TMQ(res)) {
int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->dataRsp, raw);
int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->dataRsp, raw);
if (code != 0) {
uError("tmq get raw type error:%d", terrno);
return code;
@ -2199,7 +2285,31 @@ void tmq_free_raw(tmq_raw_data raw) {
(void)memset(terrMsg, 0, ERR_MSG_LEN);
}
static int32_t writeRawInit() {
while (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_START) {
int8_t old = atomic_val_compare_exchange_8(&initFlag, 0, 1);
if (old == 0) {
int32_t code = initRawCacheHash();
if (code != 0) {
uError("tmq writeRawImpl init error:%d", code);
atomic_store_8(&initedFlag, WRITE_RAW_INIT_FAIL);
return code;
}
atomic_store_8(&initedFlag, WRITE_RAW_INIT_OK);
}
}
if (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_FAIL) {
return TSDB_CODE_INTERNAL_ERROR;
}
return 0;
}
static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) {
if (writeRawInit() != 0) {
return TSDB_CODE_INTERNAL_ERROR;
}
if (type == TDMT_VND_CREATE_STB) {
return taosCreateStb(taos, buf, len);
} else if (type == TDMT_VND_ALTER_STB) {
@ -2214,10 +2324,10 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
return taosDropTable(taos, buf, len);
} else if (type == TDMT_VND_DELETE) {
return taosDeleteData(taos, buf, len);
} else if (type == RES_TYPE__TMQ) {
return tmqWriteRawDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ_METADATA) {
return tmqWriteRawMetaDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ) {
return tmqWriteRawDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ_BATCH_META) {
return tmqWriteBatchMetaDataImpl(taos, buf, len);
}
@ -2225,7 +2335,8 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
}
int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {
if (!taos) {
if (taos == NULL || raw.raw == NULL || raw.raw_len <= 0) {
SET_ERROR_MSG("taos:%p or data:%p is NULL or raw_len <= 0", taos, raw.raw);
return TSDB_CODE_INVALID_PARA;
}

1472
source/client/src/clientSml.c
View File

File diff suppressed because it is too large Load Diff

563
source/client/src/clientSmlJson.c
View File

@ -21,259 +21,10 @@
#define OTD_JSON_SUB_FIELDS_NUM 2
#define JUMP_JSON_SPACE(start) \
while (*(start)) { \
if (unlikely(*(start) > 32)) \
break; \
else \
(start)++; \
}
static int32_t smlJsonGetObj(char **payload) {
int leftBracketCnt = 0;
bool isInQuote = false;
while (**payload) {
if (**payload == '"' && *((*payload) - 1) != '\\') {
isInQuote = !isInQuote;
} else if (!isInQuote && unlikely(**payload == '{')) {
leftBracketCnt++;
(*payload)++;
continue;
} else if (!isInQuote && unlikely(**payload == '}')) {
leftBracketCnt--;
(*payload)++;
if (leftBracketCnt == 0) {
return 0;
} else if (leftBracketCnt < 0) {
return -1;
}
continue;
}
(*payload)++;
}
return -1;
}
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
int index = 0;
while (*(*start)) {
if ((*start)[0] != '"') {
(*start)++;
continue;
}
if (unlikely(index >= OTD_JSON_FIELDS_NUM)) {
uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start);
return TSDB_CODE_TSC_INVALID_JSON;
}
char *sTmp = *start;
if ((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' && (*start)[4] == 'r' && (*start)[5] == 'i' &&
(*start)[6] == 'c' && (*start)[7] == '"') {
(*start) += 8;
bool isInQuote = false;
while (*(*start)) {
if (unlikely(!isInQuote && *(*start) == '"')) {
(*start)++;
offset[index++] = *start - sTmp;
element->measure = (*start);
isInQuote = true;
continue;
}
if (unlikely(isInQuote && *(*start) == '"')) {
element->measureLen = (*start) - element->measure;
(*start)++;
break;
}
(*start)++;
}
} else if ((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' && (*start)[4] == 'e' &&
(*start)[5] == 's' && (*start)[6] == 't' && (*start)[7] == 'a' && (*start)[8] == 'm' &&
(*start)[9] == 'p' && (*start)[10] == '"') {
(*start) += 11;
bool hasColon = false;
while (*(*start)) {
if (unlikely(!hasColon && *(*start) == ':')) {
(*start)++;
JUMP_JSON_SPACE((*start))
offset[index++] = *start - sTmp;
element->timestamp = (*start);
if (*(*start) == '{') {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start);
*start = tmp;
}
break;
}
hasColon = true;
continue;
}
if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) {
element->timestampLen = (*start) - element->timestamp;
break;
}
(*start)++;
}
} else if ((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' && (*start)[4] == 'u' &&
(*start)[5] == 'e' && (*start)[6] == '"') {
(*start) += 7;
bool hasColon = false;
while (*(*start)) {
if (unlikely(!hasColon && *(*start) == ':')) {
(*start)++;
JUMP_JSON_SPACE((*start))
offset[index++] = *start - sTmp;
element->cols = (*start);
if (*(*start) == '{') {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start);
*start = tmp;
}
break;
}
hasColon = true;
continue;
}
if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) {
element->colsLen = (*start) - element->cols;
break;
}
(*start)++;
}
} else if ((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' && (*start)[4] == 's' &&
(*start)[5] == '"') {
(*start) += 6;
while (*(*start)) {
if (unlikely(*(*start) == ':')) {
(*start)++;
JUMP_JSON_SPACE((*start))
offset[index++] = *start - sTmp;
element->tags = (*start);
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start);
*start = tmp;
}
break;
}
(*start)++;
}
}
if (*(*start) == '\0') {
break;
}
if (*(*start) == '}') {
(*start)++;
break;
}
(*start)++;
}
if (unlikely(index != OTD_JSON_FIELDS_NUM) || element->tags == NULL || element->cols == NULL ||
element->measure == NULL || element->timestamp == NULL) {
uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM);
return TSDB_CODE_TSC_INVALID_JSON;
}
return 0;
}
int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
int index = 0;
while (*(*start)) {
if ((*start)[0] != '"') {
(*start)++;
continue;
}
if (unlikely(index >= OTD_JSON_FIELDS_NUM)) {
uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start);
return TSDB_CODE_TSC_INVALID_JSON;
}
if ((*start)[1] == 'm') {
(*start) += offset[index++];
element->measure = *start;
while (*(*start)) {
if (unlikely(*(*start) == '"')) {
element->measureLen = (*start) - element->measure;
(*start)++;
break;
}
(*start)++;
}
} else if ((*start)[1] == 't' && (*start)[2] == 'i') {
(*start) += offset[index++];
element->timestamp = *start;
if (*(*start) == '{') {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start);
*start = tmp;
}
} else {
while (*(*start)) {
if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) {
element->timestampLen = (*start) - element->timestamp;
break;
}
(*start)++;
}
}
} else if ((*start)[1] == 'v') {
(*start) += offset[index++];
element->cols = *start;
if (*(*start) == '{') {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start);
*start = tmp;
}
} else {
while (*(*start)) {
if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) {
element->colsLen = (*start) - element->cols;
break;
}
(*start)++;
}
}
} else if ((*start)[1] == 't' && (*start)[2] == 'a') {
(*start) += offset[index++];
element->tags = (*start);
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start);
*start = tmp;
}
}
if (*(*start) == '}') {
(*start)++;
break;
}
(*start)++;
}
if (unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) {
uError("elements != %d", OTD_JSON_FIELDS_NUM);
return TSDB_CODE_TSC_INVALID_JSON;
}
return TSDB_CODE_SUCCESS;
}
static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) {
elements->measureLen = strlen(metric->valuestring);
if (IS_INVALID_TABLE_LEN(elements->measureLen)) {
uError("OTD:0x%" PRIx64 " Metric length is 0 or large than 192", info->id);
uError("SML:0x%" PRIx64 " Metric length is 0 or large than 192", info->id);
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
@ -293,7 +44,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) {
child = child->next;
}
if (*marks[i] == NULL) {
uError("smlGetJsonElements error, not find mark:%d:%s", i, jsonName[i]);
uError("SML %s error, not find mark:%d:%s", __FUNCTION__, i, jsonName[i]);
return TSDB_CODE_TSC_INVALID_JSON;
}
}
@ -302,7 +53,7 @@ static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) {
static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) {
if (strcasecmp(typeStr, "bool") != 0) {
uError("OTD:invalid type(%s) for JSON Bool", typeStr);
uError("SML:invalid type(%s) for JSON Bool", typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
pVal->type = TSDB_DATA_TYPE_BOOL;
@ -316,7 +67,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) {
// tinyint
if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) {
if (!IS_VALID_TINYINT(value->valuedouble)) {
uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble);
uError("SML:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_TINYINT;
@ -327,7 +78,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) {
// smallint
if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) {
if (!IS_VALID_SMALLINT(value->valuedouble)) {
uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble);
uError("SML:JSON value(%f) cannot fit in type(smallint)", value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_SMALLINT;
@ -338,7 +89,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) {
// int
if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) {
if (!IS_VALID_INT(value->valuedouble)) {
uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble);
uError("SML:JSON value(%f) cannot fit in type(int)", value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_INT;
@ -362,7 +113,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) {
// float
if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) {
if (!IS_VALID_FLOAT(value->valuedouble)) {
uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble);
uError("SML:JSON value(%f) cannot fit in type(float)", value->valuedouble);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pVal->type = TSDB_DATA_TYPE_FLOAT;
@ -379,7 +130,7 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) {
}
// if reach here means type is unsupported
uError("OTD:invalid type(%s) for JSON Number", typeStr);
uError("SML:invalid type(%s) for JSON Number", typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
@ -391,7 +142,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) {
} else if (strcasecmp(typeStr, "nchar") == 0) {
pVal->type = TSDB_DATA_TYPE_NCHAR;
} else {
uError("OTD:invalid type(%s) for JSON String", typeStr);
uError("SML:invalid type(%s) for JSON String", typeStr);
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
pVal->length = strlen(value->valuestring);
@ -474,7 +225,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
case cJSON_String: {
int32_t ret = smlConvertJSONString(kv, "binary", root);
if (ret != TSDB_CODE_SUCCESS) {
uError("OTD:Failed to parse binary value from JSON Obj");
uError("SML:Failed to parse binary value from JSON Obj");
return ret;
}
break;
@ -482,7 +233,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
case cJSON_Object: {
int32_t ret = smlParseValueFromJSONObj(root, kv);
if (ret != TSDB_CODE_SUCCESS) {
uError("OTD:Failed to parse value from JSON Obj");
uError("SML:Failed to parse value from JSON Obj");
return ret;
}
break;
@ -511,7 +262,7 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
}
size_t keyLen = strlen(tag->string);
if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
uError("OTD:Tag key length is 0 or too large than 64");
uError("SML:Tag key length is 0 or too large than 64");
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
@ -539,28 +290,24 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
}
static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
int32_t ret = 0;
if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
elements->measureTag = info->preLine.measureTag;
return TSDB_CODE_SUCCESS;
}
int32_t code = 0;
int32_t lino = 0;
if(info->dataFormat){
ret = smlProcessSuperTable(info, elements);
if(ret != 0){
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
}
ret = smlProcessTagJson(info, tags);
if(ret != 0){
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
ret = smlJoinMeasureTag(elements);
if(ret != 0){
return ret;
SML_CHECK_CODE(smlProcessSuperTable(info, elements));
}
SML_CHECK_CODE(smlProcessTagJson(info, tags));
SML_CHECK_CODE(smlJoinMeasureTag(elements));
return smlProcessChildTable(info, elements);
END:
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
RETURN
}
static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) {
@ -678,7 +425,8 @@ static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) {
}
static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) {
int32_t ret = TSDB_CODE_SUCCESS;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
cJSON *metricJson = NULL;
cJSON *tsJson = NULL;
@ -688,57 +436,27 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
int32_t size = cJSON_GetArraySize(root);
// outmost json fields has to be exactly 4
if (size != OTD_JSON_FIELDS_NUM) {
uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size);
uError("SML:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size);
return TSDB_CODE_TSC_INVALID_JSON;
}
cJSON **marks[OTD_JSON_FIELDS_NUM] = {&metricJson, &tsJson, &valueJson, &tagsJson};
ret = smlGetJsonElements(root, marks);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
return ret;
}
SML_CHECK_CODE(smlGetJsonElements(root, marks));
// Parse metric
ret = smlParseMetricFromJSON(info, metricJson, elements);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id);
return ret;
}
SML_CHECK_CODE(smlParseMetricFromJSON(info, metricJson, elements));
// Parse metric value
SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN};
ret = smlParseValueFromJSON(valueJson, &kv);
if (unlikely(ret)) {
uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id);
return ret;
}
SML_CHECK_CODE(smlParseValueFromJSON(valueJson, &kv));
// Parse tags
bool needFree = info->dataFormat;
elements->tags = cJSON_PrintUnformatted(tagsJson);
if (elements->tags == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
elements->tagsLen = strlen(elements->tags);
if (is_same_child_table_telnet(elements, &info->preLine) != 0) {
ret = smlParseTagsFromJSON(info, tagsJson, elements);
if (unlikely(ret)) {
uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id);
taosMemoryFree(elements->tags);
elements->tags = NULL;
return ret;
}
} else {
elements->measureTag = info->preLine.measureTag;
}
SML_CHECK_NULL(elements->tags);
if (needFree) {
taosMemoryFree(elements->tags);
elements->tags = NULL;
}
elements->tagsLen = strlen(elements->tags);
SML_CHECK_CODE(smlParseTagsFromJSON(info, tagsJson, elements));
if (unlikely(info->reRun)) {
return TSDB_CODE_SUCCESS;
goto END;
}
// Parse timestamp
@ -747,29 +465,34 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
if (unlikely(ts < 0)) {
char* tmp = cJSON_PrintUnformatted(tsJson);
if (tmp == NULL) {
uError("cJSON_PrintUnformatted failed since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts);
uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %" PRId64, info->id, info->msgBuf.buf, ts);
} else {
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts);
uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts);
taosMemoryFree(tmp);
}
return TSDB_CODE_INVALID_TIMESTAMP;
SML_CHECK_CODE(TSDB_CODE_INVALID_TIMESTAMP);
}
SSmlKv kvTs = {0};
smlBuildTsKv(&kvTs, ts);
if (info->dataFormat){
code = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv);
} else {
code = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv);
}
SML_CHECK_CODE(code);
taosMemoryFreeClear(info->preLine.tags);
info->preLine = *elements;
elements->tags = NULL;
return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
END:
taosMemoryFree(elements->tags);
RETURN
}
static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
int32_t payloadNum = 0;
int32_t ret = TSDB_CODE_SUCCESS;
if (unlikely(payload == NULL)) {
uError("SML:0x%" PRIx64 " empty JSON Payload", info->id);
return TSDB_CODE_TSC_INVALID_JSON;
}
info->root = cJSON_Parse(payload);
if (unlikely(info->root == NULL)) {
uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload);
@ -782,27 +505,11 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
} else if (cJSON_IsObject(info->root)) {
payloadNum = 1;
} else {
uError("SML:0x%" PRIx64 " Invalid JSON Payload 3:%s", info->id, payload);
uError("SML:0x%" PRIx64 " Invalid JSON type:%s", info->id, payload);
return TSDB_CODE_TSC_INVALID_JSON;
}
if (unlikely(info->lines != NULL)) {
for (int i = 0; i < info->lineNum; i++) {
taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
}
taosMemoryFree(info->lines);
info->lines = NULL;
}
info->lineNum = payloadNum;
info->dataFormat = true;
ret = smlClearForRerun(info);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
info->parseJsonByLib = true;
cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child;
int cnt = 0;
@ -811,6 +518,7 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
if (info->dataFormat) {
SSmlLineInfo element = {0};
ret = smlParseJSONStringExt(info, dataPoint, &element);
if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag);
} else {
ret = smlParseJSONStringExt(info, dataPoint, info->lines + cnt);
}
@ -836,164 +544,3 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) {
int32_t ret = TSDB_CODE_SUCCESS;
if (info->offset[0] == 0) {
ret = smlJsonParseObjFirst(start, elements, info->offset);
} else {
ret = smlJsonParseObj(start, elements, info->offset);
}
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS;
if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) {
smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL);
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen};
if (unlikely(elements->colsLen == 0)) {
uError("SML:colsLen == 0");
return TSDB_CODE_TSC_INVALID_VALUE;
} else if (unlikely(elements->cols[0] == '{')) {
char tmp = elements->cols[elements->colsLen];
elements->cols[elements->colsLen] = '\0';
cJSON *valueJson = cJSON_Parse(elements->cols);
if (unlikely(valueJson == NULL)) {
uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_JSON;
}
if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){
cJSON_Delete(valueJson);
elements->cols[elements->colsLen] = tmp;
return terrno;
}
ret = smlParseValueFromJSONObj(valueJson, &kv);
if (ret != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_VALUE;
}
elements->cols[elements->colsLen] = tmp;
} else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols);
return TSDB_CODE_TSC_INVALID_VALUE;
}
// Parse tags
if (is_same_child_table_telnet(elements, &info->preLine) != 0) {
char tmp = *(elements->tags + elements->tagsLen);
*(elements->tags + elements->tagsLen) = 0;
cJSON *tagsJson = cJSON_Parse(elements->tags);
*(elements->tags + elements->tagsLen) = tmp;
if (unlikely(tagsJson == NULL)) {
uError("SML:0x%" PRIx64 " parse json tag failed:%s", info->id, elements->tags);
return TSDB_CODE_TSC_INVALID_JSON;
}
if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){
cJSON_Delete(tagsJson);
uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id);
return terrno;
}
ret = smlParseTagsFromJSON(info, tagsJson, elements);
if (unlikely(ret)) {
uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id);
return ret;
}
} else {
elements->measureTag = info->preLine.measureTag;
}
if (unlikely(info->reRun)) {
return TSDB_CODE_SUCCESS;
}
// Parse timestamp
// notice!!! put ts back to tag to ensure get meta->precision
int64_t ts = 0;
if (unlikely(elements->timestampLen == 0)) {
uError("OTD:0x%" PRIx64 " elements->timestampLen == 0", info->id);
return TSDB_CODE_INVALID_TIMESTAMP;
} else if (elements->timestamp[0] == '{') {
char tmp = elements->timestamp[elements->timestampLen];
elements->timestamp[elements->timestampLen] = '\0';
cJSON *tsJson = cJSON_Parse(elements->timestamp);
ts = smlParseTSFromJSON(info, tsJson);
if (unlikely(ts < 0)) {
uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload:%s", info->id, elements->timestamp);
elements->timestamp[elements->timestampLen] = tmp;
cJSON_Delete(tsJson);
return TSDB_CODE_INVALID_TIMESTAMP;
}
elements->timestamp[elements->timestampLen] = tmp;
cJSON_Delete(tsJson);
} else {
ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen);
if (unlikely(ts < 0)) {
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id);
return TSDB_CODE_INVALID_TIMESTAMP;
}
}
SSmlKv kvTs = {0};
smlBuildTsKv(&kvTs, ts);
return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
}
int32_t smlParseJSON(SSmlHandle *info, char *payload) {
int32_t payloadNum = 1 << 15;
int32_t ret = TSDB_CODE_SUCCESS;
uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload);
int cnt = 0;
char *dataPointStart = payload;
while (1) {
if (info->dataFormat) {
SSmlLineInfo element = {0};
ret = smlParseJSONString(info, &dataPointStart, &element);
if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag);
} else {
if (cnt >= payloadNum) {
payloadNum = payloadNum << 1;
void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo));
if (tmp == NULL) {
ret = terrno;
return ret;
}
info->lines = (SSmlLineInfo *)tmp;
(void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo));
}
ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt);
if ((info->lines + cnt)->measure == NULL) break;
}
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
uError("SML:0x%" PRIx64 " Invalid JSON Payload 1:%s", info->id, payload);
return smlParseJSONExt(info, payload);
}
if (unlikely(info->reRun)) {
cnt = 0;
dataPointStart = payload;
info->lineNum = payloadNum;
ret = smlClearForRerun(info);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
continue;
}
cnt++;
if (*dataPointStart == '\0') break;
}
info->lineNum = cnt;
return TSDB_CODE_SUCCESS;
}

58
source/client/src/clientSmlLine.c
View File

@ -63,7 +63,7 @@ static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t le
int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision);
if (unlikely(ts == -1)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data);
smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid timestamp", data);
return TSDB_CODE_SML_INVALID_DATA;
}
return ts;
@ -84,7 +84,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
}
if (pVal->value[0] == 'l' || pVal->value[0] == 'L') { // nchar
if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) {
if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') {
pVal->type = TSDB_DATA_TYPE_NCHAR;
pVal->length -= NCHAR_ADD_LEN;
if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
@ -97,7 +97,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
}
if (pVal->value[0] == 'g' || pVal->value[0] == 'G') { // geometry
if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= sizeof("POINT")+3) {
if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') {
int32_t code = initCtxGeomFromText();
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -124,7 +124,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
}
if (pVal->value[0] == 'b' || pVal->value[0] == 'B') { // varbinary
if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) {
if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') {
pVal->type = TSDB_DATA_TYPE_VARBINARY;
if(isHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){
if(!isValidateHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){
@ -298,7 +298,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
}
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
return TSDB_CODE_TSC_INVALID_JSON;
return TSDB_CODE_SML_INVALID_DATA;
}
cnt++;
@ -311,31 +311,24 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
}
static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
int32_t code = 0;
int32_t lino = 0;
bool isSameCTable = IS_SAME_CHILD_TABLE;
if(isSameCTable){
return TSDB_CODE_SUCCESS;
}
int32_t ret = 0;
if(info->dataFormat){
ret = smlProcessSuperTable(info, elements);
if(ret != 0){
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
SML_CHECK_CODE(smlProcessSuperTable(info, elements));
}
ret = smlProcessTagLine(info, sql, sqlEnd);
if(ret != 0){
if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
SML_CHECK_CODE(smlProcessTagLine(info, sql, sqlEnd));
return smlProcessChildTable(info, elements);
END:
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
RETURN
}
static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) {
@ -353,7 +346,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
const char *escapeChar = NULL;
while (*sql < sqlEnd) {
if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid data", *sql);
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(IS_EQUAL(*sql,escapeChar))) {
@ -370,7 +363,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
}
if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid key or key is too long than 64", key);
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
@ -404,18 +397,18 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
valueLen = *sql - value;
if (unlikely(quoteNum != 0 && quoteNum != 2)) {
smlBuildInvalidDataMsg(&info->msgBuf, "unbalanced quotes", value);
smlBuildInvalidDataMsg(&info->msgBuf, "SML line unbalanced quotes", value);
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(valueLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
smlBuildInvalidDataMsg(&info->msgBuf, "SML line invalid value", value);
return TSDB_CODE_SML_INVALID_DATA;
}
SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen};
int32_t ret = smlParseValue(&kv, &info->msgBuf);
if (ret != TSDB_CODE_SUCCESS) {
smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value);
uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret);
return ret;
}
@ -437,11 +430,6 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
}
(void)memcpy(tmp, kv.value, kv.length);
PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
if(kv.type == TSDB_DATA_TYPE_GEOMETRY) {
uError("SML:0x%" PRIx64 " smlParseColLine error, invalid GEOMETRY type.", info->id);
taosMemoryFree((void*)kv.value);
return TSDB_CODE_TSC_INVALID_VALUE;
}
if(kv.type == TSDB_DATA_TYPE_VARBINARY){
taosMemoryFree((void*)kv.value);
}
@ -510,7 +498,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
}
elements->measureLen = sql - elements->measure;
if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL);
smlBuildInvalidDataMsg(&info->msgBuf, "SML line measure is empty or too large than 192", NULL);
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
@ -557,7 +545,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
elements->colsLen = sql - elements->cols;
if (unlikely(elements->colsLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL);
smlBuildInvalidDataMsg(&info->msgBuf, "SML line cols is empty", NULL);
return TSDB_CODE_SML_INVALID_DATA;
}
@ -574,7 +562,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
int64_t ts = smlParseInfluxTime(info, elements->timestamp, elements->timestampLen);
if (unlikely(ts <= 0)) {
uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts);
uError("SML:0x%" PRIx64 " %s error:%" PRId64, info->id, __FUNCTION__, ts);
return TSDB_CODE_INVALID_TIMESTAMP;
}

57
source/client/src/clientSmlTelnet.c
View File

@ -148,31 +148,21 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
return TSDB_CODE_SUCCESS;
}
int32_t ret = 0;
int32_t code = 0;
int32_t lino = 0;
if(info->dataFormat){
ret = smlProcessSuperTable(info, elements);
if(ret != 0){
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
SML_CHECK_CODE(smlProcessSuperTable(info, elements));
}
SML_CHECK_CODE(smlProcessTagTelnet(info, data, sqlEnd));
SML_CHECK_CODE(smlJoinMeasureTag(elements));
ret = smlProcessTagTelnet(info, data, sqlEnd);
if(ret != 0){
if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
code = smlProcessChildTable(info, elements);
END:
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
ret = smlJoinMeasureTag(elements);
if(ret != 0){
return ret;
}
return smlProcessChildTable(info, elements);
RETURN
}
// format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
@ -182,14 +172,14 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
// parse metric
smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen);
if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid measure", sql);
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
// parse timestamp
smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen);
if (unlikely(!elements->timestamp || elements->timestampLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid timestamp", sql);
return TSDB_CODE_SML_INVALID_DATA;
}
@ -199,19 +189,21 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
}
int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen);
if (unlikely(ts < 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet parse timestamp failed", sql);
return TSDB_CODE_INVALID_TIMESTAMP;
}
// parse value
smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen);
if (unlikely(!elements->cols || elements->colsLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql);
smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid value", sql);
return TSDB_CODE_TSC_INVALID_VALUE;
}
SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen};
if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) {
int ret = smlParseValue(&kv, &info->msgBuf);
if (ret != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret);
return TSDB_CODE_TSC_INVALID_VALUE;
}
@ -220,11 +212,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
elements->tags = sql;
elements->tagsLen = sqlEnd - sql;
if (unlikely(!elements->tags || elements->tagsLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql);
smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid tag value", sql);
return TSDB_CODE_TSC_INVALID_VALUE;
}
int ret = smlParseTelnetTags(info, sql, sqlEnd, elements);
ret = smlParseTelnetTags(info, sql, sqlEnd, elements);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
return ret;
}
@ -239,5 +231,12 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
}
return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
if (info->dataFormat){
ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv);
} else {
ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv);
}
info->preLine = *elements;
return ret;
}

50
source/client/src/clientStmt2.c
View File

@ -1068,6 +1068,34 @@ static int stmtFetchColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_E
return TSDB_CODE_SUCCESS;
}
static int stmtFetchStbColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIELD_STB** fields) {
if (pStmt->errCode != TSDB_CODE_SUCCESS) {
return pStmt->errCode;
}
if (STMT_TYPE_QUERY == pStmt->sql.type) {
tscError("invalid operation to get query column fileds");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
}
STableDataCxt** pDataBlock = NULL;
if (pStmt->sql.stbInterlaceMode) {
pDataBlock = &pStmt->sql.siInfo.pDataCtx;
} else {
pDataBlock =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERR_RET(TSDB_CODE_APP_ERROR);
}
}
STMT_ERR_RET(qBuildStmtStbColFields(*pDataBlock, fieldNum, fields));
return TSDB_CODE_SUCCESS;
}
/*
SArray* stmtGetFreeCol(STscStmt2* pStmt, int32_t* idx) {
while (true) {
@ -1808,7 +1836,7 @@ _return:
return code;
}
int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) {
int stmtParseColFields2(TAOS_STMT2* stmt) {
int32_t code = 0;
STscStmt2* pStmt = (STscStmt2*)stmt;
int32_t preCode = pStmt->errCode;
@ -1842,8 +1870,6 @@ int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) {
STMT_ERRI_JRET(stmtParseSql(pStmt));
}
STMT_ERRI_JRET(stmtFetchColFields2(stmt, nums, fields));
_return:
pStmt->errCode = preCode;
@ -1851,6 +1877,24 @@ _return:
return code;
}
int stmtGetColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = stmtParseColFields2(stmt);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
return stmtFetchColFields2(stmt, nums, fields);
}
int stmtGetStbColFields2(TAOS_STMT2* stmt, int* nums, TAOS_FIELD_STB** fields) {
int32_t code = stmtParseColFields2(stmt);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
return stmtFetchStbColFields2(stmt, nums, fields);
}
int stmtGetParamNum2(TAOS_STMT2* stmt, int* nums) {
STscStmt2* pStmt = (STscStmt2*)stmt;

29
source/client/src/clientTmq.c
View File

@ -24,12 +24,9 @@
#include "tref.h"
#include "ttimer.h"
#define tqFatalC(...) do { if (cDebugFlag & DEBUG_FATAL || tqClientDebug) { taosPrintLog("TQ FATAL ", DEBUG_FATAL, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebug) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqWarnC(...) do { if (cDebugFlag & DEBUG_WARN || tqClientDebug) { taosPrintLog("TQ WARN ", DEBUG_WARN, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqInfoC(...) do { if (cDebugFlag & DEBUG_INFO || tqClientDebug) { taosPrintLog("TQ ", DEBUG_INFO, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebug) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqTraceC(...) do { if (cDebugFlag & DEBUG_TRACE || tqClientDebug) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }} while(0)
#define tqInfoC(...) do { if (cDebugFlag & DEBUG_INFO || tqClientDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", DEBUG_INFO, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }} while(0)
#define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }} while(0)
#define EMPTY_BLOCK_POLL_IDLE_DURATION 10
#define DEFAULT_AUTO_COMMIT_INTERVAL 5000
@ -831,8 +828,8 @@ static int32_t innerCommitAll(tmq_t* tmq, SMqCommitCbParamSet* pParamSet){
}
code = innerCommit(tmq, pTopic->topicName, &pVg->offsetInfo.endOffset, pVg, pParamSet);
if (code != 0){
tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d",
if (code != 0 && code != TSDB_CODE_TMQ_SAME_COMMITTED_VALUE){
tqErrorC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d",
tmq->consumerId, pTopic->topicName, pVg->vgId, tstrerror(code), pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups);
}
}
@ -857,7 +854,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
return;
}
code = innerCommitAll(tmq, pParamSet);
if (code != 0){
if (code != 0 && code != TSDB_CODE_TMQ_SAME_COMMITTED_VALUE){
tqErrorC("consumer:0x%" PRIx64 " innerCommitAll failed, code:%s", tmq->consumerId, tstrerror(code));
}
@ -957,7 +954,8 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
}
}
tqClientDebug = rsp.debugFlag;
tqClientDebugFlag = rsp.debugFlag;
tDestroySMqHbRsp(&rsp);
END:
@ -978,6 +976,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
req.consumerId = tmq->consumerId;
req.epoch = tmq->epoch;
req.pollFlag = atomic_load_8(&tmq->pollFlag);
tqDebugC("consumer:0x%" PRIx64 " send heartbeat, pollFlag:%d", tmq->consumerId, req.pollFlag);
req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
if (req.topics == NULL) {
goto END;
@ -1063,7 +1062,7 @@ END:
tDestroySMqHbReq(&req);
if (tmrId != NULL) {
bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer);
tqDebugC("reset timer fo tmq hb:%d", ret);
tqDebugC("consumer:0x%" PRIx64 " reset timer for tmq heartbeat:%d, pollFlag:%d", tmq->consumerId, ret, tmq->pollFlag);
}
int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId);
if (ret != 0){
@ -1269,7 +1268,9 @@ static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
}
if (code != TSDB_CODE_SUCCESS) {
tqErrorC("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code));
if (code != TSDB_CODE_MND_CONSUMER_NOT_READY){
tqErrorC("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code));
}
goto END;
}
@ -1422,7 +1423,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
tqDebugC("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
bool ret = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->epTimer);
tqDebugC("reset timer fo tmq ask ep:%d", ret);
tqDebugC("reset timer for tmq ask ep:%d", ret);
} else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
tmq_commit_cb* pCallbackFn = (pTmq->commitCb != NULL) ? pTmq->commitCb : defaultCommitCbFn;
asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
@ -1430,7 +1431,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
pTmq->autoCommitInterval / 1000.0);
bool ret = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->commitTimer);
tqDebugC("reset timer fo commit:%d", ret);
tqDebugC("reset timer for commit:%d", ret);
} else {
tqErrorC("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
}

107
source/client/test/smlTest.cpp
View File

@ -68,6 +68,15 @@ TEST(testCase, smlParseInfluxString_Test) {
taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 0 false
tmp = "st,t1=3 c3=\"";
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 2 false
tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
(void)memcpy(sql, tmp, strlen(tmp) + 1);
@ -591,6 +600,104 @@ TEST(testCase, smlParseTelnetLine_Test) {
// smlDestroyInfo(info);
//}
bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) {
const char *pVal = kvVal->value;
int32_t len = kvVal->length;
char *endptr = NULL;
double result = taosStr2Double(pVal, &endptr);
if (pVal == endptr) {
smlBuildInvalidDataMsg(msg, "invalid data", pVal);
return false;
}
int32_t left = len - (endptr - pVal);
if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) {
kvVal->type = TSDB_DATA_TYPE_DOUBLE;
kvVal->d = result;
} else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) {
if (!IS_VALID_FLOAT(result)) {
smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal);
return false;
}
kvVal->type = TSDB_DATA_TYPE_FLOAT;
kvVal->f = (float)result;
} else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) {
if (smlDoubleToInt64OverFlow(result)) {
errno = 0;
int64_t tmp = taosStr2Int64(pVal, &endptr, 10);
if (errno == ERANGE) {
smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal);
return false;
}
kvVal->type = TSDB_DATA_TYPE_BIGINT;
kvVal->i = tmp;
return true;
}
kvVal->type = TSDB_DATA_TYPE_BIGINT;
kvVal->i = (int64_t)result;
} else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) {
if (result >= (double)UINT64_MAX || result < 0) {
errno = 0;
uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);
if (errno == ERANGE || result < 0) {
smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal);
return false;
}
kvVal->type = TSDB_DATA_TYPE_UBIGINT;
kvVal->u = tmp;
return true;
}
kvVal->type = TSDB_DATA_TYPE_UBIGINT;
kvVal->u = result;
} else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) {
if (!IS_VALID_INT(result)) {
smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal);
return false;
}
kvVal->type = TSDB_DATA_TYPE_INT;
kvVal->i = result;
} else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) {
if (!IS_VALID_UINT(result)) {
smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal);
return false;
}
kvVal->type = TSDB_DATA_TYPE_UINT;
kvVal->u = result;
} else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) {
if (!IS_VALID_SMALLINT(result)) {
smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal);
return false;
}
kvVal->type = TSDB_DATA_TYPE_SMALLINT;
kvVal->i = result;
} else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) {
if (!IS_VALID_USMALLINT(result)) {
smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal);
return false;
}
kvVal->type = TSDB_DATA_TYPE_USMALLINT;
kvVal->u = result;
} else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) {
if (!IS_VALID_TINYINT(result)) {
smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal);
return false;
}
kvVal->type = TSDB_DATA_TYPE_TINYINT;
kvVal->i = result;
} else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) {
if (!IS_VALID_UTINYINT(result)) {
smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal);
return false;
}
kvVal->type = TSDB_DATA_TYPE_UTINYINT;
kvVal->u = result;
} else {
smlBuildInvalidDataMsg(msg, "invalid data", pVal);
return false;
}
return true;
}
TEST(testCase, smlParseNumber_performance_Test) {
char msg[256] = {0};
SSmlMsgBuf msgBuf;

2
source/common/src/systable.c
View File

@ -118,7 +118,7 @@ static const SSysDbTableSchema userDBSchema[] = {
{.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "s3_chunkpages", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},

3
source/common/src/tcol.c
View File

@ -363,6 +363,9 @@ int8_t validColEncode(uint8_t type, uint8_t l1) {
if (l1 == TSDB_COLVAL_ENCODE_NOCHANGE) {
return 1;
}
if (l1 == TSDB_COLVAL_ENCODE_DISABLED) {
return 1;
}
if (type == TSDB_DATA_TYPE_BOOL) {
return TSDB_COLVAL_ENCODE_RLE == l1 ? 1 : 0;
} else if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_INT) {

13
source/common/src/tglobal.c
View File

@ -548,7 +548,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebug", tqClientDebug, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebugFlag", tqClientDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
@ -2000,7 +2000,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
{"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag},
{"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag},
{"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag},
{"tqClientDebug", &tqClientDebug},
{"tqClientDebugFlag", &tqClientDebugFlag},
};
static OptionNameAndVar options[] = {{"audit", &tsEnableAudit},
@ -2346,8 +2346,13 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag
if (noNeedToSetVars != NULL && taosArraySearch(noNeedToSetVars, name, taosLogVarComp, TD_EQ) != NULL) {
return;
}
if (taosSetDebugFlag(pFlagPtr, name, flag) != 0) {
uError("failed to set flag %s to %d", name, flag);
int32_t code = 0;
if ((code = taosSetDebugFlag(pFlagPtr, name, flag)) != 0) {
if (code != TSDB_CODE_CFG_NOT_FOUND) {
uError("failed to set flag %s to %d, since:%s", name, flag, tstrerror(code));
} else {
uDebug("failed to set flag %s to %d, since:%s", name, flag, tstrerror(code));
}
}
return;
}

7
source/common/src/tmsg.c
View File

@ -3874,6 +3874,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->s3ChunkSize));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->s3KeepLocal));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->s3Compact));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->dnodeListStr));
tEndEncode(&encoder);
@ -3962,6 +3963,10 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->s3Compact));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->dnodeListStr));
}
tEndDecode(&decoder);
_exit:
@ -10983,6 +10988,7 @@ _exit:
int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
TAOS_CHECK_RETURN(tEncodeMqDataRspCommon(pEncoder, pRsp));
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->sleepTime));
return 0;
}
@ -11094,6 +11100,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, createTableReq, createTableLen));
}
}
_exit:
return code;
}

5
source/dnode/mgmt/mgmt_snode/CMakeLists.txt
View File

@ -1,5 +1,10 @@
aux_source_directory(src MGMT_SNODE)
add_library(mgmt_snode STATIC ${MGMT_SNODE})
if(${TD_DARWIN})
target_compile_options(mgmt_snode PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories(
mgmt_snode
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"

5
source/dnode/mgmt/mgmt_vnode/CMakeLists.txt
View File

@ -1,5 +1,10 @@
aux_source_directory(src MGMT_VNODE)
add_library(mgmt_vnode STATIC ${MGMT_VNODE})
if(${TD_DARWIN})
target_compile_options(mgmt_vnode PRIVATE -Wno-error=deprecated-non-prototype)
endif()
target_include_directories(
mgmt_vnode
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"

6
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -36,12 +36,13 @@ typedef struct SVnodeMgmt {
SSingleWorker mgmtWorker;
SSingleWorker mgmtMultiWorker;
SHashObj *hash;
SHashObj *closedHash;
TdThreadRwlock lock;
SVnodesStat state;
STfs *pTfs;
TdThread thread;
bool stop;
TdThreadMutex createLock;
TdThreadMutex fileLock;
} SVnodeMgmt;
typedef struct {
@ -94,7 +95,7 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId);
SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict);
void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode);
int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl);
void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal);
void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal, bool keepClosed);
// vmHandle.c
SArray *vmGetMsgHandles();
@ -111,6 +112,7 @@ int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt);
int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
// vmWorker.c
int32_t vmStartWorker(SVnodeMgmt *pMgmt);

87
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -19,6 +19,54 @@
#define MAX_CONTENT_LEN 2 * 1024 * 1024
int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
(void)taosThreadRwlockRdlock(&pMgmt->lock);
int32_t num = 0;
int32_t size = taosHashGetSize(pMgmt->hash);
int32_t closedSize = taosHashGetSize(pMgmt->closedHash);
size += closedSize;
SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *));
if (pVnodes == NULL) {
(void)taosThreadRwlockUnlock(&pMgmt->lock);
return terrno;
}
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
SVnodeObj *pVnode = *ppVnode;
if (pVnode && num < size) {
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
dTrace("vgId:%d,acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount);
pVnodes[num++] = (*ppVnode);
pIter = taosHashIterate(pMgmt->hash, pIter);
} else {
taosHashCancelIterate(pMgmt->hash, pIter);
}
}
pIter = taosHashIterate(pMgmt->closedHash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
SVnodeObj *pVnode = *ppVnode;
if (pVnode && num < size) {
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount);
pVnodes[num++] = (*ppVnode);
pIter = taosHashIterate(pMgmt->closedHash, pIter);
} else {
taosHashCancelIterate(pMgmt->closedHash, pIter);
}
}
(void)taosThreadRwlockUnlock(&pMgmt->lock);
*numOfVnodes = num;
*ppVnodes = pVnodes;
return 0;
}
int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
(void)taosThreadRwlockRdlock(&pMgmt->lock);
@ -36,7 +84,7 @@ int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeOb
SVnodeObj *pVnode = *ppVnode;
if (pVnode && num < size) {
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
// dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount);
dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount);
pVnodes[num++] = (*ppVnode);
pIter = taosHashIterate(pMgmt->hash, pIter);
} else {
@ -203,6 +251,8 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
SVnodeObj **ppVnodes = NULL;
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
int32_t lino = 0;
int32_t ret = -1;
int32_t nBytes = snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
@ -215,8 +265,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
}
int32_t numOfVnodes = 0;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code) goto _OVER;
TAOS_CHECK_GOTO(vmGetAllVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes), &lino, _OVER);
// terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject();
@ -224,39 +273,56 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
code = terrno;
goto _OVER;
}
if ((code = vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes)) != 0) goto _OVER;
TAOS_CHECK_GOTO(vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes), &lino, _OVER);
buffer = tjsonToString(pJson);
if (buffer == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
lino = __LINE__;
goto _OVER;
}
code = taosThreadMutexLock(&pMgmt->fileLock);
if (code != 0) {
lino = __LINE__;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) {
code = terrno;
goto _OVER;
lino = __LINE__;
goto _OVER1;
}
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) {
code = terrno;
goto _OVER;
lino = __LINE__;
goto _OVER1;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
lino = __LINE__;
goto _OVER1;
}
code = taosCloseFile(&pFile);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
lino = __LINE__;
goto _OVER1;
}
TAOS_CHECK_GOTO(taosRenameFile(file, realfile), NULL, _OVER);
TAOS_CHECK_GOTO(taosRenameFile(file, realfile), &lino, _OVER1);
dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes);
_OVER1:
ret = taosThreadMutexUnlock(&pMgmt->fileLock);
if (ret != 0) {
dError("failed to unlock since %s", tstrerror(ret));
}
_OVER:
if (pJson != NULL) tjsonDelete(pJson);
if (buffer != NULL) taosMemoryFree(buffer);
@ -272,7 +338,8 @@ _OVER:
}
if (code != 0) {
dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, tstrerror(code), numOfVnodes);
dError("failed to write vnodes file:%s at line:%d since %s, vnodes:%d", realfile, lino, tstrerror(code),
numOfVnodes);
}
return code;
}

39
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -415,27 +415,30 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
goto _OVER;
}
code = taosThreadMutexLock(&pMgmt->createLock);
if (code != 0) {
dError("vgId:%d, failed to lock since %s", req.vgId, tstrerror(code));
goto _OVER;
}
code = vmWriteVnodeListToFile(pMgmt);
if (code != 0) {
code = terrno != 0 ? terrno : code;
int32_t ret = taosThreadMutexUnlock(&pMgmt->createLock);
if (ret != 0) {
dError("vgId:%d, failed to unlock since %s", req.vgId, tstrerror(ret));
}
goto _OVER;
}
int32_t ret = taosThreadMutexUnlock(&pMgmt->createLock);
if (ret != 0) {
dError("vgId:%d, failed to unlock since %s", req.vgId, tstrerror(ret));
}
_OVER:
if (code != 0) {
int32_t r = 0;
r = taosThreadRwlockWrlock(&pMgmt->lock);
if (r != 0) {
dError("vgId:%d, failed to lock since %s", req.vgId, tstrerror(r));
}
if (r == 0) {
dInfo("vgId:%d, remove from hash", req.vgId);
r = taosHashRemove(pMgmt->hash, &req.vgId, sizeof(int32_t));
if (r != 0) {
dError("vgId:%d, failed to remove vnode since %s", req.vgId, tstrerror(r));
}
}
r = taosThreadRwlockUnlock(&pMgmt->lock);
if (r != 0) {
dError("vgId:%d, failed to unlock since %s", req.vgId, tstrerror(r));
}
vnodeClose(pImpl);
vnodeDestroy(0, path, pMgmt->pTfs, 0);
} else {
@ -535,7 +538,7 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal, true);
int32_t diskPrimary = wrapperCfg.diskPrimary;
char path[TSDB_FILENAME_LEN] = {0};
@ -683,7 +686,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
dInfo("vgId:%d, close vnode", srcVgId);
vmCloseVnode(pMgmt, pVnode, true);
vmCloseVnode(pMgmt, pVnode, true, false);
int32_t diskPrimary = wrapperCfg.diskPrimary;
char srcPath[TSDB_FILENAME_LEN] = {0};
@ -738,7 +741,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t vgId = alterReq.vgId;
dInfo(
"vgId:%d,vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d "
"vgId:%d, vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d "
"learnerSelfIndex:%d strict:%d changeVersion:%d",
vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica,
alterReq.learnerSelfIndex, alterReq.strict, alterReq.changeVersion);
@ -792,7 +795,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
bool commitAndRemoveWal = vnodeShouldRemoveWal(pVnode->pImpl);
vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal);
vmCloseVnode(pMgmt, pVnode, commitAndRemoveWal, true);
int32_t diskPrimary = wrapperCfg.diskPrimary;
char path[TSDB_FILENAME_LEN] = {0};
@ -860,7 +863,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return code;
}
vmCloseVnode(pMgmt, pVnode, false);
vmCloseVnode(pMgmt, pVnode, false, false);
if (vmWriteVnodeListToFile(pMgmt) != 0) {
dError("vgId:%d, failed to write vnode list since %s", vgId, terrstr());
}

97
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -103,7 +103,7 @@ SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict) {
pVnode = NULL;
} else {
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
// dTrace("vgId:%d, acquire vnode, ref:%d", pVnode->vgId, refCount);
dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount);
}
(void)taosThreadRwlockUnlock(&pMgmt->lock);
@ -115,16 +115,24 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { return vmAcquireVno
void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
if (pVnode == NULL) return;
(void)taosThreadRwlockRdlock(&pMgmt->lock);
//(void)taosThreadRwlockRdlock(&pMgmt->lock);
int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
// dTrace("vgId:%d, release vnode, ref:%d", pVnode->vgId, refCount);
(void)taosThreadRwlockUnlock(&pMgmt->lock);
dTrace("vgId:%d, release vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount);
//(void)taosThreadRwlockUnlock(&pMgmt->lock);
}
static void vmFreeVnodeObj(SVnodeObj **ppVnode) {
if (!ppVnode || !(*ppVnode)) return;
SVnodeObj *pVnode = *ppVnode;
int32_t refCount = atomic_load_32(&pVnode->refCount);
while (refCount > 0) {
dWarn("vgId:%d, vnode is refenced, retry to free in 200ms, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount);
taosMsleep(200);
refCount = atomic_load_32(&pVnode->refCount);
}
taosMemoryFree(pVnode->path);
taosMemoryFree(pVnode);
ppVnode[0] = NULL;
@ -166,16 +174,34 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
(void)taosThreadRwlockWrlock(&pMgmt->lock);
SVnodeObj *pOld = NULL;
int32_t r = taosHashGetDup(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld);
if (r != 0) {
dError("vgId:%d, failed to get vnode from hash", pVnode->vgId);
}
if (pOld) {
vmFreeVnodeObj(&pOld);
}
int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
pOld = NULL;
r = taosHashGetDup(pMgmt->closedHash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld);
if (r != 0) {
dError("vgId:%d, failed to get vnode from closedHash", pVnode->vgId);
}
if (pOld) {
vmFreeVnodeObj(&pOld);
}
dInfo("vgId:%d, remove from closedHash", pVnode->vgId);
r = taosHashRemove(pMgmt->closedHash, &pVnode->vgId, sizeof(int32_t));
if (r != 0) {
dError("vgId:%d, failed to remove vnode from hash", pVnode->vgId);
}
(void)taosThreadRwlockUnlock(&pMgmt->lock);
return code;
}
void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) {
void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal, bool keepClosed) {
char path[TSDB_FILENAME_LEN] = {0};
bool atExit = true;
@ -185,7 +211,40 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
(void)taosThreadRwlockWrlock(&pMgmt->lock);
int32_t r = taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
if (r != 0) {
dError("vgId:%d, failed to remove vnode from hash", pVnode->vgId);
}
if (keepClosed) {
SVnodeObj *pClosedVnode = taosMemoryCalloc(1, sizeof(SVnodeObj));
(void)memset(pClosedVnode, 0, sizeof(SVnodeObj));
if (pVnode == NULL) {
dError("vgId:%d, failed to alloc vnode since %s", pVnode->vgId, terrstr());
(void)taosThreadRwlockUnlock(&pMgmt->lock);
return;
}
pClosedVnode->vgId = pVnode->vgId;
pClosedVnode->dropped = pVnode->dropped;
pClosedVnode->vgVersion = pVnode->vgVersion;
pClosedVnode->diskPrimary = pVnode->diskPrimary;
pClosedVnode->toVgId = pVnode->toVgId;
SVnodeObj *pOld = NULL;
r = taosHashGetDup(pMgmt->closedHash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld);
if (r != 0) {
dError("vgId:%d, failed to get vnode from closedHash", pVnode->vgId);
}
if (pOld) {
vmFreeVnodeObj(&pOld);
}
dInfo("vgId:%d, put vnode to closedHash", pVnode->vgId);
r = taosHashPut(pMgmt->closedHash, &pVnode->vgId, sizeof(int32_t), &pClosedVnode, sizeof(SVnodeObj *));
if (r != 0) {
dError("vgId:%d, failed to put vnode to closedHash", pVnode->vgId);
}
}
(void)taosThreadRwlockUnlock(&pMgmt->lock);
vmReleaseVnode(pMgmt, pVnode);
if (pVnode->failed) {
@ -362,9 +421,15 @@ static void *vmOpenVnodeInThread(void *param) {
static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
pMgmt->hash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
if (pMgmt->hash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to init vnode hash since %s", terrstr());
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
pMgmt->closedHash =
taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
if (pMgmt->hash == NULL) {
dError("failed to init vnode closed hash since %s", terrstr());
return TSDB_CODE_OUT_OF_MEMORY;
}
SWrapperCfg *pCfgs = NULL;
@ -459,7 +524,7 @@ static void *vmCloseVnodeInThread(void *param) {
pMgmt->state.openVnodes, pMgmt->state.totalVnodes);
tmsgReportStartup("vnode-close", stepDesc);
vmCloseVnode(pMgmt, pVnode, false);
vmCloseVnode(pMgmt, pVnode, false, false);
}
dInfo("thread:%d, numOfVnodes:%d is closed", pThread->threadIndex, pThread->vnodeNum);
@ -537,6 +602,18 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
pMgmt->hash = NULL;
}
void *pIter = taosHashIterate(pMgmt->closedHash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
vmFreeVnodeObj(ppVnode);
pIter = taosHashIterate(pMgmt->closedHash, pIter);
}
if (pMgmt->closedHash != NULL) {
taosHashCleanup(pMgmt->closedHash);
pMgmt->closedHash = NULL;
}
dInfo("total vnodes:%d are all closed", numOfVnodes);
}
@ -545,7 +622,7 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
vmStopWorker(pMgmt);
vnodeCleanup();
(void)taosThreadRwlockDestroy(&pMgmt->lock);
(void)taosThreadMutexDestroy(&pMgmt->createLock);
(void)taosThreadMutexDestroy(&pMgmt->fileLock);
taosMemoryFree(pMgmt);
}
@ -637,7 +714,7 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
goto _OVER;
}
code = taosThreadMutexInit(&pMgmt->createLock, NULL);
code = taosThreadMutexInit(&pMgmt->fileLock, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;

15
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -30,9 +30,18 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) {
dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epSet);
if (epSet.numOfEps <= 1) {
pMsg->pCont = NULL;
pMsg->code = TSDB_CODE_MNODE_NOT_FOUND;
return;
if (epSet.numOfEps == 0) {
pMsg->pCont = NULL;
pMsg->code = TSDB_CODE_MNODE_NOT_FOUND;
return;
}
// dnode is not the mnode or mnode leader and This ensures that the function correctly handles cases where the
// dnode cannot obtain a valid epSet and avoids returning an incorrect or misleading epSet.
if (strcmp(epSet.eps[0].fqdn, tsLocalFqdn) == 0 && epSet.eps[0].port == tsServerPort) {
pMsg->pCont = NULL;
pMsg->code = TSDB_CODE_MNODE_NOT_FOUND;
return;
}
}
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);

9
source/dnode/mnode/impl/inc/mndArbGroup.h
View File

@ -47,6 +47,15 @@ bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char
int32_t mndGetArbGroupSize(SMnode *pMnode);
typedef enum {
CHECK_SYNC_NONE = 0,
CHECK_SYNC_SET_ASSIGNED_LEADER = 1,
CHECK_SYNC_CHECK_SYNC = 2,
CHECK_SYNC_UPDATE = 3
} ECheckSyncOp;
void mndArbCheckSync(SArbGroup *pArbGroup, int64_t nowMs, ECheckSyncOp *pOp, SArbGroup *pNewGroup);
#ifdef __cplusplus
}
#endif

1
source/dnode/mnode/impl/inc/mndDb.h
View File

@ -37,6 +37,7 @@ const char *mndGetDbStr(const char *src);
const char *mndGetStableStr(const char *src);
int32_t mndProcessCompactDbReq(SRpcMsg *pReq);
int32_t mndCheckDbDnodeList(SMnode *pMnode, char *db, char *dnodeListStr, SArray *dnodeList);
#ifdef __cplusplus
}

2
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -70,7 +70,7 @@ typedef enum {
MND_OPER_WRITE_DB,
MND_OPER_READ_DB,
MND_OPER_READ_OR_WRITE_DB,
MND_OPER_SHOW_VARIBALES,
MND_OPER_SHOW_VARIABLES,
MND_OPER_SUBSCRIBE,
MND_OPER_CREATE_TOPIC,
MND_OPER_DROP_TOPIC,

Some files were not shown because too many files have changed in this diff Show More