xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #30188 from taosdata/merge/3.0to3.3.6 

merge: from 3.0 to 3.3.6
This commit is contained in:
Simon Guan 2025-03-15 09:39:56 +08:00 committed by GitHub
parent 79930fe8f7 8c8f0be8c5
commit d7008894a4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
424 changed files with 32233 additions and 3683 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.github/workflows/cancel-workflow-on-merge.yml vendored Normal file
View File

@ -0,0 +1,25 @@
name: Cancel Workflow on Merge
on:
pull_request:
types: [closed]
jobs:
cancel-workflow:
runs-on: ubuntu-latest
steps:
- name: Cancel Workflow if Merged or Closed
if: ${{ github.event.pull_request.merged || github.event.pull_request.state == 'closed' }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo "PR has been merged or closed, cancelling workflow..."
gh auth status
gh run list \
--repo ${{ github.repository }} \
--branch ${{ github.event.pull_request.head.ref }} \
--workflow "TDengine Test" \
--status in_progress \
--status queued \
--json databaseId --jq '.[].databaseId' | \
xargs -I {} gh run cancel --repo ${{ github.repository }} {}

7
.github/workflows/taoskeeper-ci-build.yml → .github/workflows/taoskeeper-build.yml vendored
View File

@ -1,9 +1,12 @@
name: taosKeeper Build
on:
push:
pull_request:
branches:
- 'main'
- '3.0'
paths:
- tools/keeper/**
- 'tools/keeper/**'
jobs:
build:

6
.github/workflows/taosd-ci-build.yml → .github/workflows/tdengine-build.yml vendored
View File

@ -8,12 +8,11 @@ on:
- '3.1'
- '3.3.6'
- 'enh/cmake-TD-33848'
paths-ignore:
- 'docs/**'
- 'packaging/**'
- 'tests/**'
- '*.md'
- '**/*.md'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@ -21,9 +20,10 @@ concurrency:
jobs:
build:
name: Build and test on ${{ matrix.os }}
name: Run on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-20.04

10
.github/workflows/taosd-doc-build.yml → .github/workflows/tdengine-doc-build.yml vendored
View File

@ -9,11 +9,11 @@ on:
- 'docs/**'
env:
DOC_WKC: "/root/doc_ci_work"
ZH_DOC_REPO: "docs.taosdata.com"
EN_DOC_REPO: "docs.tdengine.com"
TD_REPO: "TDengine"
TOOLS_REPO: "taos-tools"
DOC_WKC: '/root/doc_ci_work'
ZH_DOC_REPO: 'docs.taosdata.com'
EN_DOC_REPO: 'docs.tdengine.com'
TD_REPO: 'TDengine'
TOOLS_REPO: 'taos-tools'
jobs:
build-doc:

54
.github/workflows/tdengine-test-manual.yml vendored Normal file
View File

@ -0,0 +1,54 @@
name: TDengine Test (Manual)
on:
workflow_dispatch:
inputs:
specified_source_branch:
description: 'Enter the source branch name of TDengine'
required: false
default: 'unavailable'
type: string
specified_target_branch:
description: 'Enter the target branch name of TDengine'
required: false
default: 'unavailable'
type: string
specified_pr_number:
description: 'Enter the PR number of TDengine'
required: false
default: 'unavailable'
type: string
concurrency:
group: ${{ github.event.inputs.specified_target_branch }}-${{ github.event.inputs.specified_pr_number }}-TDengine
cancel-in-progress: true
env:
WKC: '/var/lib/jenkins/workspace/TDinternal/community'
jobs:
run-tests-on-linux:
uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@ci/test/workflow
with:
tdinternal: false
run_function_test: true
run_tdgpt_test: false
specified_source_branch: ${{ github.event.inputs.specified_source_branch }}
specified_target_branch: ${{ github.event.inputs.specified_target_branch }}
specified_pr_number: ${{ github.event.inputs.specified_pr_number }}
run-tests-on-mac:
uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml@ci/test/workflow
with:
tdinternal: false
specified_source_branch: ${{ github.event.inputs.specified_source_branch }}
specified_target_branch: ${{ github.event.inputs.specified_target_branch }}
specified_pr_number: ${{ github.event.inputs.specified_pr_number }}
run-tests-on-windows:
uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml@ci/test/workflow
with:
tdinternal: false
specified_source_branch: ${{ github.event.inputs.specified_source_branch }}
specified_target_branch: ${{ github.event.inputs.specified_target_branch }}
specified_pr_number: ${{ github.event.inputs.specified_pr_number }}

5
.github/workflows/taosd-ci.yml → .github/workflows/tdengine-test.yml vendored
View File

@ -1,4 +1,4 @@
name: TDengine CI Test
name: TDengine Test
on:
pull_request:
@ -17,9 +17,10 @@ on:
- 'source/common/src/tanalytics.c'
- 'tests/parallel/tdgpt_cases.task'
- 'tests/script/tsim/analytics'
- '**/*.md'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-TDengine
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-TDengine
cancel-in-progress: true
env:

2
.gitignore vendored
View File

@ -51,7 +51,6 @@ pysim/
tests/script/api/batchprepare
taosadapter
taosadapter-debug
tools/taos-tools/*
tools/taosws-rs/*
tools/taosadapter/*
tools/upx*
@ -140,7 +139,6 @@ tags
*CMakeCache*
*CMakeFiles*
.history/
*.txt
*.tcl
*.pc
contrib/geos

12
README.md
View File

@ -29,9 +29,9 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine
1. [Introduction](#1-introduction)
1. [Documentation](#2-documentation)
1. [Prerequisites](#3-prerequisites)
- [3.1 Prerequisites On Linux](#31-on-linux)
- [3.2 Prerequisites On macOS](#32-on-macos)
- [3.3 Prerequisites On Windows](#33-on-windows)
- [3.1 Prerequisites On Linux](#31-prerequisites-on-linux)
- [3.2 Prerequisites On macOS](#32-prerequisites-on-macos)
- [3.3 Prerequisites On Windows](#33-prerequisites-on-windows)
- [3.4 Clone the repo](#34-clone-the-repo)
1. [Building](#4-building)
- [4.1 Build on Linux](#41-build-on-linux)
@ -86,7 +86,7 @@ At the moment, TDengine server supports running on Linux/Windows/MacOS systems.
If you want to compile taosAdapter or taosKeeper, you need to install Go 1.18 or above.
## 3.1 On Linux
## 3.1 Prerequisites on Linux
<details>
@ -111,7 +111,7 @@ yum install -y zlib-static xz-devel snappy-devel jansson-devel pkgconfig libatom
</details>
## 3.2 On macOS
## 3.2 Prerequisites on macOS
<details>
@ -125,7 +125,7 @@ brew install argp-standalone gflags pkgconfig
</details>
## 3.3 On Windows
## 3.3 Prerequisites on Windows
<details>

243
cmake/cmake.options
View File

@ -9,61 +9,61 @@ option(
)
IF(${TD_WINDOWS})
IF(NOT TD_ASTRA)
MESSAGE("build pthread Win32")
option(
BUILD_PTHREAD
"If build pthread on Windows"
ON
)
MESSAGE("build pthread Win32")
option(
BUILD_PTHREAD
"If build pthread on Windows"
ON
)
MESSAGE("build gnu regex for Windows")
option(
BUILD_GNUREGEX
"If build gnu regex on Windows"
ON
)
MESSAGE("build gnu regex for Windows")
option(
BUILD_GNUREGEX
"If build gnu regex on Windows"
ON
)
MESSAGE("build iconv Win32")
option(
BUILD_WITH_ICONV
"If build iconv on Windows"
ON
)
MESSAGE("build iconv Win32")
option(
BUILD_WITH_ICONV
"If build iconv on Windows"
ON
)
MESSAGE("build msvcregex Win32")
option(
BUILD_MSVCREGEX
"If build msvcregex on Windows"
ON
)
MESSAGE("build msvcregex Win32")
option(
BUILD_MSVCREGEX
"If build msvcregex on Windows"
ON
)
MESSAGE("build wcwidth Win32")
option(
BUILD_WCWIDTH
"If build wcwidth on Windows"
ON
)
MESSAGE("build wcwidth Win32")
option(
BUILD_WCWIDTH
"If build wcwidth on Windows"
ON
)
MESSAGE("build wingetopt Win32")
option(
BUILD_WINGETOPT
"If build wingetopt on Windows"
ON
)
MESSAGE("build wingetopt Win32")
option(
BUILD_WINGETOPT
"If build wingetopt on Windows"
ON
)
option(
TDENGINE_3
"TDengine 3.x for taos-tools"
ON
)
option(
BUILD_CRASHDUMP
"If build crashdump on Windows"
ON
)
option(
TDENGINE_3
"TDengine 3.x for taos-tools"
ON
)
option(
BUILD_CRASHDUMP
"If build crashdump on Windows"
ON
)
ENDIF ()
ELSEIF (TD_DARWIN_64)
IF(${BUILD_TEST})
add_definitions(-DCOMPILER_SUPPORTS_CXX13)
@ -71,58 +71,102 @@ ELSEIF (TD_DARWIN_64)
ENDIF ()
option(
BUILD_GEOS
"If build with geos"
BUILD_WITH_LEMON
"If build with lemon"
ON
)
option(
BUILD_WITH_UDF
"If build with UDF"
ON
)
IF(NOT TD_ASTRA)
option(
BUILD_GEOS
"If build with geos"
ON
)
option(
BUILD_SHARED_LIBS
""
OFF
)
option(
RUST_BINDINGS
"If build with rust-bindings"
ON
)
option(
BUILD_PCRE2
"If build with pcre2"
ON
)
option(
BUILD_SHARED_LIBS
""
OFF
option(
JEMALLOC_ENABLED
"If build with jemalloc"
OFF
)
option(
BUILD_SANITIZER
"If build sanitizer"
OFF
)
option(
BUILD_ADDR2LINE
"If build addr2line"
OFF
)
option(
BUILD_WITH_LEVELDB
"If build with leveldb"
OFF
)
option(
RUST_BINDINGS
"If build with rust-bindings"
ON
option(
BUILD_WITH_ROCKSDB
"If build with rocksdb"
ON
)
option(
BUILD_PCRE2
"If build with pcre2"
ON
)
option(
BUILD_WITH_LZ4
"If build with lz4"
ON
)
ELSE ()
option(
JEMALLOC_ENABLED
"If build with jemalloc"
OFF
option(
BUILD_WITH_LZMA2
"If build with lzma2"
ON
)
option(
BUILD_SANITIZER
"If build sanitizer"
OFF
)
ENDIF ()
option(
BUILD_ADDR2LINE
"If build addr2line"
OFF
)
ADD_DEFINITIONS(-DUSE_AUDIT)
ADD_DEFINITIONS(-DUSE_GEOS)
ADD_DEFINITIONS(-DUSE_UDF)
ADD_DEFINITIONS(-DUSE_STREAM)
ADD_DEFINITIONS(-DUSE_PRCE2)
ADD_DEFINITIONS(-DUSE_RSMA)
ADD_DEFINITIONS(-DUSE_TSMA)
ADD_DEFINITIONS(-DUSE_TQ)
ADD_DEFINITIONS(-DUSE_TOPIC)
ADD_DEFINITIONS(-DUSE_MONITOR)
ADD_DEFINITIONS(-DUSE_REPORT)
option(
BUILD_WITH_LEVELDB
"If build with leveldb"
OFF
)
option(
BUILD_WITH_ROCKSDB
"If build with rocksdb"
ON
)
IF(${TD_ASTRA_RPC})
ADD_DEFINITIONS(-DTD_ASTRA_RPC)
ENDIF()
IF(${TD_LINUX})
@ -150,6 +194,12 @@ option(
ON
)
option(
BUILD_WITH_LZMA2
"If build with lzma2"
ON
)
ENDIF ()
IF(NOT TD_ENTERPRISE)
@ -191,6 +241,14 @@ option(BUILD_WITH_COS "If build with cos" OFF)
ENDIF ()
IF(${TAOSD_INTEGRATED})
add_definitions(-DTAOSD_INTEGRATED)
ENDIF()
IF(${TD_AS_LIB})
add_definitions(-DTD_AS_LIB)
ENDIF()
option(
BUILD_WITH_SQLITE
"If build with sqlite"
@ -209,6 +267,14 @@ option(
off
)
option(
BUILD_WITH_NURAFT
"If build with NuRaft"
OFF
)
IF(NOT TD_ASTRA)
option(
BUILD_WITH_UV
"If build with libuv"
@ -242,6 +308,7 @@ option(
"If use invertedIndex"
ON
)
ENDIF ()
option(
BUILD_RELEASE

3
cmake/lzma_CMakeLists.txt.in
View File

@ -1,7 +1,6 @@
# xz
if (${TD_LINUX})
if (${BUILD_WITH_LZMA2})
ExternalProject_Add(lzma2
GIT_REPOSITORY https://github.com/conor42/fast-lzma2.git
SOURCE_DIR "${TD_CONTRIB_DIR}/lzma2"

40
contrib/CMakeLists.txt
View File

@ -92,7 +92,9 @@ if(${BUILD_TEST})
endif(${BUILD_TEST})
# lz4
cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
if(${BUILD_WITH_LZ4})
cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_LZ4})
# zlib
cat("${TD_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@ -186,16 +188,22 @@ if(${BUILD_PCRE2})
cat("${TD_SUPPORT_DIR}/pcre2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
find_program(C_COMPILER_LEMON NAMES gcc)
if(C_COMPILER_LEMON)
message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}")
else()
set(C_COMPILER_LEMON ${CMAKE_C_COMPILER})
message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}")
endif()
# lemon
cat("${TD_SUPPORT_DIR}/lemon_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
if(${BUILD_WITH_LEMON})
if(${TD_ACORE})
set(C_COMPILER_LEMON ${CMAKE_C_COMPILER})
else()
find_program(C_COMPILER_LEMON NAMES gcc)
endif()
if(C_COMPILER_LEMON)
message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}")
else()
set(C_COMPILER_LEMON ${CMAKE_C_COMPILER})
message(STATUS "LEMON C compiler: ${C_COMPILER_LEMON}")
endif()
cat("${TD_SUPPORT_DIR}/lemon_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# Force specify CC=cc on MacOS. Because the default CC setting in the generated Makefile has issues finding standard library headers
IF(${TD_DARWIN})
@ -273,11 +281,13 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# endif()
# lz4
add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
target_include_directories(
lz4_static
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
)
if(${BUILD_WITH_LZ4})
add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
target_include_directories(
lz4_static
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
)
endif(${BUILD_WITH_LZ4})
# zlib
set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")

7
docs/en/06-advanced/03-stream.md
View File

@ -108,6 +108,13 @@ Under normal circumstances, stream computation tasks will not process data that
By enabling the fill_history option, the created stream computation task will be capable of processing data written before, during, and after the creation of the stream. This means that data written either before or after the creation of the stream will be included in the scope of stream computation, thus ensuring data integrity and consistency. This setting provides users with greater flexibility, allowing them to flexibly handle historical and new data according to actual needs.
Tips:
- When enabling fill_ristory, creating a stream requires finding the boundary point of historical data. If there is a lot of historical data, it may cause the task of creating a stream to take a long time. In this case, the parameter streamRunHistorySync (supported since version 3.3.6.0) can be configured to 1 (default is 0), and the task of creating a stream can be processed in the background. The statement of creating a stream can be returned immediately without blocking subsequent operations.
- Show streams can be used to view the progress of background stream creation (ready status indicates success, init status indicates stream creation in progress, failed status indicates that the stream creation has failed, and the message column can be used to view the reason for the failure. In the case of failed stream creation, the stream can be deleted and rebuilt).
- Besides, do not create multiple streams asynchronously at the same time, as transaction conflicts may cause subsequent streams to fail.
For example, create a stream to count the number of data entries generated by all smart meters every 10s, and also calculate historical data. SQL as follows:
```sql

66
docs/en/06-advanced/05-data-in/05-opcua.md
View File

@ -107,21 +107,23 @@ The header is the first line of the CSV file, with the following rules:
(1) The header of the CSV can configure the following columns:
| Number | Column Name | Description | Required | Default Behavior |
| ------ | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 1 | point_id | The id of the data point on the OPC UA server | Yes | None |
| 2 | stable | The corresponding supertable for the data point in TDengine | Yes | None |
| 3 | tbname | The corresponding subtable for the data point in TDengine | Yes | None |
| 4 | enable | Whether to collect data from this point | No | Use the unified default value `1` for enable |
| 5 | value_col | The column name in TDengine corresponding to the collected value of the data point | No | Use the unified default value `val` as the value_col |
| 6 | value_transform | The transformation function executed in taosX for the collected value of the data point | No | Do not transform the collected value uniformly |
| 7 | type | The data type of the collected value of the data point | No | Use the original type of the collected value as the data type in TDengine |
| 8 | quality_col | The column name in TDengine corresponding to the quality of the collected value | No | Do not add a quality column in TDengine uniformly |
| 9 | ts_col | The original timestamp column of the data point in TDengine | No | If both ts_col and received_ts_col are non-empty, use the former as the timestamp column; if one of ts_col or received_ts_col is non-empty, use the non-empty column as the timestamp column; if both are empty, use the original timestamp of the data point as the timestamp column with the default name `ts`. |
| 10 | received_ts_col | The timestamp column in TDengine when the data point value is received | No | Same as above |
| 11 | ts_transform | The transformation function executed in taosX for the original timestamp of the data point | No | Do not transform the original timestamp of the data point uniformly |
| 12 | received_ts_transform | The transformation function executed in taosX for the received timestamp of the data point | No | Do not transform the received timestamp of the data point uniformly |
| 13 | tag::VARCHAR(200)::name | The Tag column corresponding to the data point in TDengine. Here `tag` is a reserved keyword indicating that this column is a tag; `VARCHAR(200)` indicates the type of tag; `name` is the actual name of the tag. | No | If 1 or more tag columns are configured, use the configured tag columns; if no tag columns are configured and stable exists in TDengine, use the tags of the stable in TDengine; if no tag columns are configured and stable does not exist in TDengine, automatically add the following 2 tag columns: tag::VARCHAR(256)::point_id and tag::VARCHAR(256)::point_name |
| Number | Column Name | Description | Required | Default Behavior |
|--------|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 1 | point_id | The id of the data point on the OPC UA server | Yes | None |
| 2 | stable | The corresponding supertable for the data point in TDengine | Yes | None |
| 3 | tbname | The corresponding subtable for the data point in TDengine | Yes | None |
| 4 | enable | Whether to collect data from this point | No | Use the unified default value `1` for enable |
| 5 | value_col | The column name in TDengine corresponding to the collected value of the data point | No | Use the unified default value `val` as the value_col |
| 6 | value_transform | The transformation function executed in taosX for the collected value of the data point | No | Do not transform the collected value uniformly |
| 7 | type | The data type of the collected value of the data point | No | Use the original type of the collected value as the data type in TDengine |
| 8 | quality_col | The column name in TDengine corresponding to the quality of the collected value | No | Do not add a quality column in TDengine uniformly |
| 9 | ts_col | The original timestamp column of the data point in TDengine | No | ts_col, request_ts, received_ts these 3 columns, when there are more than 2 columns, the leftmost column is used as the primary key in TDengine. |
| 10 | request_ts_col | The timestamp column in TDengine when the data point value is request | No | Same as above |
| 11 | received_ts_col | The timestamp column in TDengine when the data point value is received | No | Same as above |
| 12 | ts_transform | The transformation function executed in taosX for the original timestamp of the data point | No | Do not transform the original timestamp of the data point uniformly |
| 13 | request_ts_transform | The transformation function executed in taosX for the request timestamp of the data point | No | Do not transform the original timestamp of the data point uniformly |
| 14 | received_ts_transform | The transformation function executed in taosX for the received timestamp of the data point | No | Do not transform the received timestamp of the data point uniformly |
| 15 | tag::VARCHAR(200)::name | The Tag column corresponding to the data point in TDengine. Here `tag` is a reserved keyword indicating that this column is a tag; `VARCHAR(200)` indicates the type of tag; `name` is the actual name of the tag. | No | If 1 or more tag columns are configured, use the configured tag columns; if no tag columns are configured and stable exists in TDengine, use the tags of the stable in TDengine; if no tag columns are configured and stable does not exist in TDengine, automatically add the following 2 tag columns: tag::VARCHAR(256)::point_id and tag::VARCHAR(256)::point_name |
(2) In the CSV Header, there cannot be duplicate columns;
@ -137,21 +139,23 @@ Each Row in the CSV file configures an OPC data point. The rules for Rows are as
(1) Correspondence with columns in the Header
| Number | Column in Header | Type of Value | Value Range | Mandatory | Default Value |
| ------ | ----------------------- | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------ |
| 1 | point_id | String | Strings like `ns=3;i=1005`, must meet the OPC UA ID specification, i.e., include ns and id parts | Yes | |
| 2 | enable | int | 0: Do not collect this point, and delete the corresponding subtable in TDengine before the OPC DataIn task starts; 1: Collect this point, do not delete the subtable before the OPC DataIn task starts. | No | 1 |
| 3 | stable | String | Any string that meets the TDengine supertable naming convention; if special character `.` exists, replace with underscore if `{type}` exists: if type in CSV file is not empty, replace with the value of type if type is empty, replace with the original type of the collected value | Yes | |
| 4 | tbname | String | Any string that meets the TDengine subtable naming convention; for OPC UA: if `{ns}` exists, replace with ns from point_id if `{id}` exists, replace with id from point_id for OPC DA: if `{tag_name}` exists, replace with tag_name | Yes | |
| 5 | value_col | String | Column name that meets TDengine naming convention | No | val |
| 6 | value_transform | String | Expressions that meet the Rhai engine, for example: `(val + 10) / 1000 * 2.0`, `log(val) + 10`, etc.; | No | None |
| 7 | type | String | Supported types include: b/bool/i8/tinyint/i16/small/inti32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/float/f64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | No | Original type of the data point value |
| 8 | quality_col | String | Column name that meets TDengine naming convention | No | None |
| 9 | ts_col | String | Column name that meets TDengine naming convention | No | ts |
| 10 | received_ts_col | String | Column name that meets TDengine naming convention | No | rts |
| 11 | ts_transform | String | Supports +, -, *, /, % operators, for example: ts / 1000* 1000, sets the last 3 digits of a timestamp in ms to 0; ts + 8 *3600* 1000, adds 8 hours to a timestamp in ms; ts - 8 *3600* 1000, subtracts 8 hours from a timestamp in ms; | No | None |
| 12 | received_ts_transform | String | No | None | |
| 13 | tag::VARCHAR(200)::name | String | The value inside a tag, when the tag type is VARCHAR, can be in Chinese | No | NULL |
| Number | Column in Header | Type of Value | Value Range | Mandatory | Default Value |
|--------|-------------------------| ------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --------- |---------------------------------------|
| 1 | point_id | String | Strings like `ns=3;i=1005`, must meet the OPC UA ID specification, i.e., include ns and id parts | Yes | |
| 2 | enable | int | 0: Do not collect this point, and delete the corresponding subtable in TDengine before the OPC DataIn task starts; 1: Collect this point, do not delete the subtable before the OPC DataIn task starts. | No | 1 |
| 3 | stable | String | Any string that meets the TDengine supertable naming convention; if special character `.` exists, replace with underscore if `{type}` exists: if type in CSV file is not empty, replace with the value of type if type is empty, replace with the original type of the collected value | Yes | |
| 4 | tbname | String | Any string that meets the TDengine subtable naming convention; for OPC UA: if `{ns}` exists, replace with ns from point_id if `{id}` exists, replace with id from point_id for OPC DA: if `{tag_name}` exists, replace with tag_name | Yes | |
| 5 | value_col | String | Column name that meets TDengine naming convention | No | val |
| 6 | value_transform | String | Expressions that meet the Rhai engine, for example: `(val + 10) / 1000 * 2.0`, `log(val) + 10`, etc.; | No | None |
| 7 | type | String | Supported types include: b/bool/i8/tinyint/i16/small/inti32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/float/f64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | No | Original type of the data point value |
| 8 | quality_col | String | Column name that meets TDengine naming convention | No | None |
| 9 | ts_col | String | Column name that meets TDengine naming convention | No | ts |
| 10 | request_ts_col | String | Column name that meets TDengine naming convention | No | qts |
| 11 | received_ts_col | String | Column name that meets TDengine naming convention | No | rts |
| 12 | ts_transform | String | Supports +, -, *, /, % operators, for example: ts / 1000* 1000, sets the last 3 digits of a timestamp in ms to 0; ts + 8 *3600* 1000, adds 8 hours to a timestamp in ms; ts - 8 *3600* 1000, subtracts 8 hours from a timestamp in ms; | No | None |
| 13 | request_ts_transform | String | Supports +, -, *, /, % operators, for example: qts / 1000* 1000, sets the last 3 digits of a timestamp in ms to 0; qts + 8 *3600* 1000, adds 8 hours to a timestamp in ms; qts - 8 *3600* 1000, subtracts 8 hours from a timestamp in ms; | No | None |
| 14 | received_ts_transform | String | Supports +, -, *, /, % operators, for example: qts / 1000* 1000, sets the last 3 digits of a timestamp in ms to 0; qts + 8 *3600* 1000, adds 8 hours to a timestamp in ms; qts - 8 *3600* 1000, subtracts 8 hours from a timestamp in ms; | None | None |
| 15 | tag::VARCHAR(200)::name | String | The value inside a tag, when the tag type is VARCHAR, can be in Chinese | No | NULL |
(2) `point_id` is unique throughout the DataIn task, meaning: in an OPC DataIn task, a data point can only be written to one subtable in TDengine. If you need to write a data point to multiple subtables, you need to create multiple OPC DataIn tasks;
@ -171,7 +175,7 @@ Data points can be filtered by configuring **Root Node ID**, **Namespace**, **Re
Configure **Supertable Name**, **Table Name** to specify the supertable and subtable where the data will be written.
Configure **Primary Key Column**, choose `origin_ts` to use the original timestamp of the OPC data point as the primary key in TDengine; choose `received_ts` to use the data's reception timestamp as the primary key in TDengine. Configure **Primary Key Alias** to specify the name of the TDengine timestamp column.
Configure **Primary Key Column**, choose `origin_ts` to use the original timestamp of the OPC data point as the primary key in TDengine; choose `request_ts` to use the data's request timestamp as the primary key in TDengine; choose `received_ts` to use the data's reception timestamp as the primary key in TDengine. Configure **Primary Key Alias** to specify the name of the TDengine timestamp column.
<figure>
<Image img={imgStep5} alt=""/>

46
docs/en/06-advanced/05-data-in/06-opcda.md
View File

@ -81,21 +81,23 @@ The header is the first line of the CSV file, with the following rules:
(1) The header of the CSV can configure the following columns:
| No. | Column Name | Description | Required | Default Behavior |
| ---- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 1 | tag_name | The id of the data point on the OPC DA server | Yes | None |
| 2 | stable | The supertable in TDengine corresponding to the data point | Yes | None |
| 3 | tbname | The subtable in TDengine corresponding to the data point | Yes | None |
| 4 | enable | Whether to collect data from this point | No | Use a unified default value `1` for enable |
| 5 | value_col | The column name in TDengine corresponding to the collected value of the data point | No | Use a unified default value `val` as the value_col |
| 6 | value_transform | The transform function executed in taosX for the collected value of the data point | No | Do not perform a transform on the collected value |
| 7 | type | The data type of the collected value of the data point | No | Use the original type of the collected value as the data type in TDengine |
| 8 | quality_col | The column name in TDengine corresponding to the quality of the collected value | No | Do not add a quality column in TDengine |
| 9 | ts_col | The timestamp column in TDengine corresponding to the original timestamp of the data point | No | If both ts_col and received_ts_col are non-empty, use the former as the timestamp column; if one of ts_col or received_ts_col is non-empty, use the non-empty column as the timestamp column; if both are empty, use the original timestamp of the data point as the timestamp column in TDengine, with the default column name ts. |
| 10 | received_ts_col | The timestamp column in TDengine corresponding to the timestamp when the data point value was received | No | |
| 11 | ts_transform | The transform function executed in taosX for the original timestamp of the data point | No | Do not perform a transform on the original timestamp of the data point |
| 12 | received_ts_transform | The transform function executed in taosX for the received timestamp of the data point | No | Do not perform a transform on the received timestamp of the data point |
| 13 | tag::VARCHAR(200)::name | The Tag column in TDengine corresponding to the data point. Where `tag` is a reserved keyword, indicating that this column is a tag column; `VARCHAR(200)` indicates the type of this tag, which can also be other legal types; `name` is the actual name of this tag. | No | If configuring more than one tag column, use the configured tag columns; if no tag columns are configured, and stable exists in TDengine, use the tags of the stable in TDengine; if no tag columns are configured, and stable does not exist in TDengine, automatically add the following two tag columns by default: tag::VARCHAR(256)::point_idtag::VARCHAR(256)::point_name |
| No. | Column Name | Description | Required | Default Behavior |
|-----|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| 1 | tag_name | The id of the data point on the OPC DA server | Yes | None |
| 2 | stable | The supertable in TDengine corresponding to the data point | Yes | None |
| 3 | tbname | The subtable in TDengine corresponding to the data point | Yes | None |
| 4 | enable | Whether to collect data from this point | No | Use a unified default value `1` for enable |
| 5 | value_col | The column name in TDengine corresponding to the collected value of the data point | No | Use a unified default value `val` as the value_col |
| 6 | value_transform | The transform function executed in taosX for the collected value of the data point | No | Do not perform a transform on the collected value |
| 7 | type | The data type of the collected value of the data point | No | Use the original type of the collected value as the data type in TDengine |
| 8 | quality_col | The column name in TDengine corresponding to the quality of the collected value | No | Do not add a quality column in TDengine |
| 9 | ts_col | The timestamp column in TDengine corresponding to the original timestamp of the data point | No | ts_col, request_ts, received_ts these 3 columns, when there are more than 2 columns, the leftmost column is used as the primary key in TDengine. |
| 10 | request_ts_col | The timestamp column in TDengine corresponding to the timestamp when the data point value was request | No | Same as above |
| 11 | received_ts_col | The timestamp column in TDengine corresponding to the timestamp when the data point value was received | No | Same as above |
| 12 | ts_transform | The transform function executed in taosX for the original timestamp of the data point | No | Do not perform a transform on the original timestamp of the data point |
| 13 | request_ts_transform | The transform function executed in taosX for the request timestamp of the data point | No | Do not perform a transform on the received timestamp of the data point |
| 14 | received_ts_transform | The transform function executed in taosX for the received timestamp of the data point | No | Do not perform a transform on the received timestamp of the data point |
| 15 | tag::VARCHAR(200)::name | The Tag column in TDengine corresponding to the data point. Where `tag` is a reserved keyword, indicating that this column is a tag column; `VARCHAR(200)` indicates the type of this tag, which can also be other legal types; `name` is the actual name of this tag. | No | If configuring more than one tag column, use the configured tag columns; if no tag columns are configured, and stable exists in TDengine, use the tags of the stable in TDengine; if no tag columns are configured, and stable does not exist in TDengine, automatically add the following two tag columns by default: tag::VARCHAR(256)::point_idtag::VARCHAR(256)::point_name |
(2) In the CSV Header, there cannot be duplicate columns;
@ -112,7 +114,7 @@ Each Row in the CSV file configures an OPC data point. The rules for Rows are as
(1) Correspondence with columns in the Header
| Number | Column in Header | Type of Value | Range of Values | Mandatory | Default Value |
| ------ | ----------------------- | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------ |
|--------|-------------------------| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------ |
| 1 | tag_name | String | Strings like `root.parent.temperature`, must meet the OPC DA ID specification | Yes | |
| 2 | enable | int | 0: Do not collect this point, and delete the corresponding subtable in TDengine before the OPC DataIn task starts; 1: Collect this point, do not delete the subtable before the OPC DataIn task starts. | No | 1 |
| 3 | stable | String | Any string that meets the TDengine supertable naming convention; if there are special characters `.`, replace with underscore. If `{type}` exists: if type in CSV file is not empty, replace with the value of type; if empty, replace with the original type of the collected value | Yes | |
@ -122,10 +124,12 @@ Each Row in the CSV file configures an OPC data point. The rules for Rows are as
| 7 | type | String | Supported types include: b/bool/i8/tinyint/i16/smallint/i32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/floatf64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | No | Original type of data point value |
| 8 | quality_col | String | Column name that meets TDengine naming convention | No | None |
| 9 | ts_col | String | Column name that meets TDengine naming convention | No | ts |
| 10 | received_ts_col | String | Column name that meets TDengine naming convention | No | rts |
| 11 | ts_transform | String | Supports +, -, *, /, % operators, for example: ts / 1000* 1000, sets the last 3 digits of a ms unit timestamp to 0; ts + 8 *3600* 1000, adds 8 hours to a ms precision timestamp; ts - 8 *3600* 1000, subtracts 8 hours from a ms precision timestamp; | No | None |
| 12 | received_ts_transform | String | No | None | |
| 13 | tag::VARCHAR(200)::name | String | The value in tag, when the tag type is VARCHAR, it can be in Chinese | No | NULL |
| 10 | request_ts_col | String | Column name that meets TDengine naming convention | No | rts |
| 11 | received_ts_col | String | Column name that meets TDengine naming convention | No | rts |
| 12 | ts_transform | String | Supports +, -, *, /, % operators, for example: ts / 1000* 1000, sets the last 3 digits of a ms unit timestamp to 0; ts + 8 *3600* 1000, adds 8 hours to a ms precision timestamp; ts - 8 *3600* 1000, subtracts 8 hours from a ms precision timestamp; | No | None |
| 13 | request_ts_transform | String | No | None | |
| 14 | received_ts_transform | String | No | None | |
| 15 | tag::VARCHAR(200)::name | String | The value in tag, when the tag type is VARCHAR, it can be in Chinese | No | NULL |
(2) `tag_name` is unique throughout the DataIn task, that is: in an OPC DataIn task, a data point can only be written to one subtable in TDengine. If you need to write a data point to multiple subtables, you need to create multiple OPC DataIn tasks;
@ -145,7 +149,7 @@ Data points can be filtered by configuring the **Root Node ID** and **Regular Ex
Configure **Supertable Name** and **Table Name** to specify the supertable and subtable where the data will be written.
Configure **Primary Key Column**, choosing `origin_ts` to use the original timestamp of the OPC data point as the primary key in TDengine; choosing `received_ts` to use the timestamp when the data is received as the primary key. Configure **Primary Key Alias** to specify the name of the TDengine timestamp column.
Configure **Primary Key Column**, choosing `origin_ts` to use the original timestamp of the OPC data point as the primary key in TDengine; choosing `request_ts` to use the timestamp when the data is request as the primary key; choosing `received_ts` to use the timestamp when the data is received as the primary key. Configure **Primary Key Alias** to specify the name of the TDengine timestamp column.
<figure>
<Image img={imgStep4} alt=""/>

276
docs/en/08-operation/03-deployment/03-kubernetes.md
View File

@ -339,20 +339,272 @@ Helm operates Kubernetes using kubectl and kubeconfig configurations, which can
The TDengine Chart has not yet been released to the Helm repository, it can currently be downloaded directly from GitHub:
```shell
wget https://github.com/taosdata/TDengine-Operator/raw/refs/heads/3.0/helm/tdengine-enterprise-3.5.0.tgz
wget https://github.com/taosdata/TDengine-Operator/raw/refs/heads/3.0/helm/tdengine-3.5.0.tgz
```
Note that it's for the enterprise edition, and the community edition is not yet available.
Follow the steps below to install the TDengine Chart:
```shell
# Edit the values.yaml file to set the topology of the cluster
vim values.yaml
helm install tdengine tdengine-enterprise-3.5.0.tgz -f values.yaml
helm install tdengine tdengine-3.5.0.tgz -f values.yaml
```
#### Case 1: Simple 1-node Deployment
If you are using community images, you can use the following command to install TDengine with Helm Chart:
<details>
<summary>Helm Chart Use Cases for Community</summary>
#### Community Case 1: Simple 1-node Deployment
The following is a simple example of deploying a single-node TDengine cluster using Helm.
```yaml
# This example is a simple deployment with one server replica.
name: "tdengine"
image:
repository: # Leave a trailing slash for the repository, or "" for no repository
server: tdengine/tdengine:latest
# Set timezone here, not in taoscfg
timezone: "Asia/Shanghai"
labels:
app: "tdengine"
# Add more labels as needed.
services:
server:
type: ClusterIP
replica: 1
ports:
# TCP range required
tcp: [6041, 6030, 6060]
# UDP range, optional
udp:
volumes:
- name: data
mountPath: /var/lib/taos
spec:
storageClassName: "local-path"
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: "10Gi"
- name: log
mountPath: /var/log/taos/
spec:
storageClassName: "local-path"
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: "10Gi"
files:
- name: cfg # must be lower case.
mountPath: /etc/taos/taos.cfg
content: |
dataDir /var/lib/taos/
logDir /var/log/taos/
```
Let's explain the above configuration:
- name: The name of the deployment, here it is "tdengine".
- image:
- repository: The image repository address, remember to leave a trailing slash for the repository, or set it to an empty string to use docker.io.
- server: The specific name and tag of the server image. You need to ask your business partner for the TDengine Enterprise image.
- timezone: Set the timezone, here it is "Asia/Shanghai".
- labels: Add labels to the deployment, here is an app label with the value "tdengine", more labels can be added as needed.
- services:
- server: Configure the server service.
- type: The service type, here it is **ClusterIP**.
- replica: The number of replicas, here it is 1.
- ports: Configure the ports of the service.
- tcp: The required TCP port range, here it is [6041, 6030, 6060].
- udp: The optional UDP port range, which is not configured here.
- volumes: Configure the volumes.
- name: The name of the volume, here there are two volumes, data and log.
- mountPath: The mount path of the volume.
- spec: The specification of the volume.
- storageClassName: The storage class name, here it is **local-path**.
- accessModes: The access mode, here it is **ReadWriteOnce**.
- resources.requests.storage: The requested storage size, here it is **10Gi**.
- files: Configure the files to mount in TDengine server.
- name: The name of the file, here it is **cfg**.
- mountPath: The mount path of the file, which is **taos.cfg**.
- content: The content of the file, here the **dataDir** and **logDir** are configured.
After configuring the values.yaml file, use the following command to install the TDengine Chart:
```shell
helm install simple tdengine-3.5.0.tgz -f values.yaml
```
After installation, you can see the instructions to see the status of the TDengine cluster:
```shell
NAME: simple
LAST DEPLOYED: Sun Feb 9 13:40:00 2025 default
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
1. Get first POD name:
export POD_NAME=$(kubectl get pods --namespace default \
-l "app.kubernetes.io/name=tdengine,app.kubernetes.io/instance=simple" -o jsonpath="{.items[0].metadata.name}")
2. Show dnodes/mnodes:
kubectl --namespace default exec $POD_NAME -- taos -s "show dnodes; show mnodes"
3. Run into TDengine CLI:
kubectl --namespace default exec -it $POD_NAME -- taos
```
Follow the instructions to check the status of the TDengine cluster:
```shell
root@u1-58:/data1/projects/helm# kubectl --namespace default exec $POD_NAME -- taos -s "show dnodes; show mnodes"
Welcome to the TDengine Command Line Interface, Client Version:3.3.5.8
Copyright (c) 2023 by TDengine, all rights reserved.
taos> show dnodes; show mnodes
id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note |
=============================================================================================================================================================================
1 | oss-tdengine-0.oss-tdengine... | 0 | 21 | ready | 2025-03-12 19:05:42.224 | 2025-03-12 19:05:42.044 | |
Query OK, 1 row(s) in set (0.002545s)
id | endpoint | role | status | create_time | role_time |
==================================================================================================================================
1 | oss-tdengine-0.oss-tdengine... | leader | ready | 2025-03-12 19:05:42.239 | 2025-03-12 19:05:42.137 |
Query OK, 1 row(s) in set (0.001343s)
```
To clean up the TDengine cluster, use the following command:
```shell
helm uninstall simple
kubectl delete pvc -l app.kubernetes.io/instance=simple
```
#### Community Case 2: 3-replica Deployment with Single taosX
```yaml
# This example shows how to deploy a 3-replica TDengine cluster with separate taosx/explorer service.
# Users should know that the explorer/taosx service is not cluster-ready, so it is recommended to deploy it separately.
name: "tdengine"
image:
repository: # Leave a trailing slash for the repository, or "" for no repository
server: tdengine/tdengine:latest
# Set timezone here, not in taoscfg
timezone: "Asia/Shanghai"
labels:
# Add more labels as needed.
services:
server:
type: ClusterIP
replica: 3
ports:
# TCP range required
tcp: [6041, 6030]
# UDP range, optional
udp:
volumes:
- name: data
mountPath: /var/lib/taos
spec:
storageClassName: "local-path"
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: "10Gi"
- name: log
mountPath: /var/log/taos/
spec:
storageClassName: "local-path"
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: "10Gi"
```
You can see that the configuration is similar to the first one, with the addition of the taosx configuration. The taosx service is configured with similar storage configuration as the server service, and the server service is configured with 3 replicas. Since the taosx service is not cluster-ready, it is recommended to deploy it separately.
After configuring the values.yaml file, use the following command to install the TDengine Chart:
```shell
helm install replica3 tdengine-3.5.0.tgz -f values.yaml
```
To clean up the TDengine cluster, use the following command:
```shell
helm uninstall replica3
kubectl delete pvc -l app.kubernetes.io/instance=replica3
```
You can use the following command to expose the explorer service to the outside world with ingress:
```shell
tee replica3-ingress.yaml <<EOF
# This is a helm chart example for deploying 3 replicas of TDengine Explorer
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: replica3-ingress
namespace: default
spec:
rules:
- host: replica3.local.tdengine.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: replica3-tdengine-taosx
port:
number: 6060
EOF
kubectl apply -f replica3-ingress.yaml
```
Use `kubectl get ingress` to view the ingress service.
```shell
root@server:/data1/projects/helm# kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
replica3-ingress nginx replica3.local.tdengine.com 192.168.1.58 80 48m
```
You can configure the domain name resolution to point to the ingress service's external IP address. For example, add the following line to the hosts file:
```conf
192.168.1.58 replica3.local.tdengine.com
```
Now you can access the explorer service through the domain name `replica3.local.tdengine.com`.
```shell
curl http://replica3.local.tdengine.com
```
</details>
With TDengine Enterprise images, you can use the following command to install TDengine with Helm Chart:
<details>
<summary>Helm Chart Use Cases for Enterprise</summary>
#### Enterprise Case 1: Simple 1-node Deployment
The following is a simple example of deploying a single-node TDengine cluster using Helm.
@ -435,7 +687,7 @@ Let's explain the above configuration:
After configuring the values.yaml file, use the following command to install the TDengine Chart:
```shell
helm install simple tdengine-enterprise-3.5.0.tgz -f values.yaml
helm install simple tdengine-3.5.0.tgz -f values.yaml
```
After installation, you can see the instructions to see the status of the TDengine cluster:
@ -487,7 +739,7 @@ helm uninstall simple
kubectl delete pvc -l app.kubernetes.io/instance=simple
```
#### Case 2: Tiered-Storage Deployment
#### Enterprise Case 2: Tiered-Storage Deployment
The following is an example of deploying a TDengine cluster with tiered storage using Helm.
@ -563,10 +815,10 @@ You can see that the configuration is similar to the previous one, with the addi
After configuring the values.yaml file, use the following command to install the TDengine Chart:
```shell
helm install tiered tdengine-enterprise-3.5.0.tgz -f values.yaml
helm install tiered tdengine-3.5.0.tgz -f values.yaml
```
#### Case 3: 2-replica Deployment
#### Enterprise Case 3: 2-replica Deployment
TDengine support 2-replica deployment with an arbitrator, which can be configured as follows:
@ -634,7 +886,7 @@ services:
You can see that the configuration is similar to the first one, with the addition of the arbitrator configuration. The arbitrator service is configured with the same storage as the server service, and the server service is configured with 2 replicas (the arbitrator should be 1 replica and not able to be changed).
#### Case 4: 3-replica Deployment with Single taosX
#### Enterprise Case 4: 3-replica Deployment with Single taosX
```yaml
# This example shows how to deploy a 3-replica TDengine cluster with separate taosx/explorer service.
@ -761,7 +1013,7 @@ You can see that the configuration is similar to the first one, with the additio
After configuring the values.yaml file, use the following command to install the TDengine Chart:
```shell
helm install replica3 tdengine-enterprise-3.5.0.tgz -f values.yaml
helm install replica3 tdengine-3.5.0.tgz -f values.yaml
```
You can use the following command to expose the explorer service to the outside world with ingress:
@ -810,3 +1062,5 @@ Now you can access the explorer service through the domain name `replica3.local.
```shell
curl http://replica3.local.tdengine.com
```
</details>

6
docs/en/14-reference/01-components/02-taosc.md
View File

@ -72,6 +72,12 @@ The TDengine client driver provides all the APIs needed for application programm
| tempDir | |Supported, effective immediately | Specifies the directory for generating temporary files during operation, default on Linux platform is /tmp |
| minimalTmpDirGB | |Supported, effective immediately | Minimum space required to be reserved in the directory specified by tempDir, in GB, default value: 1 |
### Stream Related
| Parameter Name |Supported Version|Dynamic Modification| Description |
|-----------------------|----------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| streamRunHistoryAsync | 3.3.6.0 |Supported, effective immediately | When creating a stream with the fill_history parameter, should the stream statement be executed asynchronously. Boolean value, async if true, sync if false. default is false |
### Log Related
|Parameter Name|Supported Version|Dynamic Modification|Description|

3
docs/en/14-reference/03-taos-sql/04-stable.md
View File

@ -20,6 +20,7 @@ table_options:
table_option: {
COMMENT 'string_value'
| SMA(col_name [, col_name] ...)
| KEEP value
}
```
@ -34,6 +35,7 @@ table_option: {
- TAGS can have up to 128 columns, at least 1, with a total length not exceeding 16 KB.
4. For the use of `ENCODE` and `COMPRESS`, please refer to [Column Compression](../manage-data-compression/)
5. For explanations of parameters in table_option, please refer to [Table SQL Description](../manage-tables/)
6. Regarding the keep parameter in table_option, it only takes effect for super tables. For detailed explanation of the keep parameter, please refer to [Database Description](02-database.md). The only difference is that the super table's keep parameter does not immediately affect query results, but only takes effect after compaction.
## View Supertables
@ -144,6 +146,7 @@ alter_table_options:
alter_table_option: {
COMMENT 'string_value'
| KEEP value
}
```

9
docs/en/14-reference/03-taos-sql/06-select.md
View File

@ -276,6 +276,15 @@ TDengine supports INNER JOIN based on the timestamp primary key, with the follow
5. Both sides of JOIN support subqueries.
6. Does not support mixing with the FILL clause.
## INTERP
The INTERP clause is a dedicated syntax for the INTERP function (../function/#interp). When an SQL statement contains an INTERP clause, it can only query the INTERP function and cannot be used with other functions. Additionally, the INTERP clause cannot be used simultaneously with window clauses (window_clause) or group by clauses (group_by_clause). The INTERP function must be used with the RANGE, EVERY, and FILL clauses; stream computing does not support the use of RANGE but requires the use of the EVERY and FILL keywords.
- The output time range for INTERP is specified by the RANGE(timestamp1, timestamp2) field, which must satisfy timestamp1 \<= timestamp2. Here, timestamp1 is the start value of the output time range, i.e., if the conditions for interpolation are met at timestamp1, then timestamp1 is the first record output, and timestamp2 is the end value of the output time range, i.e., the timestamp of the last record output cannot be greater than timestamp2.
- INTERP determines the number of results within the output time range based on the EVERY(time_unit) field, starting from timestamp1 and interpolating at fixed intervals of time (time_unit value), where time_unit can be time units: 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), 1w (weeks). For example, EVERY(500a) will interpolate the specified data every 500 milliseconds.
- INTERP determines how to interpolate at each time point that meets the output conditions based on the FILL field. For how to use the FILL clause, refer to [FILL Clause](../time-series-extensions/)
- INTERP can interpolate at a single time point specified in the RANGE field, in which case the EVERY field can be omitted. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear).
- INTERP query supports NEAR FILL mode, i.e., when FILL is needed, it uses the data closest to the current time point for interpolation. When the timestamps before and after are equally close to the current time slice, FILL the previous row's value. This mode is not supported in stream computing and window queries. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR).(Supported from version 3.3.4.9).
- INTERP `RANGE` clause supports the expansion of the time range (supported from version 3.3.4.9), such as `RANGE('2023-01-01 00:00:00', 10s)` means to find data 10s before and after the time point '2023-01-01 00:00:00' for interpolation, FILL PREV/NEXT/NEAR respectively means to look for data forward/backward/around the time point, if there is no data around the time point, then use the value specified by FILL for interpolation, therefore the FILL clause must specify a value at this time. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1). Currently, only the combination of time point and time range is supported, not the combination of time interval and time range, i.e., RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h) is not supported. The specified time range rules are similar to EVERY, the unit cannot be year or month, the value cannot be 0, and cannot have quotes. When using this extension, other FILL modes except FILL PREV/NEXT/NEAR are not supported, and the EVERY clause cannot be specified.
## GROUP BY
If a GROUP BY clause is specified in the statement, the SELECT list can only contain the following expressions:

65
docs/en/14-reference/03-taos-sql/10-function.md
View File

@ -1965,42 +1965,6 @@ FIRST(expr)
- If all columns in the result set are NULL, no results are returned.
- For tables with composite primary keys, if there are multiple entries with the smallest timestamp, only the data with the smallest composite primary key is returned.
### INTERP
```sql
INTERP(expr [, ignore_null_values])
ignore_null_values: {
0
| 1
}
```
**Function Description**: Returns the record value or interpolated value of a specified column at a specified time slice. The ignore_null_values parameter can be 0 or 1, where 1 means to ignore NULL values, default is 0.
**Return Data Type**: Same as the field type.
**Applicable Data Types**: Numeric types.
**Applicable to**: Tables and supertables.
**Usage Instructions**
- INTERP is used to obtain the record value of a specified column at a specified time slice. If there is no row data that meets the conditions at that time slice, interpolation will be performed according to the settings of the FILL parameter.
- The input data for INTERP is the data of the specified column, which can be filtered through conditional statements (where clause). If no filtering condition is specified, the input is all data.
- INTERP SQL queries need to be used together with the RANGE, EVERY, and FILL keywords; stream computing cannot use RANGE, needs EVERY and FILL keywords together.
- The output time range for INTERP is specified by the RANGE(timestamp1, timestamp2) field, which must satisfy timestamp1 \<= timestamp2. Here, timestamp1 is the start value of the output time range, i.e., if the conditions for interpolation are met at timestamp1, then timestamp1 is the first record output, and timestamp2 is the end value of the output time range, i.e., the timestamp of the last record output cannot be greater than timestamp2.
- INTERP determines the number of results within the output time range based on the EVERY(time_unit) field, starting from timestamp1 and interpolating at fixed intervals of time (time_unit value), where time_unit can be time units: 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), 1w (weeks). For example, EVERY(500a) will interpolate the specified data every 500 milliseconds.
- INTERP determines how to interpolate at each time point that meets the output conditions based on the FILL field. For how to use the FILL clause, refer to [FILL Clause](../time-series-extensions/)
- INTERP can interpolate at a single time point specified in the RANGE field, in which case the EVERY field can be omitted. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear).
- When INTERP is applied to a supertable, it will sort all the subtable data under that supertable by primary key column and perform interpolation calculations, and can also be used with PARTITION BY tbname to force the results to a single timeline.
- INTERP can be used with the pseudocolumn _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0).
- INTERP can be used with the pseudocolumn _isfilled to display whether the return result is from the original record or generated by the interpolation algorithm (supported from version 3.0.3.0).
- For queries on tables with composite primary keys, if there are data with the same timestamp, only the data with the smallest composite primary key participates in the calculation.
- INTERP query supports NEAR FILL mode, i.e., when FILL is needed, it uses the data closest to the current time point for interpolation. When the timestamps before and after are equally close to the current time slice, FILL the previous row's value. This mode is not supported in stream computing and window queries. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR).(Supported from version 3.3.4.9).
- INTERP can only use the pseudocolumn `_irowts_origin` when using FILL PREV/NEXT/NEAR modes. `_irowts_origin` is supported from version 3.3.4.9.
- INTERP `RANGE` clause supports the expansion of the time range (supported from version 3.3.4.9), For example, `RANGE('2023-01-01 00:00:00', 10s)` means that only data within 10s around the time point '2023-01-01 00:00:00' can be used for interpolation. `FILL PREV/NEXT/NEAR` respectively means to look for data forward/backward/around the time point. If there is no data around the time point, the default value specified by `FILL` is used for interpolation. Therefore the `FILL` clause must specify the default value at the same time. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1). Starting from the 3.3.6.0 version, the combination of time period and time range is supported. When interpolating for each point within the time period, the time range requirement must be met. Prior versions only supported single time point and its time range. The available values for time range are similar to `EVERY`, the unit cannot be year or month, the value must be greater than 0, and cannot be in quotes. When using this extension, `FILL` modes other than `PREV/NEXT/NEAR` are not supported.
### LAST
```sql
@ -2266,6 +2230,35 @@ ignore_option: {
- When there is no composite primary key, if different subtables have data with the same timestamp, a "Duplicate timestamps not allowed" message will be displayed
- When using composite primary keys, the timestamp and primary key combinations of different subtables may be the same, which row is used depends on which one is found first, meaning that the results of running diff() multiple times in this situation may vary.
### INTERP
```sql
INTERP(expr [, ignore_null_values])
ignore_null_values: {
0
| 1
}
```
**Function Description**: Returns the record value or interpolated value of a specified column at a specified time slice. The ignore_null_values parameter can be 0 or 1, where 1 means to ignore NULL values, default is 0.
**Return Data Type**: Same as the field type.
**Applicable Data Types**: Numeric types.
**Applicable to**: Tables and supertables.
**Usage Instructions**
- INTERP is used to obtain the record value of a specified column at the specified time slice. It has a dedicated syntax (interp_clause) when used. For syntax introduction, see [reference link](../query-data/#interp).
- When there is no row data that meets the conditions at the specified time slice, the INTERP function will interpolate according to the settings of the [FILL](../time-series-extensions/#fill-clause) parameter.
- When INTERP is applied to a supertable, it will sort all the subtable data under that supertable by primary key column and perform interpolation calculations, and can also be used with PARTITION BY tbname to force the results to a single timeline.
- INTERP can be used with the pseudocolumn _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0).
- INTERP can be used with the pseudocolumn _isfilled to display whether the return result is from the original record or generated by the interpolation algorithm (supported from version 3.0.3.0).
- INTERP can only use the pseudocolumn `_irowts_origin` when using FILL PREV/NEXT/NEAR modes. `_irowts_origin` is supported from version 3.3.4.9.
- For queries on tables with composite primary keys, if there are data with the same timestamp, only the data with the smallest composite primary key participates in the calculation.
### IRATE
```sql

2
docs/en/14-reference/09-error-code.md
View File

@ -73,6 +73,8 @@ This document details the server error codes that may be encountered when using
| 0x80000134 | Invalid value | Invalid value | Preserve the scene and logs, report issue on github |
| 0x80000135 | Invalid fqdn | Invalid FQDN | Check if the configured or input FQDN value is correct |
| 0x8000013C | Invalid disk id | Invalid disk id | Check users whether the mounted disk is invalid or use the parameter diskIDCheckEnabled to skip the disk check. |
| 0x8000013D | Decimal value overflow | Decimal value overflow | Check query expression and decimal values |
| 0x8000013E | Division by zero error | Division by zero | Check division expression |
## tsc

7
docs/zh/06-advanced/03-stream.md
View File

@ -101,6 +101,13 @@ PARTITION 子句中，为 tbname 定义了一个别名 tname， 在 PARTITION
通过启用 fill_history 选项，创建的流计算任务将具备处理创建前、创建过程中以及创建后写入的数据的能力。这意味着，无论数据是在流创建之前还是之后写入的，都将纳入流计算的范围，从而确保数据的完整性和一致性。这一设置为用户提供了更大的灵活性，使其能够根据实际需求灵活处理历史数据和新数据。
注意：
- 开启 fill_history 时，创建流需要找到历史数据的分界点，如果历史数据很多，可能会导致创建流任务耗时较长，此时可以配置参数 streamRunHistoryAsync（3.3.6.0版本开始支持） 为 1 （默认为0），将创建流的任务放在后台处理，创建流的语句可立即返回，不阻塞后面的操作。
- 通过 show streams 可查看后台建流的进度（ready 状态表示成功，init 状态表示正在建流，failed 状态表示建流失败，失败时 message 列可以查看原因。对于建流失败的情况可以删除流重新建立）。
- 另外，不要同时异步创建多个流，可能由于事务冲突导致后面创建的流失败。
比如，创建一个流，统计所有智能电表每 10s 产生的数据条数，并且计算历史数据。SQL 如下：
```sql
create stream if not exists count_history_s fill_history 1 into count_history as select count(*) from power.meters interval(10s)

66
docs/zh/06-advanced/05-data-in/05-opcua.md
View File

@ -89,21 +89,23 @@ Header 是 CSV 文件的第一行，规则如下：
(1) CSV 的 Header 中可以配置以下列：
| 序号 | 列名 | 描述 | 是否必填 | 默认行为 |
| ---- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 1 | point_id | 数据点位在 OPC UA 服务器上的 id | 是 | 无 |
| 2 | stable | 数据点位在 TDengine 中对应的超级表 | 是 | 无 |
| 3 | tbname | 数据点位在 TDengine 中对应的子表 | 是 | 无 |
| 4 | enable | 是否采集该点位的数据 | 否 | 使用统一的默认值`1`作为 enable 的值 |
| 5 | value_col | 数据点位采集值在 TDengine 中对应的列名 | 否 | 使用统一的默认值`val` 作为 value_col 的值 |
| 6 | value_transform | 数据点位采集值在 taosX 中执行的变换函数 | 否 | 统一不进行采集值的 transform |
| 7 | type | 数据点位采集值的数据类型 | 否 | 统一使用采集值的原始类型作为 TDengine 中的数据类型 |
| 8 | quality_col | 数据点位采集值质量在 TDengine 中对应的列名 | 否 | 统一不在 TDengine 添加 quality 列 |
| 9 | ts_col | 数据点位的原始时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col 和 received_ts_col 都非空时，使用前者作为时间戳列；ts_col 和 received_ts_col 有一列非空时，使用不为空的列作时间戳列；ts_col 和 received_ts_col 都为空时，使用数据点位原始时间戳作 TDengine 中的时间戳列，且列名为默认值`ts`。 |
| 10 | received_ts_col | 接收到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | 同上 |
| 11 | ts_transform | 数据点位时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform |
| 12 | received_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform |
| 13 | tag::VARCHAR(200)::name | 数据点位在 TDengine 中对应的 Tag 列。其中`tag` 为保留关键字，表示该列为一个 tag 列；`VARCHAR(200)` 表示该 tag 的类型，也可以是其它合法的类型；`name` 是该 tag 的实际名称。 | 否 | 配置 1 个以上的 tag 列，则使用配置的 tag 列；没有配置任何 tag 列，且 stable 在 TDengine 中存在，使用 TDengine 中的 stable 的 tag；没有配置任何 tag 列，且 stable 在 TDengine 中不存在，则默认自动添加以下 2 个 tag 列：tag::VARCHAR(256)::point_id 和 tag::VARCHAR(256)::point_name |
| 序号 | 列名 | 描述 | 是否必填 | 默认行为 |
|----|-------------------------|--------------------------------------------------------------------------------------------------------------------| -------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 1 | point_id | 数据点位在 OPC UA 服务器上的 id | 是 | 无 |
| 2 | stable | 数据点位在 TDengine 中对应的超级表 | 是 | 无 |
| 3 | tbname | 数据点位在 TDengine 中对应的子表 | 是 | 无 |
| 4 | enable | 是否采集该点位的数据 | 否 | 使用统一的默认值`1`作为 enable 的值 |
| 5 | value_col | 数据点位采集值在 TDengine 中对应的列名 | 否 | 使用统一的默认值`val` 作为 value_col 的值 |
| 6 | value_transform | 数据点位采集值在 taosX 中执行的变换函数 | 否 | 统一不进行采集值的 transform |
| 7 | type | 数据点位采集值的数据类型 | 否 | 统一使用采集值的原始类型作为 TDengine 中的数据类型 |
| 8 | quality_col | 数据点位采集值质量在 TDengine 中对应的列名 | 否 | 统一不在 TDengine 添加 quality 列 |
| 9 | ts_col | 数据点位的原始时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键。 |
| 10 | request_ts_col | 请求到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键。 |
| 11 | received_ts_col | 接收到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键。 |
| 12 | ts_transform | 数据点位时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform |
| 13 | request_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform |
| 14 | received_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform |
| 15 | tag::VARCHAR(200)::name | 数据点位在 TDengine 中对应的 Tag 列。其中`tag` 为保留关键字，表示该列为一个 tag 列；`VARCHAR(200)` 表示该 tag 的类型，也可以是其它合法的类型；`name` 是该 tag 的实际名称。 | 否 | 配置 1 个以上的 tag 列，则使用配置的 tag 列；没有配置任何 tag 列，且 stable 在 TDengine 中存在，使用 TDengine 中的 stable 的 tag；没有配置任何 tag 列，且 stable 在 TDengine 中不存在，则默认自动添加以下 2 个 tag 列：tag::VARCHAR(256)::point_id 和 tag::VARCHAR(256)::point_name |
(2) CSV Header 中，不能有重复的列；
@ -120,21 +122,23 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下
(1) 与 Header 中的列有如下对应关系
| 序号 | Header 中的列 | 值的类型 | 值的范围 | 是否必填 | 默认值 |
| ---- | ----------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------ |
| 1 | point_id | String | 类似`ns=3;i=1005`这样的字符串，要满足 OPC UA 的 ID 的规范，即：包含 ns 和 id 部分 | 是 | |
| 2 | enable | int | 0：不采集该点位，且在 OPC DataIn 任务开始前，删除 TDengine 中点位对应的子表；1：采集该点位，在 OPC DataIn 任务开始前，不删除子表。 | 否 | 1 |
| 3 | stable | String | 符合 TDengine 超级表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换如果存在`{type}`，则：CSV 文件的 type 不为空，使用 type 的值进行替换CSV 文件的 type 为空，使用采集值的原始类型进行替换 | 是 | |
| 4 | tbname | String | 符合 TDengine 子表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换对于 OPC UA：如果存在`{ns}`，使用 point_id 中的 ns 替换如果存在`{id}`，使用 point_id 中的 id 替换对于 OPC DA：如果存在`{tag_name}`，使用 tag_name 替换 | 是 | |
| 5 | value_col | String | 符合 TDengine 命名规范的列名 | 否 | val |
| 6 | value_transform | String | 符合 Rhai 引擎的计算表达式，例如：`(val + 10) / 1000 * 2.0`，`log(val) + 10`等； | 否 | None |
| 7 | type | String | 支持类型包括：b/bool/i8/tinyint/i16/small/inti32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/float/f64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | 否 | 数据点位采集值的原始类型 |
| 8 | quality_col | String | 符合 TDengine 命名规范的列名 | 否 | None |
| 9 | ts_col | String | 符合 TDengine 命名规范的列名 | 否 | ts |
| 10 | received_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts |
| 11 | ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；ts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；ts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None |
| 12 | received_ts_transform | String | 否 | None | |
| 13 | tag::VARCHAR(200)::name | String | tag 里的值，当 tag 的类型是 VARCHAR 时，可以是中文 | 否 | NULL |
| 序号 | Header 中的列 | 值的类型 | 值的范围 | 是否必填 | 默认值 |
|----|-------------------------| -------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------| ------------------------ |
| 1 | point_id | String | 类似`ns=3;i=1005`这样的字符串，要满足 OPC UA 的 ID 的规范，即：包含 ns 和 id 部分 | 是 | |
| 2 | enable | int | 0：不采集该点位，且在 OPC DataIn 任务开始前，删除 TDengine 中点位对应的子表；1：采集该点位，在 OPC DataIn 任务开始前，不删除子表。 | 否 | 1 |
| 3 | stable | String | 符合 TDengine 超级表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换如果存在`{type}`，则：CSV 文件的 type 不为空，使用 type 的值进行替换CSV 文件的 type 为空，使用采集值的原始类型进行替换 | 是 | |
| 4 | tbname | String | 符合 TDengine 子表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换对于 OPC UA：如果存在`{ns}`，使用 point_id 中的 ns 替换如果存在`{id}`，使用 point_id 中的 id 替换对于 OPC DA：如果存在`{tag_name}`，使用 tag_name 替换 | 是 | |
| 5 | value_col | String | 符合 TDengine 命名规范的列名 | 否 | val |
| 6 | value_transform | String | 符合 Rhai 引擎的计算表达式，例如：`(val + 10) / 1000 * 2.0`，`log(val) + 10`等； | 否 | None |
| 7 | type | String | 支持类型包括：b/bool/i8/tinyint/i16/small/inti32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/float/f64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | 否 | 数据点位采集值的原始类型 |
| 8 | quality_col | String | 符合 TDengine 命名规范的列名 | 否 | None |
| 9 | ts_col | String | 符合 TDengine 命名规范的列名 | 否 | ts |
| 10 | request_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts |
| 11 | received_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts |
| 12 | ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；ts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；ts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None |
| 13 | request_ts_transform | String | 支持 +、-、*、/、% 操作符，例如：qts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；qts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；qts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None |
| 14 | received_ts_transform | String | 支持 +、-、*、/、% 操作符，例如：rts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；rts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；rts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None |
| 15 | tag::VARCHAR(200)::name | String | tag 里的值，当 tag 的类型是 VARCHAR 时，可以是中文 | 否 | NULL |
(2) point_id 在整个 DataIn 任务中是唯一的，即：在一个 OPC DataIn 任务中，一个数据点位只能被写入到 TDengine 的一张子表。如果需要将一个数据点位写入多张子表，需要建多个 OPC DataIn 任务；
@ -154,7 +158,7 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下
通过配置 **超级表名**、**表名称**，指定数据要写入的超级表、子表。
配置**主键列**，选择 origin_ts 表示使用 OPC 点位数据的原始时间戳作 TDengine 中的主键；选择 received_ts 表示使用数据的接收时间戳作 TDengine 中的主键。配置**主键别名**，指定 TDengine 时间戳列的名称。
配置**主键列**，选择 origin_ts 表示使用 OPC 点位数据的原始时间戳作 TDengine 中的主键；选择 request_ts 表示使用数据的请求时间戳作 TDengine 中的主键；选择 received_ts 表示使用数据的接收时间戳作 TDengine 中的主键。配置**主键别名**，指定 TDengine 时间戳列的名称。
![point.png](./pic/opcua-06-point.png)

66
docs/zh/06-advanced/05-data-in/06-opcda.md
View File

@ -65,21 +65,23 @@ Header 是 CSV 文件的第一行，规则如下：
(1) CSV 的 Header 中可以配置以下列：
| 序号 | 列名 | 描述 | 是否必填 | 默认行为 |
| ---- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 1 | tag_name | 数据点位在 OPC DA 服务器上的 id | 是 | 无 |
| 2 | stable | 数据点位在 TDengine 中对应的超级表 | 是 | 无 |
| 3 | tbname | 数据点位在 TDengine 中对应的子表 | 是 | 无 |
| 4 | enable | 是否采集该点位的数据 | 否 | 使用统一的默认值`1`作为 enable 的值 |
| 5 | value_col | 数据点位采集值在 TDengine 中对应的列名 | 否 | 使用统一的默认值`val` 作为 value_col 的值 |
| 6 | value_transform | 数据点位采集值在 taosX 中执行的变换函数 | 否 | 统一不进行采集值的 transform |
| 7 | type | 数据点位采集值的数据类型 | 否 | 统一使用采集值的原始类型作为 TDengine 中的数据类型 |
| 8 | quality_col | 数据点位采集值质量在 TDengine 中对应的列名 | 否 | 统一不在 TDengine 添加 quality 列 |
| 9 | ts_col | 数据点位的原始时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col 和 received_ts_col 都非空时，使用前者作为时间戳列；ts_col 和 received_ts_col 有一列非空时，使用不为空的列作时间戳列；ts_col 和 received_ts_col 都为空时，使用数据点位原始时间戳作 TDengine 中的时间戳列，且列名为默认值ts。 |
| 10 | received_ts_col | 接收到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | |
| 11 | ts_transform | 数据点位时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform |
| 12 | received_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform |
| 13 | tag::VARCHAR(200)::name | 数据点位在 TDengine 中对应的 Tag 列。其中`tag` 为保留关键字，表示该列为一个 tag 列；`VARCHAR(200)` 表示该 tag 的类型，也可以是其它合法的类型；`name` 是该 tag 的实际名称。 | 否 | 配置 1 个以上的 tag 列，则使用配置的 tag 列；没有配置任何 tag 列，且 stable 在 TDengine 中存在，使用 TDengine 中的 stable 的 tag；没有配置任何 tag 列，且 stable 在 TDengine 中不存在，则默认自动添加以下 2 个 tag 列：tag::VARCHAR(256)::point_idtag::VARCHAR(256)::point_name |
| 序号 | 列名 | 描述 | 是否必填 | 默认行为 |
|----|-------------------------|-------------------------------------------------------------------------------------------------------------------| -------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 1 | tag_name | 数据点位在 OPC DA 服务器上的 id | 是 | 无 |
| 2 | stable | 数据点位在 TDengine 中对应的超级表 | 是 | 无 |
| 3 | tbname | 数据点位在 TDengine 中对应的子表 | 是 | 无 |
| 4 | enable | 是否采集该点位的数据 | 否 | 使用统一的默认值`1`作为 enable 的值 |
| 5 | value_col | 数据点位采集值在 TDengine 中对应的列名 | 否 | 使用统一的默认值`val` 作为 value_col 的值 |
| 6 | value_transform | 数据点位采集值在 taosX 中执行的变换函数 | 否 | 统一不进行采集值的 transform |
| 7 | type | 数据点位采集值的数据类型 | 否 | 统一使用采集值的原始类型作为 TDengine 中的数据类型 |
| 8 | quality_col | 数据点位采集值质量在 TDengine 中对应的列名 | 否 | 统一不在 TDengine 添加 quality 列 |
| 9 | ts_col | 数据点位的原始时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键 |
| 10 | request_ts_col | 请求该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键 |
| 11 | received_ts_col | 接收到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键 |
| 12 | ts_transform | 数据点位时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform |
| 13 | request_ts_transform | 数据点位请求时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform |
| 14 | received_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform |
| 15 | tag::VARCHAR(200)::name | 数据点位在 TDengine 中对应的 Tag 列。其中`tag` 为保留关键字，表示该列为一个 tag 列；`VARCHAR(200)` 表示该 tag 的类型，也可以是其它合法的类型；`name` 是该 tag 的实际名称。 | 否 | 配置 1 个以上的 tag 列，则使用配置的 tag 列；没有配置任何 tag 列，且 stable 在 TDengine 中存在，使用 TDengine 中的 stable 的 tag；没有配置任何 tag 列，且 stable 在 TDengine 中不存在，则默认自动添加以下 2 个 tag 列：tag::VARCHAR(256)::point_idtag::VARCHAR(256)::point_name |
(2) CSV Header 中，不能有重复的列；
@ -96,21 +98,23 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下
(1) 与 Header 中的列有如下对应关系
| 序号 | Header 中的列 | 值的类型 | 值的范围 | 是否必填 | 默认值 |
| ---- | ----------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------ |
| 1 | tag_name | String | 类似`root.parent.temperature`这样的字符串，要满足 OPC DA 的 ID 规范 | 是 | |
| 2 | enable | int | 0：不采集该点位，且在 OPC DataIn 任务开始前，删除 TDengine 中点位对应的子表；1：采集该点位，在 OPC DataIn 任务开始前，不删除子表。 | 否 | 1 |
| 3 | stable | String | 符合 TDengine 超级表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换如果存在`{type}`，则：CSV 文件的 type 不为空，使用 type 的值进行替换CSV 文件的 type 为空，使用采集值的原始类型进行替换 | 是 | |
| 4 | tbname | String | 符合 TDengine 子表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换对于 OPC UA：如果存在`{ns}`，使用 point_id 中的 ns 替换如果存在`{id}`，使用 point_id 中的 id 替换对于 OPC DA：如果存在`{tag_name}`，使用 tag_name 替换 | 是 | |
| 5 | value_col | String | 符合 TDengine 命名规范的列名 | 否 | val |
| 6 | value_transform | String | 符合 Rhai 引擎的计算表达式，例如：`(val + 10) / 1000 * 2.0`，`log(val) + 10`等； | 否 | None |
| 7 | type | String | 支持类型包括：b/bool/i8/tinyint/i16/smallint/i32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/floatf64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | 否 | 数据点位采集值的原始类型 |
| 8 | quality_col | String | 符合 TDengine 命名规范的列名 | 否 | None |
| 9 | ts_col | String | 符合 TDengine 命名规范的列名 | 否 | ts |
| 10 | received_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts |
| 11 | ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；ts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；ts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None |
| 12 | received_ts_transform | String | 否 | None | |
| 13 | tag::VARCHAR(200)::name | String | tag 里的值，当 tag 的类型是 VARCHAR 时，可以是中文 | 否 | NULL |
| 序号 | Header 中的列 | 值的类型 | 值的范围 | 是否必填 | 默认值 |
|----|-------------------------| -------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------|--------------|
| 1 | tag_name | String | 类似`root.parent.temperature`这样的字符串，要满足 OPC DA 的 ID 规范 | 是 | |
| 2 | enable | int | 0：不采集该点位，且在 OPC DataIn 任务开始前，删除 TDengine 中点位对应的子表；1：采集该点位，在 OPC DataIn 任务开始前，不删除子表。 | 否 | 1 |
| 3 | stable | String | 符合 TDengine 超级表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换如果存在`{type}`，则：CSV 文件的 type 不为空，使用 type 的值进行替换CSV 文件的 type 为空，使用采集值的原始类型进行替换 | 是 | |
| 4 | tbname | String | 符合 TDengine 子表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换对于 OPC UA：如果存在`{ns}`，使用 point_id 中的 ns 替换如果存在`{id}`，使用 point_id 中的 id 替换对于 OPC DA：如果存在`{tag_name}`，使用 tag_name 替换 | 是 | |
| 5 | value_col | String | 符合 TDengine 命名规范的列名 | 否 | val |
| 6 | value_transform | String | 符合 Rhai 引擎的计算表达式，例如：`(val + 10) / 1000 * 2.0`，`log(val) + 10`等； | 否 | None |
| 7 | type | String | 支持类型包括：b/bool/i8/tinyint/i16/smallint/i32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/floatf64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | 否 | 数据点位采集值的原始类型 |
| 8 | quality_col | String | 符合 TDengine 命名规范的列名 | 否 | None |
| 9 | ts_col | String | 符合 TDengine 命名规范的列名 | 否 | ts |
| 10 | request_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | qts |
| 11 | received_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts |
| 12 | ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；ts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；ts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None |
| 13 | request_ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；qts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；qts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None |
| 14 | received_ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；rts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；rts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None |
| 15 | tag::VARCHAR(200)::name | String | tag 里的值，当 tag 的类型是 VARCHAR 时，可以是中文 | 否 | NULL |
(2) tag_name 在整个 DataIn 任务中是唯一的，即：在一个 OPC DataIn 任务中，一个数据点位只能被写入到 TDengine 的一张子表。如果需要将一个数据点位写入多张子表，需要建多个 OPC DataIn 任务；
@ -130,7 +134,7 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下
通过配置 **超级表名**、**表名称**，指定数据要写入的超级表、子表。
配置**主键列**，选择 origin_ts 表示使用 OPC 点位数据的原始时间戳作 TDengine 中的主键；选择 received_ts 表示使用数据的接收时间戳作 TDengine 中的主键。配置**主键别名**，指定 TDengine 时间戳列的名称。
配置**主键列**，选择 origin_ts 表示使用 OPC 点位数据的原始时间戳作 TDengine 中的主键；选择 request_ts 表示使用数据的请求时间戳作 TDengine 中的主键；选择 received_ts 表示使用数据的接收时间戳作 TDengine 中的主键。配置**主键别名**，指定 TDengine 时间戳列的名称。
![point.png](./pic/opcda-06-point.png)

32
docs/zh/06-advanced/05-data-in/07-mqtt.mdx
View File

@ -37,6 +37,14 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T
在 **MQTT 端口** 中填写 MQTT 代理的端口，例如：`1883`
在 **TLS 校验** 中选择 TLS 证书的校验方式
1. 不开启：表示不进行 TLS 证书认证。在连接 MQTT 时，会先进行 TCP 连接，如果连接失败，会进行无证书认证模式的 TLS 连接。
2. 单向认证：开启 TLS 连接，并验证服务端证书，此时需要上传 CA 证书。
3. 双向认证：开启 TLS 连接，并与服务端进行双向认证，此时需要上传 CA 证书，客户端证书以及客户端密钥。
在 **用户** 中填写 MQTT 代理的用户名。
在 **密码** 中填写 MQTT 代理的密码。
@ -44,13 +52,7 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T
![mqtt-03.png](./mqtt-03.png)
### 4. 配置 SSL 证书
如果 MQTT 代理使用了 SSL 证书，需要在 **SSL证书** 中上传证书文件。
![mqtt-04.png](./mqtt-04.png)
### 5. 配置采集信息
### 4. 配置采集信息
在 **采集配置** 区域填写采集任务相关的配置参数。
@ -75,13 +77,13 @@ TDengine 可以通过 MQTT 连接器从 MQTT 代理订阅数据并将其写入 T
![mqtt-05.png](./mqtt-05.png)
### 6. 配置 MQTT Payload 解析
### 5. 配置 MQTT Payload 解析
在 **MQTT Payload 解析** 区域填写 Payload 解析相关的配置参数。
taosX 可以使用 JSON 提取器解析数据，并允许用户在数据库中指定数据模型，包括，指定表名称和超级表名，设置普通列和标签列等。
#### 6.1 解析
#### 5.1 解析
有三种获取示例数据的方法：
@ -112,7 +114,7 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解
![mqtt-07.png](./mqtt-07.png)
#### 6.2 字段拆分
#### 5.2 字段拆分
在 **从列中提取或拆分** 中填写从消息体中提取或拆分的字段，例如：将 message 字段拆分成 `message_0` 和 `message_1` 这2 个字段，选择 split 提取器，seperator 填写 -, number 填写 2。
@ -126,7 +128,7 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解
![mqtt-09.png](./mqtt-09.png)
#### 6.3 数据过滤
#### 5.3 数据过滤
在 **过滤** 中，填写过滤条件，例如：填写`id != 1`，则只有 id 不为 1 的数据才会被写入 TDengine。
@ -138,7 +140,7 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解
![mqtt-11.png](./mqtt-11.png)
#### 6.4 表映射
#### 5.4 表映射
在 **目标超级表** 的下拉列表中选择一个目标超级表，也可以先点击右侧的 **创建超级表** 按钮创建新的超级表。
@ -164,7 +166,7 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解
![mqtt-19.png](./mqtt-19.png)
### 7. 高级选项
### 6. 高级选项
在 **消息等待队列大小** 中填写接收 MQTT 消息的缓存队列大小，当队列满时，新到达的数据会直接丢弃。可设置为 0，即不缓存。
@ -182,12 +184,12 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解
![mqtt-14](./mqtt-14.png)
### 8. 异常处理策略
### 7. 异常处理策略
import Contributing from './_03-exception-handling-strategy.mdx'
<Contributing />
### 9. 创建完成
### 8. 创建完成
点击 **提交** 按钮，完成创建 MQTT 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。

BIN
docs/zh/06-advanced/05-data-in/mqtt-03.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 30 KiB

After

Width:  |  Height:  |  Size: 37 KiB

BIN
docs/zh/06-advanced/05-data-in/mqtt-04.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 24 KiB

9
docs/zh/14-reference/01-components/02-taosc.md
View File

@ -305,6 +305,15 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在
- 动态修改：不支持
- 支持版本：从 v3.1.0.0 版本开始引入
### 流相关
#### streamRunHistoryAsync
- 说明：创建流有 fill_history 参数时，是否异步执行建流语句
- 类型：布尔；false：同步；true：异步
- 默认值：false
- 动态修改：支持通过 SQL 修改，立即生效
- 支持版本：从 v3.3.6.0 版本开始引入
### 日志相关
#### logDir

1
docs/zh/14-reference/02-tools/10-taosbenchmark.md
View File

@ -294,7 +294,6 @@ taosBenchmark -f <json file>
查询指定表（可以指定超级表、子表或普通表）的配置参数在 `specified_table_query` 中设置。
- **mixed_query**：混合查询开关。
“yes”: 开启 “混合查询”。
“no” : 关闭 “混合查询” ，即 “普通查询”。

3
docs/zh/14-reference/03-taos-sql/04-stable.md
View File

@ -21,6 +21,7 @@ table_options:
table_option: {
COMMENT 'string_value'
| SMA(col_name [, col_name] ...)
| KEEP value
}
```
@ -34,6 +35,7 @@ table_option: {
- TAGS 最多允许 128 个，至少 1 个，总长度不超过 16 KB。
4. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考 [按列压缩](../compress)
5. 关于 table_option 中的参数说明，请参考 [建表 SQL 说明](../table)
6. 关于 table_option 中的 keep 参数，仅对超级表生效，keep 参数的详细说明可以参考 [数据库说明](02-database.md)，唯一不同的是超级表 keep 不会立即影响查询结果，仅在 compact 后生效。
## 查看超级表
@ -145,6 +147,7 @@ alter_table_options:
alter_table_option: {
COMMENT 'string_value'
| KEEP value
}
```

10
docs/zh/14-reference/03-taos-sql/06-select.md
View File

@ -276,6 +276,16 @@ TDengine 支持基于时间戳主键的 INNER JOIN，规则如下：
5. JOIN 两侧均支持子查询。
6. 不支持与 FILL 子句混合使用。
## INTERP
interp 子句是 INTERP 函数(../function/#interp)的专用语法，当 SQL 语句中存在 interp 子句时，只能查询 INTERP 函数而不能与其他函数一起查询，同时 interp 子句与窗口子句(window_clause)、分组子句(group_by_clause)也不能同时使用。INTERP 函数在使用时需要与 RANGE、EVERY 和 FILL 子句一起使用；流计算不支持使用 RANGE，但需要与 EVERY 和 FILL 关键字一起使用。
- INTERP 的输出时间范围根据 RANGE(timestamp1, timestamp2) 字段来指定，需满足 timestamp1 \<= timestamp2。其中 timestamp1 为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2 为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。
- INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（time_unit 值）进行插值，time_unit 可取值时间单位：1a(毫秒)、1s(秒)、1m(分)、1h(小时)、1d(天)、1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值。
- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](./distinguished#fill-子句)
- INTERP 可以在 RANGE 字段中只指定唯一的时间戳对单个时间点进行插值，在这种情况下，EVERY 字段可以省略。例如 `SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear)`。
- INTERP 查询支持 NEAR FILL 模式，即当需要 FILL 时，使用距离当前时间点最近的数据进行插值，当前后时间戳与当前时间断面一样近时，FILL 前一行的值. 此模式在流计算中和窗口查询中不支持。例如 `SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)` (v3.3.4.9 及以后支持)。
- INTERP `RANGE`子句支持时间范围的扩展(v3.3.4.9 及以后支持)，如`RANGE('2023-01-01 00:00:00', 10s)`表示在时间点 '2023-01-01 00:00:00' 查找前后 10s 的数据进行插值，FILL PREV/NEXT/NEAR 分别表示从时间点向前/向后/前后查找数据，若时间点周围没有数据，则使用 FILL 指定的值进行插值，因此此时 FILL 子句必须指定值。例如 `SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1)`。目前仅支持时间点和时间范围的组合，不支持时间区间和时间范围的组合，即不支持 `RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h)`。所指定的时间范围规则与 EVERY 类似，单位不能是年或月，值不能为 0，不能带引号。使用该扩展时，不支持除 `FILL PREV/NEXT/NEAR` 外的其他 FILL 模式，且不能指定 EVERY 子句。
## GROUP BY
如果在语句中同时指定了 GROUP BY 子句，那么 SELECT 列表只能包含如下表达式：

64
docs/zh/14-reference/03-taos-sql/10-function.md
View File

@ -1889,42 +1889,6 @@ FIRST(expr)
- 如果结果集中所有列全部为 NULL 值，则不返回结果。
- 对于存在复合主键的表的查询，若最小时间戳的数据有多条，则只有对应的复合主键最小的数据被返回。
### INTERP
```sql
INTERP(expr [, ignore_null_values])
ignore_null_values: {
0
| 1
}
```
**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值，缺省值为 0。
**返回数据类型**：同字段类型。
**适用数据类型**：数值类型。
**适用于**：表和超级表。
**使用说明**
- INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。
- INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。
- INTERP SQL 查询需要同时与 RANGE、EVERY 和 FILL 关键字一起使用；流计算不能使用 RANGE，需要 EVERY 和 FILL 关键字一起使用。
- INTERP 的输出时间范围根据 RANGE(timestamp1, timestamp2) 字段来指定，需满足 timestamp1 \<= timestamp2。其中 timestamp1 为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2 为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。
- INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（time_unit 值）进行插值，time_unit 可取值时间单位：1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)，1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值。
- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](../distinguished/#fill-子句)
- INTERP 可以在 RANGE 字段中只指定唯一的时间戳对单个时间点进行插值，在这种情况下，EVERY 字段可以省略。例如 SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear)。
- INTERP 作用于超级表时，会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。
- INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(v3.0.2.0 以后支持)。
- INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(v3.0.3.0 以后支持)。
- INTERP 对于带复合主键的表的查询，若存在相同时间戳的数据，则只有对应的复合主键最小的数据参与运算。
- INTERP 查询支持 NEAR FILL 模式，即当需要 FILL 时，使用距离当前时间点最近的数据进行插值，当前后时间戳与当前时间断面一样近时，FILL 前一行的值。此模式在流计算中和窗口查询中不支持。例如 SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)(v3.3.4.9 及以后支持)。
- INTERP 只有在使用 FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`。`_irowts_origin` 在 v3.3.4.9 以后支持。
- INTERP `RANGE`子句从 v3.3.4.9 开始支持时间范围的扩展，如 `RANGE('2023-01-01 00:00:00', 10s)` 表示只能使用时间点 '2023-01-01 00:00:00' 周边 10s 内的数据进行插值，FILL PREV/NEXT/NEAR 分别表示从时间点开始向前/向后/前后在时间范围内查找数据，若时间点周边在指定时间范围内没有数据，则使用 FILL 指定的默认值进行插值，因此此时 FILL 子句必须同时指定默认值。例如 SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1)。从 v3.3.6.0 开始支持时间区间和时间范围的组合，对于时间区间内的每个断面进行插值时都需要满足时间范围的要求，在此之前的版本仅支持时间点和时间范围的组合。时间范围的值域规则与 EVERY 类似，单位不能是年或月，值必须大于 0，不能带引号。使用该扩展时，不支持除 `FILL PREV/NEXT/NEAR` 外的其他 FILL 模式。
### LAST
```sql
@ -2187,6 +2151,34 @@ ignore_option: {
- 当没有复合主键时，如果不同的子表有相同时间戳的数据，会提示 "Duplicate timestamps not allowed"。
- 当使用复合主键时，不同子表的时间戳和主键组合可能相同，使用哪一行取决于先找到哪一行，这意味着在这种情况下多次运行 diff() 的结果可能会不同。
### INTERP
```sql
INTERP(expr [, ignore_null_values])
ignore_null_values: {
0
| 1
}
```
**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值，缺省值为 0。
**返回数据类型**：同字段类型。
**适用数据类型**：数值类型。
**适用于**：表和超级表。
**使用说明**
- INTERP 用于在指定时间断面获取指定列的记录值，使用时有专用语法(interp_clause)，语法介绍[参考链接](../select/#interp) 。
- 当指定时间断面不存在符合条件的行数据时，INTERP 函数会根据 [FILL](../distinguished/#fill-子句) 参数的设定进行插值。
- INTERP 作用于超级表时，会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。
- INTERP 可以与伪列 `_irowts` 一起使用，返回插值点所对应的时间戳(v3.0.2.0 以后支持)。
- INTERP 可以与伪列 `_isfilled` 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(v3.0.3.0 以后支持)。
- 只有在使用 FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`, 用于返回 `interp` 函数所使用的原始数据的时间戳列。若范围内无值, 则返回 NULL。`_irowts_origin` 在 v3.3.4.9 以后支持。
- 对于带复合主键的表的查询，若存在相同时间戳的数据，则只有对应的复合主键最小的数据参与运算。
### IRATE
```sql

2
docs/zh/14-reference/09-error-code.md
View File

@ -76,6 +76,8 @@ description: TDengine 服务端的错误码列表和详细说明
| 0x80000134 | Invalid value | 无效值 | 保留现场和日志，github 上报 issue |
| 0x80000135 | Invalid fqdn | 无效 FQDN | 检查配置或输入的 FQDN 值是否正确 |
| 0x8000013C | Invalid disk id | 不合法的 disk id | 建议用户检查挂载磁盘是否失效或者使用参数 diskIDCheckEnabled 来跳过磁盘检查 |
| 0x8000013D | Decimal value overflow | Decimal 计算溢出 | 检查计算表达式和参数值是否计算结果导致类型溢出 |
| 0x8000013E | Division by zero error | Division by zero | 检查除法操作是否除以0 |

4
include/client/taos.h
View File

@ -52,7 +52,8 @@ typedef void TAOS_SUB;
#define TSDB_DATA_TYPE_MEDIUMBLOB 19
#define TSDB_DATA_TYPE_BINARY TSDB_DATA_TYPE_VARCHAR // string
#define TSDB_DATA_TYPE_GEOMETRY 20 // geometry
#define TSDB_DATA_TYPE_MAX 21
#define TSDB_DATA_TYPE_DECIMAL64 21 // decimal64
#define TSDB_DATA_TYPE_MAX 22
typedef enum {
TSDB_OPTION_LOCALE,
@ -270,6 +271,7 @@ DLL_EXPORT int taos_affected_rows(TAOS_RES *res);
DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res);
DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res);
DLL_EXPORT TAOS_FIELD_E *taos_fetch_fields_e(TAOS_RES *res);
DLL_EXPORT int taos_select_db(TAOS *taos, const char *db);
DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
DLL_EXPORT int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);

1
include/common/streamMsg.h
View File

@ -40,6 +40,7 @@ typedef struct SStreamUpstreamEpInfo {
SEpSet epSet;
bool dataAllowed; // denote if the data from this upstream task is allowed to put into inputQ, not serialize it
int64_t stage; // upstream task stage value, to denote if the upstream node has restart/replica changed/transfer
int64_t lastMsgId;
} SStreamUpstreamEpInfo;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo);

2
include/common/tcol.h
View File

@ -79,7 +79,7 @@ uint8_t columnLevelVal(const char* level);
uint8_t columnEncodeVal(const char* encode);
uint16_t columnCompressVal(const char* compress);
bool useCompress(uint8_t tableType);
bool withExtSchema(uint8_t tableType);
bool checkColumnEncode(char encode[TSDB_CL_COMPRESS_OPTION_LEN]);
bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OPTION_LEN]);
bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]);

45
include/common/tcommon.h
View File

@ -132,7 +132,7 @@ enum {
STREAM_INPUT__DATA_SUBMIT = 1,
STREAM_INPUT__DATA_BLOCK,
STREAM_INPUT__MERGED_SUBMIT,
STREAM_INPUT__TQ_SCAN,
STREAM_INPUT__RECALCULATE,
STREAM_INPUT__DATA_RETRIEVE,
STREAM_INPUT__GET_RES,
STREAM_INPUT__CHECKPOINT,
@ -162,18 +162,43 @@ typedef enum EStreamType {
STREAM_GET_RESULT,
STREAM_DROP_CHILD_TABLE,
STREAM_NOTIFY_EVENT,
STREAM_RECALCULATE_DATA,
STREAM_RECALCULATE_DELETE,
STREAM_RECALCULATE_START,
STREAM_RECALCULATE_END,
} EStreamType;
#pragma pack(push, 1)
typedef struct SColumnDataAgg {
int16_t colId;
int32_t colId;
int16_t numOfNull;
int64_t sum;
int64_t max;
int64_t min;
union {
struct {
int64_t sum;
int64_t max;
int64_t min;
};
struct {
uint64_t decimal128Sum[2];
uint64_t decimal128Max[2];
uint64_t decimal128Min[2];
uint8_t overflow;
};
};
} SColumnDataAgg;
#pragma pack(pop)
#define DECIMAL_AGG_FLAG 0x80000000
#define COL_AGG_GET_SUM_PTR(pAggs, dataType) \
(!IS_DECIMAL_TYPE(dataType) ? (void*)&pAggs->sum : (void*)pAggs->decimal128Sum)
#define COL_AGG_GET_MAX_PTR(pAggs, dataType) \
(!IS_DECIMAL_TYPE(dataType) ? (void*)&pAggs->max : (void*)pAggs->decimal128Max)
#define COL_AGG_GET_MIN_PTR(pAggs, dataType) \
(!IS_DECIMAL_TYPE(dataType) ? (void*)&pAggs->min : (void*)pAggs->decimal128Min)
typedef struct SBlockID {
// The uid of table, from which current data block comes. And it is always 0, if current block is the
// result of calculation.
@ -204,8 +229,8 @@ typedef struct SPkInfo {
typedef struct SDataBlockInfo {
STimeWindow window;
int32_t rowSize;
int64_t rows; // todo hide this attribute
uint32_t capacity;
int64_t rows; // todo hide this attribute
SBlockID id;
int16_t hasVarCol;
int16_t dataLoad; // denote if the data is loaded or not
@ -430,6 +455,14 @@ static inline bool isTsmaResSTb(const char* stbName) {
return false;
}
static inline STypeMod typeGetTypeModFromColInfo(const SColumnInfo* pCol) {
return typeGetTypeMod(pCol->type, pCol->precision, pCol->scale, pCol->bytes);
}
static inline STypeMod typeGetTypeModFromCol(const SColumn* pCol) {
return typeGetTypeMod(pCol->type, pCol->precision, pCol->scale, pCol->bytes);
}
#ifdef __cplusplus
}
#endif

9
include/common/tdatablock.h
View File

@ -80,6 +80,9 @@ typedef struct SBlockOrderInfo {
#define IS_JSON_NULL(type, data) \
((type) == TSDB_DATA_TYPE_JSON && (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data)))
#define GET_COL_DATA_TYPE(col) \
{ .type = (col).type, .precision = (col).precision, .bytes = (col).bytes, .scale = (col).scale }
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (!pColumnInfoData->hasNull) {
return false;
@ -174,17 +177,17 @@ static FORCE_INLINE void colDataSetInt32(SColumnInfoData* pColumnInfoData, uint3
static FORCE_INLINE void colDataSetInt64(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, int64_t* v) {
int32_t type = pColumnInfoData->info.type;
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex;
*(int64_t*)p = *(int64_t*)v;
taosSetPInt64Aligned((int64_t*)p, v);
}
static FORCE_INLINE void colDataSetFloat(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, float* v) {
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex;
*(float*)p = *(float*)v;
taosSetPFloatAligned((float*)p, v);
}
static FORCE_INLINE void colDataSetDouble(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, double* v) {
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex;
*(double*)p = *(double*)v;
taosSetPDoubleAligned((double*)p, v);
}
int32_t getJsonValueLen(const char* data);

30
include/common/tdataformat.h
View File

@ -43,6 +43,8 @@ typedef struct SColData SColData;
typedef struct SRowKey SRowKey;
typedef struct SValueColumn SValueColumn;
struct SColumnDataAgg;
typedef struct SColumnDataAgg* SColumnDataAggPtr;
#define HAS_NONE ((uint8_t)0x1)
#define HAS_NULL ((uint8_t)0x2)
@ -95,13 +97,13 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111
#define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL)
#define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE)
#define tRowGetKey(_pRow, _pKey) \
do { \
(_pKey)->ts = (_pRow)->ts; \
(_pKey)->numOfPKs = 0; \
if ((_pRow)->numOfPKs > 0) { \
tRowGetPrimaryKey((_pRow), (_pKey)); \
} \
#define tRowGetKey(_pRow, _pKey) \
do { \
(_pKey)->ts = taosGetInt64Aligned(&((_pRow)->ts)); \
(_pKey)->numOfPKs = 0; \
if ((_pRow)->numOfPKs > 0) { \
tRowGetPrimaryKey((_pRow), (_pKey)); \
} \
} while (0)
// SValueColumn ================================
@ -187,7 +189,7 @@ uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal);
int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg);
void tColDataArrGetRowKey(SColData *aColData, int32_t nColData, int32_t iRow, SRowKey *key);
extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull);
extern void (*tColDataCalcSMA[])(SColData *pColData, SColumnDataAggPtr pAggs);
int32_t tColDataCompress(SColData *colData, SColDataCompressInfo *info, SBuffer *output, SBuffer *assist);
int32_t tColDataDecompress(void *input, SColDataCompressInfo *info, SColData *colData, SBuffer *assist);
@ -244,6 +246,8 @@ typedef struct {
uint32_t offset;
} SPrimaryKeyIndex;
#define DATUM_MAX_SIZE 16
struct SValue {
int8_t type;
union {
@ -255,6 +259,16 @@ struct SValue {
};
};
#define VALUE_GET_DATUM(pVal, type) \
(IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) ? (pVal)->pData : (void*)&(pVal)->val
#define VALUE_GET_TRIVIAL_DATUM(pVal) ((pVal)->val)
#define VALUE_SET_TRIVIAL_DATUM(pVal, v) (pVal)->val = v
void valueSetDatum(SValue *pVal, int8_t type, void *pDatum, uint32_t len);
void valueCloneDatum(SValue *pDst, const SValue *pSrc, int8_t type);
void valueClearDatum(SValue *pVal, int8_t type);
#define TD_MAX_PK_COLS 2
struct SRowKey {
TSKEY ts;

6
include/common/tglobal.h
View File

@ -291,16 +291,22 @@ extern int32_t tsUptimeInterval;
extern bool tsUpdateCacheBatch;
extern bool tsDisableStream;
extern int64_t tsStreamBufferSize;
extern int64_t tsStreamFailedTimeout;
extern int tsStreamAggCnt;
extern bool tsFilterScalarMode;
extern int32_t tsMaxStreamBackendCache;
extern int32_t tsPQSortMemThreshold;
extern bool tsStreamCoverage;
extern bool tsStreamRunHistoryAsync;
extern int8_t tsS3EpNum;
extern int32_t tsStreamNotifyMessageSize;
extern int32_t tsStreamNotifyFrameSize;
extern bool tsCompareAsStrInGreatest;
extern char tsAdapterFqdn[];
extern uint16_t tsAdapterPort;
extern char tsAdapterToken[];
extern bool tsExperimental;
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)

55
include/common/tmsg.h
View File

@ -485,6 +485,14 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_FORECAST_FUNC,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERP_FUNC,
QUERY_NODE_RESET_STREAM_STMT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_SESSION,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_STATE,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_EVENT,
QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_COUNT,
} ENodeType;
typedef struct {
@ -515,6 +523,7 @@ typedef struct SFieldWithOptions {
int8_t flags;
int32_t bytes;
uint32_t compress;
STypeMod typeMod;
} SFieldWithOptions;
typedef struct SRetention {
@ -527,7 +536,6 @@ typedef struct SRetention {
#define RETENTION_VALID(l, r) ((((l) == 0 && (r)->freq >= 0) || ((r)->freq > 0)) && ((r)->keep > 0))
#pragma pack(push, 1)
// null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta
typedef struct SEp {
char fqdn[TSDB_FQDN_LEN];
@ -597,6 +605,7 @@ struct SSchema {
struct SSchemaExt {
col_id_t colId;
uint32_t compress;
STypeMod typeMod;
};
//
@ -660,6 +669,7 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp);
#define COL_SET_NULL ((int8_t)0x10)
#define COL_SET_VAL ((int8_t)0x20)
#define COL_IS_SYSINFO ((int8_t)0x40)
#define COL_HAS_TYPE_MOD ((int8_t)0x80)
#define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0)
#define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL)))
@ -678,6 +688,13 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp);
(s)->flags &= (~COL_IDX_ON); \
} while (0)
#define SSCHEMA_SET_TYPE_MOD(s) \
do { \
(s)->flags |= COL_HAS_TYPE_MOD; \
} while (0)
#define HAS_TYPE_MOD(s) (((s)->flags & COL_HAS_TYPE_MOD))
#define SSCHMEA_TYPE(s) ((s)->type)
#define SSCHMEA_FLAGS(s) ((s)->flags)
#define SSCHMEA_COLID(s) ((s)->colId)
@ -694,6 +711,12 @@ typedef struct {
char tsSlowLogExceptDb[TSDB_DB_NAME_LEN];
} SMonitorParas;
typedef struct {
STypeMod typeMod;
} SExtSchema;
bool hasExtSchema(const SExtSchema* pExtSchema);
typedef struct {
int32_t nCols;
int32_t version;
@ -844,12 +867,14 @@ static FORCE_INLINE int32_t tDecodeSSchema(SDecoder* pDecoder, SSchema* pSchema)
static FORCE_INLINE int32_t tEncodeSSchemaExt(SEncoder* pEncoder, const SSchemaExt* pSchemaExt) {
TAOS_CHECK_RETURN(tEncodeI16v(pEncoder, pSchemaExt->colId));
TAOS_CHECK_RETURN(tEncodeU32(pEncoder, pSchemaExt->compress));
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pSchemaExt->typeMod));
return 0;
}
static FORCE_INLINE int32_t tDecodeSSchemaExt(SDecoder* pDecoder, SSchemaExt* pSchemaExt) {
TAOS_CHECK_RETURN(tDecodeI16v(pDecoder, &pSchemaExt->colId));
TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &pSchemaExt->compress));
TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pSchemaExt->typeMod));
return 0;
}
@ -882,6 +907,7 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp
}
static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) {
if (pSW == NULL) {return TSDB_CODE_INVALID_PARA;}
TAOS_CHECK_RETURN(tEncodeI32v(pEncoder, pSW->nCols));
TAOS_CHECK_RETURN(tEncodeI32v(pEncoder, pSW->version));
for (int32_t i = 0; i < pSW->nCols; i++) {
@ -891,6 +917,7 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSch
}
static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) {
if (pSW == NULL) {return TSDB_CODE_INVALID_PARA;}
TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pSW->nCols));
TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pSW->version));
@ -949,6 +976,7 @@ typedef struct {
int64_t deleteMark2;
int32_t sqlLen;
char* sql;
int64_t keep;
} SMCreateStbReq;
int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
@ -987,6 +1015,8 @@ typedef struct {
char* comment;
int32_t sqlLen;
char* sql;
int64_t keep;
SArray* pTypeMods;
} SMAlterStbReq;
int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAlterStbReq* pReq);
@ -2935,10 +2965,11 @@ typedef struct {
int32_t code;
} STaskDropRsp;
#define STREAM_TRIGGER_AT_ONCE 1
#define STREAM_TRIGGER_WINDOW_CLOSE 2
#define STREAM_TRIGGER_MAX_DELAY 3
#define STREAM_TRIGGER_FORCE_WINDOW_CLOSE 4
#define STREAM_TRIGGER_AT_ONCE 1
#define STREAM_TRIGGER_WINDOW_CLOSE 2
#define STREAM_TRIGGER_MAX_DELAY 3
#define STREAM_TRIGGER_FORCE_WINDOW_CLOSE 4
#define STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE 5
#define STREAM_DEFAULT_IGNORE_EXPIRED 1
#define STREAM_FILL_HISTORY_ON 1
@ -2991,6 +3022,11 @@ typedef struct {
int32_t notifyEventTypes;
int32_t notifyErrorHandle;
int8_t notifyHistory;
int64_t recalculateInterval;
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
} SCMCreateStreamReq;
typedef struct STaskNotifyEventStat {
@ -3241,6 +3277,8 @@ typedef struct SVCreateStbReq {
int8_t source;
int8_t colCmpred;
SColCmprWrapper colCmpr;
int64_t keep;
SExtSchema* pExtSchemas;
} SVCreateStbReq;
int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);
@ -3281,6 +3319,7 @@ typedef struct SVCreateTbReq {
int32_t sqlLen;
char* sql;
SColCmprWrapper colCmpr;
SExtSchema* pExtSchemas;
} SVCreateTbReq;
int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq);
@ -3304,6 +3343,7 @@ static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) {
taosMemoryFreeClear(req->ntb.schemaRow.pSchema);
}
taosMemoryFreeClear(req->colCmpr.pColCmpr);
taosMemoryFreeClear(req->pExtSchemas);
}
typedef struct {
@ -3420,6 +3460,8 @@ typedef struct {
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
uint32_t compress; // TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS
SArray* pMultiTag; // TSDB_ALTER_TABLE_ADD_MULTI_TAGS
// for Add column
STypeMod typeMod;
} SVAlterTbReq;
int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
@ -4233,7 +4275,7 @@ typedef struct {
int8_t rawData;
int32_t minPollRows;
int8_t enableBatchMeta;
SHashObj *uidHash; // to find if uid is duplicated
SHashObj *uidHash; // to find if uid is duplicated
} SMqPollReq;
int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq);
@ -4314,7 +4356,6 @@ typedef struct {
};
void* data; //for free in client, only effected if type is data or metadata. raw data not effected
bool blockDataElementFree; // if true, free blockDataElement in blockData,(true in server, false in client)
} SMqDataRsp;
int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pObj);

2
include/common/tmsgdef.h
View File

@ -130,6 +130,8 @@
TD_DEF_MSG_TYPE(TDMT_MND_UPDATE_ANODE, "update-anode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_ANODE, "drop-anode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_ANAL_ALGO, "retrieve-anal-algo", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_FAILED_STREAM, "create-stream-failed", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CHECK_STREAM_TIMER, "check-stream-status", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_DND_MSG)
TD_NEW_MSG_SEG(TDMT_MND_MSG) // 1<<8

135
include/common/ttypes.h
View File

@ -45,6 +45,9 @@ typedef struct {
} SNCharNullT;
#pragma pack(pop)
#define STypeMod int32_t
void extractTypeFromTypeMod(uint8_t type, STypeMod typeMod, uint8_t *prec, uint8_t* scale, int32_t *bytes);
#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
#define varDataCopy(dst, v) (void)memcpy((dst), (void *)(v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
@ -53,45 +56,74 @@ typedef struct {
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
#define GET_TYPED_DATA(_v, _finalType, _type, _data) \
do { \
switch (_type) { \
case TSDB_DATA_TYPE_BOOL: \
case TSDB_DATA_TYPE_TINYINT: \
(_v) = (_finalType)GET_INT8_VAL(_data); \
break; \
case TSDB_DATA_TYPE_UTINYINT: \
(_v) = (_finalType)GET_UINT8_VAL(_data); \
break; \
case TSDB_DATA_TYPE_SMALLINT: \
(_v) = (_finalType)GET_INT16_VAL(_data); \
break; \
case TSDB_DATA_TYPE_USMALLINT: \
(_v) = (_finalType)GET_UINT16_VAL(_data); \
break; \
case TSDB_DATA_TYPE_TIMESTAMP: \
case TSDB_DATA_TYPE_BIGINT: \
(_v) = (_finalType)(GET_INT64_VAL(_data)); \
break; \
case TSDB_DATA_TYPE_UBIGINT: \
(_v) = (_finalType)(GET_UINT64_VAL(_data)); \
break; \
case TSDB_DATA_TYPE_FLOAT: \
(_v) = (_finalType)GET_FLOAT_VAL(_data); \
break; \
case TSDB_DATA_TYPE_DOUBLE: \
(_v) = (_finalType)GET_DOUBLE_VAL(_data); \
break; \
case TSDB_DATA_TYPE_UINT: \
(_v) = (_finalType)GET_UINT32_VAL(_data); \
break; \
case TSDB_DATA_TYPE_INT: \
(_v) = (_finalType)GET_INT32_VAL(_data); \
break; \
default: \
(_v) = (_finalType)varDataLen(_data); \
break; \
} \
#define DEFINE_TYPE_FROM_DECIMAL_FUNC(oType, decimalType) \
oType oType##From##decimalType(const void* pDec, uint8_t prec, uint8_t scale)
#define DEFINE_TYPE_FROM_DECIMAL_FUNCS(prefix, decimalType) \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(bool, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int8_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint8_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int16_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint16_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int32_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint32_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(int64_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(uint64_t, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(float, decimalType); \
prefix DEFINE_TYPE_FROM_DECIMAL_FUNC(double, decimalType);
DEFINE_TYPE_FROM_DECIMAL_FUNCS(extern, Decimal64);
DEFINE_TYPE_FROM_DECIMAL_FUNCS(extern, Decimal128);
#define GET_TYPED_DATA(_v, _finalType, _type, _data, inputTypeMod) \
do { \
switch (_type) { \
case TSDB_DATA_TYPE_BOOL: \
case TSDB_DATA_TYPE_TINYINT: \
(_v) = (_finalType)GET_INT8_VAL(_data); \
break; \
case TSDB_DATA_TYPE_UTINYINT: \
(_v) = (_finalType)GET_UINT8_VAL(_data); \
break; \
case TSDB_DATA_TYPE_SMALLINT: \
(_v) = (_finalType)GET_INT16_VAL(_data); \
break; \
case TSDB_DATA_TYPE_USMALLINT: \
(_v) = (_finalType)GET_UINT16_VAL(_data); \
break; \
case TSDB_DATA_TYPE_TIMESTAMP: \
case TSDB_DATA_TYPE_BIGINT: \
(_v) = (_finalType)(GET_INT64_VAL(_data)); \
break; \
case TSDB_DATA_TYPE_UBIGINT: \
(_v) = (_finalType)(GET_UINT64_VAL(_data)); \
break; \
case TSDB_DATA_TYPE_FLOAT: \
(_v) = (_finalType)GET_FLOAT_VAL(_data); \
break; \
case TSDB_DATA_TYPE_DOUBLE: \
(_v) = (_finalType)GET_DOUBLE_VAL(_data); \
break; \
case TSDB_DATA_TYPE_UINT: \
(_v) = (_finalType)GET_UINT32_VAL(_data); \
break; \
case TSDB_DATA_TYPE_INT: \
(_v) = (_finalType)GET_INT32_VAL(_data); \
break; \
case TSDB_DATA_TYPE_DECIMAL: { \
uint8_t prec = 0, scale = 0; \
extractTypeFromTypeMod(_type, inputTypeMod, &prec, &scale, NULL); \
(_v) = _finalType##FromDecimal128(_data, prec, scale); \
} break; \
case TSDB_DATA_TYPE_DECIMAL64: { \
uint8_t prec = 0, scale = 0; \
extractTypeFromTypeMod(_type, inputTypeMod, &prec, &scale, NULL); \
(_v) = _finalType##FromDecimal64(_data, prec, scale); \
} break; \
default: \
(_v) = (_finalType)varDataLen(_data); \
break; \
} \
} while (0)
#define SET_TYPED_DATA(_v, _type, _data) \
@ -265,8 +297,9 @@ typedef struct {
#define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)))
#define IS_TIMESTAMP_TYPE(_t) ((_t) == TSDB_DATA_TYPE_TIMESTAMP)
#define IS_BOOLEAN_TYPE(_t) ((_t) == TSDB_DATA_TYPE_BOOL)
#define IS_DECIMAL_TYPE(_t) ((_t) == TSDB_DATA_TYPE_DECIMAL || (_t) == TSDB_DATA_TYPE_DECIMAL64)
#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)))
#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)) || (IS_DECIMAL_TYPE(_t)))
#define IS_MATHABLE_TYPE(_t) \
(IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
@ -364,6 +397,13 @@ typedef struct tDataTypeCompress {
extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX];
extern tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX];
typedef struct SDataType {
uint8_t type;
uint8_t precision;
uint8_t scale;
int32_t bytes;
} SDataType;
bool isValidDataType(int32_t type);
int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type);
@ -371,6 +411,21 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type);
void *getDataMin(int32_t type, void *value);
void *getDataMax(int32_t type, void *value);
STypeMod typeGetTypeMod(uint8_t type, uint8_t prec, uint8_t scale, int32_t bytes);
STypeMod typeGetTypeModFromDataType(const SDataType* pDataType);
uint8_t decimalTypeFromPrecision(uint8_t precision);
STypeMod decimalCalcTypeMod(uint8_t prec, uint8_t scale);
void decimalFromTypeMod(STypeMod typeMod, uint8_t *precision, uint8_t *scale);
// pType->type should has been set
void fillTypeFromTypeMod(SDataType *pType, STypeMod mod);
uint8_t getScaleFromTypeMod(int32_t type, STypeMod mod);
// TODO fix me!! for compatibility issue, save precision in scale in bytes, move it to somewhere else
void fillBytesForDecimalType(int32_t *pBytes, int32_t type, uint8_t precision, uint8_t scale);
void extractDecimalTypeInfoFromBytes(int32_t *pBytes, uint8_t *precision, uint8_t *scale);
int32_t calcTypeBytesFromSchemaBytes(int32_t type, int32_t schemaBytes, bool isStmt);
int32_t calcSchemaBytesFromTypeBytes(int32_t type, int32_t varTypeBytes, bool isStmt);
#ifdef __cplusplus
}
#endif

143
include/libs/decimal/decimal.h Normal file
View File

@ -0,0 +1,143 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_DECIMAL_H_
#define _TD_DECIMAL_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "tdef.h"
#include "ttypes.h"
typedef struct SValue SValue;
typedef void DecimalType;
typedef struct Decimal64 {
DecimalWord words[1]; // do not touch it directly, use DECIMAL64_GET_VALUE MACRO
} Decimal64;
#define DECIMAL64_GET_VALUE(pDec) (int64_t)((pDec)->words[0])
#define DECIMAL64_SET_VALUE(pDec, val) (*(int64_t*)((pDec)->words)) = (int64_t)(val)
#define DECIMAL64_CLONE(pDst, pFrom) ((Decimal64*)(pDst))->words[0] = ((Decimal64*)(pFrom))->words[0]
static const Decimal64 decimal64Zero = {0};
static const Decimal64 decimal64Two = {2};
static const Decimal64 decimal64Min = {(uint64_t)-999999999999999999LL};
static const Decimal64 decimal64Max = {(uint64_t)999999999999999999LL};
#define DECIMAL64_ZERO decimal64Zero
#define DECIMAL64_MAX decimal64Max
#define DECIMAL64_MIN decimal64Min
typedef struct Decimal128 {
DecimalWord words[2]; // do not touch it directly, use DECIMAL128_HIGH_WORD/DECIMAL128_LOW_WORD
} Decimal128;
#define Decimal Decimal128
#define decimalFromStr decimal128FromStr
#define makeDecimal makeDecimal128
#define DEFINE_DECIMAL128(lo, hi) {lo, hi}
static const Decimal128 decimal128Zero = DEFINE_DECIMAL128(0, 0);
static const Decimal128 decimal128Two = DEFINE_DECIMAL128(2, 0);
static const Decimal128 decimal128Max = DEFINE_DECIMAL128(687399551400673280ULL - 1, 5421010862427522170LL);
static const Decimal128 decimal128Min = DEFINE_DECIMAL128(17759344522308878337ULL, 13025733211282029445ULL);
#define DECIMAL128_LOW_WORD(pDec) (uint64_t)((pDec)->words[0])
#define DECIMAL128_SET_LOW_WORD(pDec, val) (pDec)->words[0] = val
#define DECIMAL128_HIGH_WORD(pDec) (int64_t)((pDec)->words[1])
#define DECIMAL128_SET_HIGH_WORD(pDec, val) *(int64_t*)((pDec)->words + 1) = val
#define DECIMAL128_ZERO decimal128Zero
#define DECIMAL128_MAX decimal128Max
#define DECIMAL128_MIN decimal128Min
#define DECIMAL128_CLONE(pDst, pFrom) makeDecimal128(pDst, DECIMAL128_HIGH_WORD(pFrom), DECIMAL128_LOW_WORD(pFrom))
typedef struct SDecimalCompareCtx {
const void* pData;
int8_t type;
STypeMod typeMod;
} SDecimalCompareCtx;
void makeDecimal64(Decimal64* pDec64, int64_t w);
void makeDecimal128(Decimal128* pDec128, int64_t hi, uint64_t low);
void decimalFromTypeMod(STypeMod typeMod, uint8_t* precision, uint8_t* scale);
int32_t decimal64FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale, Decimal64* result);
int32_t decimal128FromStr(const char* str, int32_t len, uint8_t expectPrecision, uint8_t expectScale,
Decimal128* result);
int32_t decimal64ToDataVal(const Decimal64* dec, SValue* pVal);
int32_t decimal128ToDataVal(Decimal128* dec, SValue* pVal);
int32_t decimalToStr(const DecimalType* pDec, int8_t type, int8_t precision, int8_t scale, char* pBuf, int32_t bufLen);
int32_t decimalGetRetType(const SDataType* pLeftT, const SDataType* pRightT, EOperatorType opType, SDataType* pOutType);
bool decimal64Compare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDecimalCompareCtx* pRight);
bool decimalCompare(EOperatorType op, const SDecimalCompareCtx* pLeft, const SDecimalCompareCtx* pRight);
int32_t decimalOp(EOperatorType op, const SDataType* pLeftT, const SDataType* pRightT, const SDataType* pOutT,
const void* pLeftData, const void* pRightData, void* pOutputData);
int32_t convertToDecimal(const void* pData, const SDataType* pInputType, void* pOut, const SDataType* pOutType);
bool decimal128AddCheckOverflow(const Decimal128* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
void encodeDecimal(const DecimalType* pDec, int8_t type, void* pBuf);
void decodeDecimal(const void* pBuf, int8_t type, DecimalType* pDec);
DEFINE_TYPE_FROM_DECIMAL_FUNCS(, Decimal64);
DEFINE_TYPE_FROM_DECIMAL_FUNCS(, Decimal128);
// word num use DECIMAL_WORD_NUM(Decimal64) or DECIMAL_WORD_NUM(Decimal128)
typedef struct SDecimalOps {
void (*negate)(DecimalType* pWord);
void (*abs)(DecimalType* pWord);
void (*add)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
void (*subtract)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
void (*multiply)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
void (*divide)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum, DecimalType* pRemainder);
void (*mod)(DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
bool (*lt)(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
bool (*gt)(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
bool (*eq)(const DecimalType* pLeft, const DecimalType* pRight, uint8_t rightWordNum);
int32_t (*toStr)(const DecimalType* pInt, uint8_t scale, char* pBuf, int32_t bufLen);
} SDecimalOps;
// all these ops only used for operations on decimal types with same scale
const SDecimalOps* getDecimalOps(int8_t dataType);
#if 0
__int128 decimal128ToInt128(const Decimal128* pDec);
#endif
int32_t TEST_decimal64From_int64_t(Decimal64* pDec, uint8_t prec, uint8_t scale, int64_t v);
int32_t TEST_decimal64From_uint64_t(Decimal64* pDec, uint8_t prec, uint8_t scale, uint64_t v);
int32_t TEST_decimal64From_double(Decimal64* pDec, uint8_t prec, uint8_t scale, double v);
double TEST_decimal64ToDouble(Decimal64* pDec, uint8_t prec, uint8_t scale);
int32_t TEST_decimal64FromDecimal64(const Decimal64* pInput, uint8_t inputPrec, uint8_t inputScale, Decimal64* pOutput,
uint8_t outputPrec, uint8_t outputScale);
int32_t TEST_decimal64FromDecimal128(const Decimal128* pInput, uint8_t prec, uint8_t scale, Decimal64* pOutput,
uint8_t outputPrec, uint8_t outputScale);
int32_t TEST_decimal128From_int64_t(Decimal128* pDec, uint8_t prec, uint8_t scale, int64_t v);
int32_t TEST_decimal128From_uint64_t(Decimal128* pDec, uint8_t prec, uint8_t scale, uint64_t v);
int32_t TEST_decimal128From_double(Decimal128* pDec, uint8_t prec, uint8_t scale, double v);
double TEST_decimal128ToDouble(Decimal128* pDec, uint8_t prec, uint8_t scale);
int32_t TEST_decimal128FromDecimal64(const Decimal64* pInput, uint8_t inputPrec, uint8_t inputScale,
Decimal128* pOutput, uint8_t outputPrec, uint8_t outputScale);
int32_t TEST_decimal128FromDecimal128(const Decimal128* pDec, uint8_t prec, uint8_t scale, Decimal128* pOutput,
uint8_t outputPrec, uint8_t outputScale);
#ifdef __cplusplus
}
#endif
#endif /*_TD_DECIMAL_H_*/

6
include/libs/executor/executor.h
View File

@ -53,6 +53,7 @@ typedef struct {
int32_t numOfVgroups;
void* sContext; // SSnapContext*
void* pStateBackend;
void* pOtherBackend;
int8_t fillHistory;
STimeWindow winRange;
@ -219,8 +220,7 @@ const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
const char* qExtractTbnameFromTask(qTaskInfo_t tinfo);
void* qExtractReaderFromStreamScanner(void* scanner);
void qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner);
void qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner);
int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo);
int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow);
@ -229,7 +229,7 @@ int32_t qStreamRecoverFinish(qTaskInfo_t tinfo);
bool qStreamScanhistoryFinished(qTaskInfo_t tinfo);
int32_t qStreamInfoResetTimewindowFilter(qTaskInfo_t tinfo);
void qResetTaskInfoCode(qTaskInfo_t tinfo);
int32_t qGetStreamIntervalExecInfo(qTaskInfo_t tinfo, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow);
int32_t qGetStreamIntervalExecInfo(qTaskInfo_t tinfo, int64_t* pWaterMark, SInterval* pInterval, STimeWindow* pLastWindow, TSKEY* pRecInteral);
int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo);
int32_t qStreamOperatorReloadState(qTaskInfo_t tInfo);

91
include/libs/executor/storageapi.h
View File

@ -58,6 +58,7 @@ typedef struct SMetaEntry {
SSchemaWrapper schemaRow;
SSchemaWrapper schemaTag;
SRSmaParam rsmaParam;
int64_t keep;
} stbEntry;
struct {
int64_t btime;
@ -83,6 +84,7 @@ typedef struct SMetaEntry {
uint8_t* pBuf;
SColCmprWrapper colCmpr; // col compress alg
SExtSchema* pExtSchemas;
} SMetaEntry;
typedef struct SMetaReader {
@ -125,6 +127,7 @@ typedef struct SRowBuffPos {
bool beUsed;
bool needFree;
bool beUpdated;
bool invalid;
} SRowBuffPos;
// tq
@ -135,6 +138,11 @@ typedef struct SMetaTableInfo {
char tbName[TSDB_TABLE_NAME_LEN];
} SMetaTableInfo;
static FORCE_INLINE void destroyMetaTableInfo(SMetaTableInfo* mtInfo){
if (mtInfo == NULL) return;
tDeleteSchemaWrapper(mtInfo->schema);
}
typedef struct SSnapContext {
struct SMeta* pMeta;
int64_t snapVersion;
@ -154,6 +162,7 @@ typedef struct {
int64_t uid;
int64_t ctbNum;
int32_t colNum;
int64_t keep;
} SMetaStbStats;
// clang-format off
@ -199,7 +208,9 @@ typedef struct TsdReader {
int32_t (*fileSetReadNext)(struct SFileSetReader *);
int32_t (*fileSetGetEntryField)(struct SFileSetReader *, const char *, void *);
void (*fileSetReaderClose)(struct SFileSetReader **);
int32_t (*getProgress)(const void* pReader, void** pBuf, uint64_t* pLen);
int32_t (*setProgress)(void *pReader, const void *pBuf, uint64_t len);
} TsdReader;
typedef struct SStoreCacheReader {
@ -318,7 +329,7 @@ typedef struct SUpdateInfo {
TSKEY minTS;
SScalableBf* pCloseWinSBF;
SHashObj* pMap;
uint64_t maxDataVersion;
int64_t maxDataVersion;
int8_t pkColType;
int32_t pkColLen;
char* pKeyBuff;
@ -328,6 +339,27 @@ typedef struct SUpdateInfo {
__compar_fn_t comparePkCol;
} SUpdateInfo;
typedef struct SRecDataInfo {
STimeWindow calWin;
uint64_t tableUid;
int64_t dataVersion;
EStreamType mode;
char pPkColData[];
} SRecDataInfo;
typedef struct SScanRange {
STimeWindow win;
STimeWindow calWin;
SSHashObj* pGroupIds;
SSHashObj* pUIds;
} SScanRange;
typedef struct SResultWindowInfo {
SRowBuffPos* pStatePos;
SSessionKey sessionWin;
bool isOutput;
} SResultWindowInfo;
typedef struct {
void* iter; // rocksdb_iterator_t* iter;
void* snapshot; // rocksdb_snapshot_t* snapshot;
@ -342,23 +374,44 @@ typedef struct {
int64_t minGpId;
} SStreamStateCur;
typedef struct STableTsDataState {
SSHashObj* pTableTsDataMap;
__compar_fn_t comparePkColFn;
void* pPkValBuff;
int32_t pkValLen;
SStreamState* pState;
int32_t curRecId;
void* pStreamTaskState;
SArray* pScanRanges;
SRecDataInfo* pRecValueBuff;
int32_t recValueLen;
SStreamStateCur* pRecCur;
int32_t cfgIndex;
void* pBatch;
int32_t batchBufflen;
void* pBatchBuff;
} STableTsDataState;
typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache,
int32_t* pWinCode);
int32_t (*streamStateDeleteParName)(SStreamState* pState, int64_t groupId);
void (*streamStateSetParNameInvalid)(SStreamState* pState);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used);
void (*streamStateClearBuff)(SStreamState* pState, void* pVal);
void (*streamStateFreeVal)(void* val);
int32_t (*streamStateGetPrev)(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateGetAllPrev)(SStreamState* pState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key, bool hasLimit, bool* pIsLast);
bool (*streamStateCheckSessionState)(SStreamState* pState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal);
void (*streamStateDel)(SStreamState* pState, const SWinKey* key);
void (*streamStateDelByGroupId)(SStreamState* pState, uint64_t groupId);
@ -366,6 +419,8 @@ typedef struct SStateStore {
void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex);
void (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t (*streamStateGetNumber)(SStreamState* pState);
int32_t (*streamStateDeleteInfo)(SStreamState* pState, void* pKey, int32_t keyLen);
int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
@ -390,7 +445,10 @@ typedef struct SStateStore {
int32_t (*streamStateFillGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
void (*streamStateClearExpiredState)(SStreamState* pState);
void (*streamStateClearExpiredState)(SStreamState* pState, int32_t numOfKeep, TSKEY minTs);
void (*streamStateClearExpiredSessionState)(SStreamState* pState, int32_t numOfKeep, TSKEY minTs, SSHashObj* pFlushGroup);
int32_t (*streamStateSetRecFlag)(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t mode);
int32_t (*streamStateGetRecFlag)(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t* pMode);
int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
@ -407,6 +465,9 @@ typedef struct SStateStore {
int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionSaveToDisk)(STableTsDataState* pTblState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
int32_t (*streamStateFlushReaminInfoToDisk)(STableTsDataState* pTblState);
int32_t (*streamStateSessionDeleteAll)(SStreamState* pState);
int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount,
void** ppVal, int32_t* pVLen, int32_t* pWinCode);
@ -452,12 +513,32 @@ typedef struct SStateStore {
bool (*needClearDiskBuff)(struct SStreamFileState* pFileState);
SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
SStreamState* (*streamStateRecalatedOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState);
void (*streamStateCommit)(SStreamState* pState);
void (*streamStateDestroy)(SStreamState* pState, bool remove);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
int32_t (*streamStateGetAndSetTsData)(STableTsDataState* pState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
int32_t (*streamStateTsDataCommit)(STableTsDataState* pState);
int32_t (*streamStateInitTsDataState)(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState, void* pOtherState);
void (*streamStateDestroyTsDataState)(STableTsDataState* pTsDataState);
int32_t (*streamStateRecoverTsData)(STableTsDataState* pTsDataState);
int32_t (*streamStateReloadTsDataState)(STableTsDataState* pTsDataState);
int32_t (*streamStateMergeAndSaveScanRange)(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId,
SRecDataInfo* pRecData, int32_t len);
int32_t (*streamStateMergeAllScanRange)(STableTsDataState* pTsDataState);
int32_t (*streamStatePopScanRange)(STableTsDataState* pTsDataState, SScanRange* pRange);
SStreamStateCur* (*streamStateGetLastStateCur)(SStreamState* pState);
void (*streamStateLastStateCurNext)(SStreamStateCur* pCur);
int32_t (*streamStateNLastStateGetKVByCur)(SStreamStateCur* pCur, int32_t num, SArray* pRes);
SStreamStateCur* (*streamStateGetLastSessionStateCur)(SStreamState* pState);
void (*streamStateLastSessionStateCurNext)(SStreamStateCur* pCur);
int32_t (*streamStateNLastSessionStateGetKVByCur)(SStreamStateCur* pCur, int32_t num, SArray* pRes);
} SStateStore;
typedef struct SStorageAPI {

86
include/libs/function/function.h
View File

@ -20,10 +20,10 @@
extern "C" {
#endif
#include "functionResInfo.h"
#include "tcommon.h"
#include "tsimplehash.h"
#include "tvariant.h"
#include "functionResInfo.h"
struct SqlFunctionCtx;
struct SResultRowEntryInfo;
@ -38,14 +38,15 @@ typedef struct SFuncExecEnv {
} SFuncExecEnv;
typedef bool (*FExecGetEnv)(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
typedef void (*FExecCleanUp)(struct SqlFunctionCtx* pCtx);
typedef void (*FExecCleanUp)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef int32_t (*FExecCombine)(struct SqlFunctionCtx *pDestCtx, struct SqlFunctionCtx *pSourceCtx);
typedef int32_t (*FExecDecode)(struct SqlFunctionCtx *pCtx, const char *buf, struct SResultRowEntryInfo *pResultCellInfo, int32_t version);
typedef int32_t (*processFuncByRow)(SArray* pCtx); // array of SqlFunctionCtx
typedef int32_t (*FExecDecode)(struct SqlFunctionCtx *pCtx, const char *buf,
struct SResultRowEntryInfo *pResultCellInfo, int32_t version);
typedef int32_t (*processFuncByRow)(SArray *pCtx); // array of SqlFunctionCtx
typedef struct SScalarFuncExecFuncs {
FExecGetEnv getEnv;
@ -119,8 +120,8 @@ typedef struct SInputColumnInfoData {
int32_t startRowIndex; // handle started row index
int64_t numOfRows; // the number of rows needs to be handled
bool blankFill; // fill blank data to block for empty table
int32_t numOfInputCols; // PTS is not included
bool colDataSMAIsSet; // if agg is set or not
int32_t numOfInputCols; // PTS is not included
SColumnInfoData *pPTS; // primary timestamp column
SColumnInfoData *pPrimaryKey; // primary key column
SColumnInfoData **pData;
@ -131,7 +132,7 @@ typedef struct SInputColumnInfoData {
typedef struct SSerializeDataHandle {
struct SDiskbasedBuf *pBuf;
int32_t currentPage;
SStreamState *pState;
SStreamState *pState;
} SSerializeDataHandle;
// incremental state storage
@ -168,11 +169,14 @@ typedef struct STdbState {
void *pParNameDb;
void *pParTagDb;
void *txn;
int8_t recalc;
} STdbState;
typedef struct SResultRowStore {
int32_t (*resultRowPut)(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
int32_t (*resultRowGet)(struct SExprSupp *pSup, const char* inBuf, size_t inBufSize, char **outBuf, size_t *outBufSize);
int32_t (*resultRowPut)(struct SExprSupp *pSup, const char *inBuf, size_t inBufSize, char **outBuf,
size_t *outBufSize);
int32_t (*resultRowGet)(struct SExprSupp *pSup, const char *inBuf, size_t inBufSize, char **outBuf,
size_t *outBufSize);
} SResultRowStore;
struct SStreamState {
@ -187,6 +191,7 @@ struct SStreamState {
int32_t tsIndex;
SResultRowStore pResultRowStore;
struct SExprSupp *pExprSupp;
char pTaskIdStr[65];
};
typedef struct SFunctionStateStore {
@ -196,37 +201,37 @@ typedef struct SFunctionStateStore {
typedef struct SFuncInputRow {
TSKEY ts;
bool isDataNull;
char* pData;
char* pPk;
bool isDataNull;
char *pData;
char *pPk;
SSDataBlock* block; // prev row block or src block
int32_t rowIndex; // prev row block ? 0 : rowIndex in srcBlock
SSDataBlock *block; // prev row block or src block
int32_t rowIndex; // prev row block ? 0 : rowIndex in srcBlock
//TODO:
// int32_t startOffset; // for diff, derivative
// SPoint1 startPoint; // for twa
// TODO:
// int32_t startOffset; // for diff, derivative
// SPoint1 startPoint; // for twa
} SFuncInputRow;
typedef struct SFuncInputRowIter {
bool hasPrev;
SInputColumnInfoData* pInput;
SColumnInfoData* pDataCol;
SColumnInfoData* pPkCol;
TSKEY* tsList;
int32_t rowIndex;
int32_t inputEndIndex;
SSDataBlock* pSrcBlock;
bool hasPrev;
TSKEY prevBlockTsEnd;
bool prevIsDataNull;
char* pPrevData;
char* pPrevPk;
SSDataBlock* pPrevRowBlock; // pre one row block
SInputColumnInfoData *pInput;
SColumnInfoData *pDataCol;
SColumnInfoData *pPkCol;
TSKEY *tsList;
int32_t rowIndex;
int32_t inputEndIndex;
SSDataBlock *pSrcBlock;
TSKEY prevBlockTsEnd;
bool prevIsDataNull;
char *pPrevData;
char *pPrevPk;
SSDataBlock *pPrevRowBlock; // pre one row block
uint64_t groupId;
bool hasGroupId;
bool hasGroupId;
bool finalRow;
} SFuncInputRowIter;
@ -263,8 +268,9 @@ typedef struct SqlFunctionCtx {
bool hasPrimaryKey;
SFuncInputRowIter rowIter;
bool bInputFinished;
bool hasWindowOrGroup; // denote that the function is used with time window or group
bool needCleanup; // denote that the function need to be cleaned up
bool hasWindowOrGroup; // denote that the function is used with time window or group
bool needCleanup; // denote that the function need to be cleaned up
int32_t inputType; // save the fuction input type funcs like finalize
} SqlFunctionCtx;
typedef struct tExprNode {
@ -291,21 +297,23 @@ struct SScalarParam {
SColumnInfoData *columnData;
SHashObj *pHashFilter;
SHashObj *pHashFilterOthers;
int32_t hashValueType;
int32_t filterValueType;
void *param; // other parameter, such as meta handle from vnode, to extract table name/tag value
int32_t numOfRows;
int32_t numOfQualified; // number of qualified elements in the final results
timezone_t tz;
void *charsetCxt;
SArray *pFilterArr; // for types that can't filter with hash
STypeMod filterValueTypeMod;
};
static inline void setTzCharset(SScalarParam* param, timezone_t tz, void* charsetCxt){
static inline void setTzCharset(SScalarParam *param, timezone_t tz, void *charsetCxt) {
if (param == NULL) return;
param->tz = tz;
param->charsetCxt = charsetCxt;
}
#define cleanupResultRowEntry(p) p->initialized = false
#define cleanupResultRowEntry(p) p->initialized = false
#define isRowEntryCompleted(p) (p->complete)
#define isRowEntryInitialized(p) (p->initialized)
@ -315,11 +323,11 @@ typedef struct SPoint {
} SPoint;
void taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2,
int32_t inputType);
int32_t inputType, STypeMod inputTypeMod);
#define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25
#define LEASTSQUARES_BUFF_LENGTH 128
#define DOUBLE_PRECISION_DIGITS "16e"
#define LEASTSQUARES_BUFF_LENGTH 128
#define DOUBLE_PRECISION_DIGITS "16e"
#ifdef __cplusplus
}

3
include/libs/function/functionMgt.h
View File

@ -159,6 +159,8 @@ typedef enum EFunctionType {
FUNCTION_TYPE_FORECAST_ROWTS,
FUNCTION_TYPE_COLS,
FUNCTION_TYPE_IROWTS_ORIGIN,
FUNCTION_TYPE_GROUP_ID,
FUNCTION_TYPE_IS_WINDOW_FILLED,
// internal function
FUNCTION_TYPE_SELECT_VALUE = 3750,
@ -300,6 +302,7 @@ bool fmIsElapsedFunc(int32_t funcId);
bool fmIsDBUsageFunc(int32_t funcId);
bool fmIsRowTsOriginFunc(int32_t funcId);
bool fmIsSelectColsFunc(int32_t funcId);
bool fmIsGroupIdFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);

2
include/libs/geometry/geomFunc.h
View File

@ -20,6 +20,7 @@
extern "C" {
#endif
#ifdef USE_GEOS
#include "function.h"
int32_t makePointFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
@ -33,6 +34,7 @@ int32_t touchesFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pO
int32_t coversFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t containsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t containsProperlyFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
#endif
#ifdef __cplusplus
}

8
include/libs/geometry/geosWrapper.h
View File

@ -23,6 +23,7 @@ extern "C" {
#include <stdint.h>
#include "os.h"
#ifdef USE_GEOS
#include "tgeosctx.h"
void geosFreeBuffer(void *buffer);
@ -55,6 +56,13 @@ int32_t readGeometry(const unsigned char *input, GEOSGeometry **outputGeom,
const GEOSPreparedGeometry **outputPreparedGeom);
void destroyGeometry(GEOSGeometry **geom, const GEOSPreparedGeometry **preparedGeom);
#else
int32_t initCtxAsText();
int32_t doAsText(const unsigned char *inputGeom, size_t size, char **outputWKT);
int32_t checkWKB(const unsigned char *wkb, size_t size);
#endif
#ifdef __cplusplus
}
#endif

10
include/libs/nodes/cmdnodes.h
View File

@ -55,9 +55,10 @@ extern "C" {
#define SHOW_ALIVE_RESULT_COLS 1
#define BIT_FLAG_MASK(n) (1 << n)
#define BIT_FLAG_SET_MASK(val, mask) ((val) |= (mask))
#define BIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
#define BIT_FLAG_MASK(n) (1 << n)
#define BIT_FLAG_SET_MASK(val, mask) ((val) |= (mask))
#define BIT_FLAG_UNSET_MASK(val, mask) ((val) &= ~(mask))
#define BIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
#define PRIVILEGE_TYPE_ALL BIT_FLAG_MASK(0)
#define PRIVILEGE_TYPE_READ BIT_FLAG_MASK(1)
@ -197,6 +198,8 @@ typedef struct STableOptions {
SNodeList* pRollupFuncs;
int32_t ttl;
SNodeList* pSma;
SValueNode* pKeepNode;
int32_t keep;
} STableOptions;
typedef struct SColumnOptions {
@ -563,6 +566,7 @@ typedef struct SStreamOptions {
SNode* pDelay;
SNode* pWatermark;
SNode* pDeleteMark;
SNode* pRecInterval;
int8_t fillHistory;
int8_t ignoreExpired;
int8_t ignoreUpdate;

13
include/libs/nodes/plannodes.h
View File

@ -316,6 +316,12 @@ typedef enum EWindowAlgorithm {
SESSION_ALGO_STREAM_SINGLE,
SESSION_ALGO_MERGE,
INTERVAL_ALGO_STREAM_MID,
INTERVAL_ALGO_STREAM_CONTINUE_SINGLE,
INTERVAL_ALGO_STREAM_CONTINUE_FINAL,
INTERVAL_ALGO_STREAM_CONTINUE_SEMI,
SESSION_ALGO_STREAM_CONTINUE_SINGLE,
SESSION_ALGO_STREAM_CONTINUE_FINAL,
SESSION_ALGO_STREAM_CONTINUE_SEMI,
} EWindowAlgorithm;
typedef struct SWindowLogicNode {
@ -347,6 +353,7 @@ typedef struct SWindowLogicNode {
SNodeList* pTsmaSubplans;
SNode* pAnomalyExpr;
char anomalyOpt[TSDB_ANALYTIC_ALGO_OPTION_LEN];
int64_t recalculateInterval;
} SWindowLogicNode;
typedef struct SFillLogicNode {
@ -508,6 +515,11 @@ typedef struct STableScanPhysiNode {
bool needCountEmptyTable;
bool paraTablesSort;
bool smallDataTsSort;
char pStbFullName[TSDB_TABLE_FNAME_LEN];
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
} STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode;
@ -683,6 +695,7 @@ typedef struct SWindowPhysiNode {
int8_t igExpired;
int8_t destHasPrimaryKey;
bool mergeDataBlock;
int64_t recalculateInterval;
} SWindowPhysiNode;
typedef struct SIntervalPhysiNode {

28
include/libs/nodes/querynodes.h
View File

@ -25,6 +25,7 @@ extern "C" {
#include "tmsg.h"
#include "tsimplehash.h"
#include "tvariant.h"
#include "ttypes.h"
#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
#define TABLE_META_SIZE(pMeta) \
@ -45,13 +46,6 @@ typedef struct SRawExprNode {
bool isPseudoColumn;
} SRawExprNode;
typedef struct SDataType {
uint8_t type;
uint8_t precision;
uint8_t scale;
int32_t bytes;
} SDataType;
typedef struct SExprNode {
ENodeType type;
SDataType resType;
@ -76,7 +70,8 @@ typedef enum EColumnType {
COLUMN_TYPE_WINDOW_START,
COLUMN_TYPE_WINDOW_END,
COLUMN_TYPE_WINDOW_DURATION,
COLUMN_TYPE_GROUP_KEY
COLUMN_TYPE_GROUP_KEY,
COLUMN_TYPE_IS_WINDOW_FILLED,
} EColumnType;
typedef struct SColumnNode {
@ -203,6 +198,8 @@ typedef struct SFunctionNode {
bool dual; // whether select stmt without from stmt, true for without.
timezone_t tz;
void *charsetCxt;
const struct SFunctionNode* pSrcFuncRef;
SDataType srcFuncInputType;
} SFunctionNode;
typedef struct STableNode {
@ -440,6 +437,7 @@ typedef struct SRangeAroundNode {
typedef struct SSelectStmt {
ENodeType type; // QUERY_NODE_SELECT_STMT
bool isDistinct;
STimeWindow timeRange;
SNodeList* pProjectionList;
SNodeList* pProjectionBindList;
SNode* pFromTable;
@ -457,7 +455,6 @@ typedef struct SSelectStmt {
SNodeList* pOrderByList; // SOrderByExprNode
SLimitNode* pLimit;
SLimitNode* pSlimit;
STimeWindow timeRange;
SNodeList* pHint;
char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision;
@ -638,23 +635,24 @@ typedef struct SQuery {
ENodeType type;
EQueryExecStage execStage;
EQueryExecMode execMode;
int32_t msgType;
int32_t numOfResCols;
int32_t placeholderNum;
int8_t precision;
bool haveResultSet;
bool showRewrite;
bool stableQuery;
SNode* pPrevRoot;
SNode* pRoot;
SNode* pPostRoot;
int32_t numOfResCols;
SSchema* pResSchema;
int8_t precision;
SCmdMsgInfo* pCmdMsg;
int32_t msgType;
SArray* pTargetTableList;
SArray* pTableList;
SArray* pDbList;
bool showRewrite;
int32_t placeholderNum;
SArray* pPlaceholderValues;
SNode* pPrepareRoot;
bool stableQuery;
SExtSchema* pResExtSchema;
} SQuery;
void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);

10
include/libs/parser/parser.h
View File

@ -103,6 +103,7 @@ typedef struct SParseContext {
setQueryFn setQueryFp;
timezone_t timezone;
void *charsetCxt;
bool streamRunHistory;
} SParseContext;
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
@ -127,7 +128,7 @@ int32_t qInitKeywordsTable();
void qCleanupKeywordsTable();
int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx,
SStbInterlaceInfo* pBuildInfo);
SStbInterlaceInfo* pBuildInfo, SVCreateTbReq* ctbReq);
int32_t qBuildStmtFinOutput(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDataBlocks);
// int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx,
// int32_t tbNum);
@ -165,7 +166,8 @@ int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* b
int32_t qBindStmt2RowValue(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo2* pBindInfos, void* charsetCxt);
int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt);
TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt,
SVCreateTbReq* pCreateTbReq);
void destroyBoundColumnInfo(void* pBoundInfo);
int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
@ -187,7 +189,7 @@ int32_t smlBuildOutputRaw(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data);
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);
int32_t checkSchema(SSchema* pColSchema, int8_t* fields, char* errstr, int32_t errstrLen);
int32_t checkSchema(SSchema* pColSchema, SSchemaExt* pColExtSchema, int8_t* fields, char* errstr, int32_t errstrLen);
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);
@ -195,7 +197,7 @@ int32_t serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);
void destoryCatalogReq(SCatalogReq* pCatalogReq);
bool isPrimaryKeyImpl(SNode* pExpr);
int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx,
SStbInterlaceInfo* pBuildInfo);
SStbInterlaceInfo* pBuildInfo, SVCreateTbReq* ctbReq);
#ifdef __cplusplus
}

6
include/libs/planner/planner.h
View File

@ -47,6 +47,12 @@ typedef struct SPlanContext {
bool destHasPrimaryKey;
bool sourceHasPrimaryKey;
void* timezone;
int64_t recalculateInterval;
char pStbFullName[TSDB_TABLE_FNAME_LEN];
char pWstartName[TSDB_COL_NAME_LEN];
char pWendName[TSDB_COL_NAME_LEN];
char pGroupIdName[TSDB_COL_NAME_LEN];
char pIsWindowFilledName[TSDB_COL_NAME_LEN];
} SPlanContext;
// Create the physical plan for the query, according to the AST.

6
include/libs/scalar/scalar.h
View File

@ -41,11 +41,12 @@ pDst need to freed in caller
int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst);
int32_t scalarGetOperatorParamNum(EOperatorType type);
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_t processType);
int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, STypeMod typeMod, int8_t processType);
int32_t vectorGetConvertType(int32_t type1, int32_t type2);
int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows);
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows);
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, STypeMod typeMod, int32_t startIndex, int32_t numOfRows);
STypeMod getConvertTypeMod(int32_t type, const SColumnInfo *pCol1, const SColumnInfo *pCol2);
/* Math functions */
int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
@ -122,6 +123,7 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t isWinFilledFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qPseudoTagFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

52
include/libs/stream/streamState.h
View File

@ -18,7 +18,9 @@
#include "tdatablock.h"
#ifdef USE_ROCKSDB
#include "rocksdb/c.h"
#endif
#include "tdbInt.h"
#include "tsimplehash.h"
#include "tstreamFileState.h"
@ -30,6 +32,7 @@ extern "C" {
#include "storageapi.h"
SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
SStreamState* streamStateRecalatedOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState);
void streamStateCommit(SStreamState* pState);
@ -41,7 +44,7 @@ int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVa
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool streamStateCheck(SStreamState* pState, const SWinKey* key);
bool streamStateCheck(SStreamState* pState, const SWinKey* key, bool hasLimit, bool* pIsLast);
int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal);
void streamStateDel(SStreamState* pState, const SWinKey* key);
void streamStateDelByGroupId(SStreamState* pState, uint64_t groupId);
@ -49,8 +52,11 @@ void streamStateClear(SStreamState* pState);
void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex);
void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t streamStateGetNumber(SStreamState* pState);
int32_t streamStateDeleteInfo(SStreamState* pState, void* pKey, int32_t keyLen);
int32_t streamStateGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateGetAllPrev(SStreamState* pState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
// session window
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen,
@ -65,6 +71,9 @@ int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey*
int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t streamStateSessionSaveToDisk(STableTsDataState* pTblState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
int32_t streamStateFlushReaminInfoToDisk(STableTsDataState* pTblState);
int32_t streamStateSessionDeleteAll(SStreamState* pState);
SStreamStateCur *streamStateSessionSeekKeyPrev(SStreamState *pState, const SSessionKey *key);
SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key);
@ -80,14 +89,14 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch
int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
int32_t streamStateFillAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t* pWinCode);
void streamStateFillDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateFillGetNext(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateFillGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used);
void streamStateClearBuff(SStreamState* pState, void* pVal);
@ -96,7 +105,8 @@ void streamStateFreeVal(void* val);
// count window
int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal,
int32_t* pVLen);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key);
@ -109,7 +119,10 @@ int32_t streamStateFillGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, con
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
// twa
void streamStateClearExpiredState(SStreamState* pState);
void streamStateClearExpiredState(SStreamState* pState, int32_t numOfKeep, TSKEY minTs);
void streamStateClearExpiredSessionState(SStreamState* pState, int32_t numOfKeep, TSKEY minTs, SSHashObj* pFlushGroup);
int32_t streamStateSetRecFlag(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t mode);
int32_t streamStateGetRecFlag(SStreamState* pState, const void* pKey, int32_t keyLen, int32_t* pMode);
void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
@ -117,12 +130,35 @@ void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode);
int32_t streamStateDeleteParName(SStreamState* pState, int64_t groupId);
void streamStateSetParNameInvalid(SStreamState* pState);
// group id
int32_t streamStateGroupPut(SStreamState* pState, int64_t groupId, void* value, int32_t vLen);
int32_t streamStateGroupPut(SStreamState* pState, int64_t groupId, void* value, int32_t vLen);
SStreamStateCur* streamStateGroupGetCur(SStreamState* pState);
void streamStateGroupCurNext(SStreamStateCur* pCur);
int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** pVal, int32_t* pVLen);
void streamStateGroupCurNext(SStreamStateCur* pCur);
int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** pVal, int32_t* pVLen);
// ts data
int32_t streamStateGetAndSetTsData(STableTsDataState* pState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
int32_t streamStateTsDataCommit(STableTsDataState* pState);
int32_t streamStateInitTsDataState(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState, void* pOtherState);
void streamStateDestroyTsDataState(STableTsDataState* pTsDataState);
int32_t streamStateRecoverTsData(STableTsDataState* pTsDataState);
int32_t streamStateReloadTsDataState(STableTsDataState* pTsDataState);
int32_t streamStateMergeAndSaveScanRange(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId,
SRecDataInfo* pRecData, int32_t len);
int32_t streamStateMergeAllScanRange(STableTsDataState* pTsDataState);
int32_t streamStatePopScanRange(STableTsDataState* pTsDataState, SScanRange* pRange);
// continuous
bool streamStateCheckSessionState(SStreamState* pState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
SStreamStateCur* streamStateGetLastStateCur(SStreamState* pState);
void streamStateLastStateCurNext(SStreamStateCur* pCur);
int32_t streamStateNLastStateGetKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
SStreamStateCur* streamStateGetLastSessionStateCur(SStreamState* pState);
void streamStateLastSessionStateCurNext(SStreamStateCur* pCur);
int32_t streamStateNLastSessionStateGetKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts);

42
include/libs/stream/tstream.h
View File

@ -303,7 +303,7 @@ typedef struct SStreamStatus {
bool appendTranstateBlock; // has appended the transfer state data block already
bool removeBackendFiles; // remove backend files on disk when free stream tasks
SConsenChkptInfo consenChkptInfo;
STimeWindow latestForceWindow; // latest generated time window, only valid in
STimeWindow latestForceWindow; // latest generated time window, only valid in
} SStreamStatus;
typedef struct SDataRange {
@ -318,9 +318,10 @@ typedef struct SSTaskBasicInfo {
int32_t selfChildId;
int32_t trigger;
int8_t taskLevel;
int8_t fillHistory; // is fill history task or not
int8_t fillHistory; // enum , 1. is fill history task or not 2. recal
int64_t delaySchedParam; // in msec
int64_t watermark; // extracted from operators
int8_t hasAggTasks; // has agg tasks in current stream
SInterval interval;
} SSTaskBasicInfo;
@ -461,7 +462,6 @@ struct SStreamTask {
STaskExecStatisInfo execInfo;
TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ
SMsgCb* pMsgCb; // msg handle
SStreamState* pState; // state backend
SUpstreamInfo upstreamInfo;
STaskCheckInfo taskCheckInfo;
SNotifyInfo notifyInfo;
@ -470,12 +470,18 @@ struct SStreamTask {
// the followings attributes don't be serialized
SScanhistorySchedInfo schedHistoryInfo;
int32_t transferStateAlignCnt;
int32_t recalculateAlignCnt;
struct SStreamMeta* pMeta;
SSHashObj* pNameMap;
void* pBackend;
int8_t subtableWithoutMd5; // only for tsma stream tasks
char reserve[256];
char* backendPath;
int8_t subtableWithoutMd5; // only for tsma stream tasks
char reserve[256];
char* backendPath;
void* pBackend;
void* pRecalBackend;
SStreamState* pRecalState;
SStreamState* pState; // state backend
};
typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*);
@ -558,16 +564,23 @@ typedef struct STaskUpdateEntry {
int32_t transId;
} STaskUpdateEntry;
typedef enum {
STREAM_NORMAL_TASK = 0,
STREAM_HISTORY_TASK = 1,
STREAM_RECALCUL_TASK = 2,
} EStreamTaskType;
typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int32_t trigger,
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, EStreamTaskType type, int32_t trigger,
int64_t triggerParam, SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5,
SStreamTask** pTask);
int8_t hasAggTasks, SStreamTask** pTask);
void tFreeStreamTask(void* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver);
void streamFreeTaskState(SStreamTask* pTask, int8_t remove);
int32_t streamCreateAddRecalculateEndBlock(SStreamTask* pTask);
int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo);
int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId);
@ -640,12 +653,6 @@ typedef struct STaskStatusEntry {
STaskNotifyEventStat notifyEventStat;
} STaskStatusEntry;
//typedef struct SNodeUpdateInfo {
// int32_t nodeId;
// SEpSet prevEp;
// SEpSet newEp;
//} SNodeUpdateInfo;
typedef struct SStreamTaskState {
ETaskStatus state;
char* name;
@ -662,7 +669,7 @@ typedef struct SCheckpointConsensusEntry {
int64_t ts;
} SCheckpointConsensusEntry;
void streamSetupScheduleTrigger(SStreamTask* pTask);
void streamSetupScheduleTrigger(SStreamTask* pTask);
// dispatch related
int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
@ -753,9 +760,10 @@ int32_t streamTaskReloadState(SStreamTask* pTask);
void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId);
void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId);
void streamTaskOpenAllUpstreamInput(SStreamTask* pTask);
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key);
int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key, uint8_t recalated);
bool streamTaskIsSinkTask(const SStreamTask* pTask);
void streamTaskSetRemoveBackendFiles(SStreamTask* pTask);
int8_t streamTaskShouldRecalated(SStreamTask* pTask);
void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask);
void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc);

40
include/libs/stream/tstreamFileState.h
View File

@ -56,12 +56,14 @@ void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuf
int32_t addRowBuffIfNotExist(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t createRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen);
void deleteRowBuffByGroupId(SStreamFileState* pFileState, uint64_t groupId);
int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal);
bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen);
bool hasRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, bool hasLimit, bool* pIsLast);
int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
SStreamSnapshot* getSnapshot(SStreamFileState* pFileState);
@ -80,6 +82,7 @@ bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap);
TSKEY getFlushMark(SStreamFileState* pFileState);
SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState);
int32_t getRowStateRowSize(SStreamFileState* pFileState);
void freeArrayPtr(void* ptr);
// session window
int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen,
@ -147,12 +150,43 @@ SSHashObj* getGroupIdCache(SStreamFileState* pFileState);
int fillStateKeyCompare(const void* pWin1, const void* pDatas, int pos);
int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SWinKey* pResKey, void** ppVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SWinKey* pKey);
int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SWinKey* pKey, bool* pIsEnd);
int32_t getRowStateAllPrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
//twa
void setFillInfo(SStreamFileState* pFileState);
void clearExpiredState(SStreamFileState* pFileState);
void clearExpiredState(SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs);
int32_t addArrayBuffIfNotExist(SSHashObj* pSearchBuff, uint64_t groupId, SArray** ppResStates);
int32_t recoverHashSortBuff(SStreamFileState* pFileState, SArray* pWinStates, uint64_t groupId);
int32_t getAndSetTsData(STableTsDataState* pTsDataState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
int32_t doTsDataCommit(STableTsDataState* pTsDataState);
int32_t doRangeDataCommit(STableTsDataState* pTsDataState);
int32_t initTsDataState(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState,
void* pOtherState);
void destroyTsDataState(STableTsDataState* pTsDataState);
int32_t recoverTsData(STableTsDataState* pTsDataState);
int32_t mergeAndSaveScanRange(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId, SRecDataInfo* pRecData,
int32_t len);
int32_t mergeAllScanRange(STableTsDataState* pTsDataState);
int32_t popScanRange(STableTsDataState* pTsDataState, SScanRange* pRange);
// continuous
typedef void* (*getStateBuffFn)(SStreamFileState* pFileState);
SStreamStateCur* getLastStateCur(SStreamFileState* pFileState, getStateBuffFn fn);
void moveLastStateCurNext(SStreamStateCur* pCur, getStateBuffFn fn);
void moveOneStateCurNext(SStreamStateCur* pCur);
int32_t getNLastStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
int32_t getNLastSessionStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
int32_t reloadTsDataState(STableTsDataState* pTsDataState);
int32_t setStateRecFlag(SStreamFileState* pState, const void* pKey, int32_t keyLen, int32_t mode);
int32_t getStateRecFlag(SStreamFileState* pFileState, const void* pKey, int32_t keyLen, int32_t* pMode);
void clearExpiredSessionState(struct SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs,
SSHashObj* pFlushGroup);
bool hasSessionState(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
int32_t saveRecInfoToDisk(STableTsDataState* pTsDataState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
int32_t flushRemainRecInfoToDisk(STableTsDataState* pTsDataState);
#ifdef __cplusplus
}

4
include/libs/transport/thttp.h
View File

@ -16,9 +16,7 @@
#ifndef _TD_UTIL_HTTP_H_
#define _TD_UTIL_HTTP_H_
#include "os.h"
#include "tdef.h"
#include "tref.h"
#include <stdint.h>
#ifdef __cplusplus
extern "C" {

193
include/libs/transport/trpc.h
View File

@ -19,6 +19,7 @@
extern "C" {
#endif
#ifndef TD_ASTRA_RPC
#include <stdbool.h>
#include <stdint.h>
#include "taosdef.h"
@ -190,6 +191,198 @@ int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf);
int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf);
int32_t rpcCvtErrCode(int32_t code);
#else
#include <stdbool.h>
#include <stdint.h>
#include "taosdef.h"
#include "tmsg.h"
#include "ttrace.h"
#define TAOS_CONN_SERVER 0
#define TAOS_CONN_CLIENT 1
#define IsReq(pMsg) (pMsg->msgType & 1U)
extern int32_t tsRpcHeadSize;
typedef struct {
uint32_t clientIp;
uint16_t clientPort;
int64_t applyIndex;
uint64_t applyTerm;
char user[TSDB_USER_LEN];
} SRpcConnInfo;
typedef enum {
TD_ASTRA_CLIENT = 1,
TD_ASTRA_DSVR_CLIENT = 2,
TD_ASTRA_DSVR_STA_CLIENT = 4,
TD_ASTRA_DSVR_SYNC_CLIENT = 8,
TD_ASTRA_DSVR = 16
} RPC_TYPE;
typedef struct SRpcHandleInfo {
// rpc info
void *handle; // rpc handle returned to app
int64_t refId; // refid, used by server
int8_t noResp; // has response or not(default 0, 0: resp, 1: no resp)
int8_t persistHandle; // persist handle or not
int8_t hasEpSet;
int32_t cliVer;
// app info
void *ahandle; // app handle set by client
void *wrapper; // wrapper handle
void *node; // node mgmt handle
#ifdef TD_ASTRA_32
void *ahandleEx; // app handle set by client
#endif
// resp info
void *rsp;
int32_t rspLen;
STraceId traceId;
SRpcConnInfo conn;
int8_t forbiddenIp;
int8_t notFreeAhandle;
int8_t compressed;
int16_t connType;
int64_t seq;
int64_t qId;
int32_t msgType;
void *reqWithSem;
int32_t refIdMgt;
} SRpcHandleInfo;
typedef struct SRpcMsg {
tmsg_t msgType;
void *pCont;
int32_t contLen;
int32_t code;
int32_t type;
void *parent;
SRpcHandleInfo info;
} SRpcMsg;
typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset);
typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType);
typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType);
typedef bool (*RpcFFfp)(tmsg_t msgType);
typedef bool (*RpcNoDelayfp)(tmsg_t msgType);
typedef void (*RpcDfp)(void *ahandle);
typedef struct SRpcInit {
char localFqdn[TSDB_FQDN_LEN];
uint16_t localPort; // local port
char *label; // for debug purpose
int32_t numOfThreads; // number of threads to handle connections
int32_t sessions; // number of sessions allowed
int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
int32_t idleTime; // milliseconds, 0 means idle timer is disabled
int32_t compatibilityVer;
int32_t retryMinInterval; // retry init interval
int32_t retryStepFactor; // retry interval factor
int32_t retryMaxInterval; // retry max interval
int64_t retryMaxTimeout;
int32_t failFastThreshold;
int32_t failFastInterval;
int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size
int8_t encryption; // encrypt or not
// the following is for client app ecurity only
char *user; // user name
// call back to process incoming msg
RpcCfp cfp;
// retry not not for particular msg
RpcRfp rfp;
// set up timeout for particular msg
RpcTfp tfp;
// destroy client ahandle;
RpcDfp dfp;
// fail fast fp
RpcFFfp ffp;
RpcNoDelayfp noDelayFp;
int32_t connLimitNum;
int32_t connLimitLock;
int32_t timeToGetConn;
int8_t supportBatch; // 0: no batch, 1. batch
int32_t batchSize;
int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait
int32_t shareConnLimit;
int8_t shareConn; // 0: no share, 1. share
int8_t startReadTimer;
int64_t readTimeout; // s
void *parent;
} SRpcInit;
typedef struct {
void *val;
int32_t (*clone)(void *src, void **dst);
} SRpcCtxVal;
typedef struct {
int32_t msgType;
void *val;
int32_t (*clone)(void *src, void **dst);
} SRpcBrokenlinkVal;
typedef struct {
SHashObj *args;
SRpcBrokenlinkVal brokenVal;
void (*freeFunc)(const void *arg);
int64_t st;
} SRpcCtx;
int32_t rpcInit();
void rpcCleanup();
void *rpcOpen(const SRpcInit *pRpc);
void rpcClose(void *);
void rpcCloseImpl(void *);
void *rpcMallocCont(int64_t contLen);
void rpcFreeCont(void *pCont);
void *rpcReallocCont(void *ptr, int64_t contLen);
// Because taosd supports multi-process mode
// These functions should not be used on the server side
// Please use tmsg<xx> functions, which are defined in tmsgcb.h
int32_t rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
int32_t rpcSendResponse(SRpcMsg *pMsg);
int32_t rpcRegisterBrokenLinkArg(SRpcMsg *msg);
int32_t rpcReleaseHandle(void *handle, int8_t type,
int32_t status); // just release conn to rpc instance, no close sock
// These functions will not be called in the child process
int32_t rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
int32_t rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int32_t rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int8_t *epUpdated,
int32_t timeoutMs);
int32_t rpcFreeConnById(void *shandle, int64_t connId);
int32_t rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn);
int32_t rpcAllocHandle(int64_t *refId);
int32_t rpcSetIpWhite(void *thandl, void *arg);
int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf);
int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf);
int32_t rpcCvtErrCode(int32_t code);
#endif
#ifdef __cplusplus
}
#endif

22
include/os/os.h
View File

@ -28,29 +28,34 @@ extern "C" {
#if !defined(WINDOWS)
#include <dirent.h>
#if !defined(_ALPINE)
#if !defined(_ALPINE) && !defined(TD_ASTRA)
#include <execinfo.h>
#endif
#if !defined(TD_ASTRA)
#include <libgen.h>
#include <sched.h>
#include <unistd.h>
#include <wordexp.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/param.h>
#include <sys/shm.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <termios.h>
#else
#include <astra.h>
#endif
#include <sched.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <sys/wait.h>
#include <termios.h>
#if defined(DARWIN)
#else
#if !defined(TD_ASTRA)
#include <argp.h>
#include <sys/prctl.h>
#include <sys/sysinfo.h>
@ -58,6 +63,7 @@ extern "C" {
#include <cpuid.h>
#endif
#endif
#endif
#else
#ifndef __func__

18
include/os/osDef.h
View File

@ -108,6 +108,9 @@ void wordfree(wordexp_t *pwordexp);
#define LOG_ERR 0
#define LOG_INFO 1
void syslog(int unused, const char *format, ...);
#elif defined(TD_ASTRA)
char *stpncpy(char *dest, const char *src, int n);
char *strsep(char **stringp, const char *delim);
#endif // WINDOWS
#ifndef WINDOWS
@ -216,6 +219,9 @@ void syslog(int unused, const char *format, ...);
// pthread_setname_np not defined
#define setThreadName(name)
#endif
#elif defined(TD_ASTRA)
#define setThreadName(name)
#define getThreadName(name)
#else
// Linux, length of name must <= 16 (the last '\0' included)
#define setThreadName(name) \
@ -237,12 +243,24 @@ void syslog(int unused, const char *format, ...);
#if defined(_WIN32)
#define TD_DIRSEP "\\"
#elif defined(TD_ASTRA)
#ifdef TD_ASTRA_TARGET
#define TD_DIRSEP "/"
#else
#define TD_DIRSEP "\\"
#endif
#else
#define TD_DIRSEP "/"
#endif
#if defined(_WIN32)
#define TD_DIRSEP_CHAR '\\'
#elif defined(TD_ASTRA)
#ifdef TD_ASTRA_TARGET
#define TD_DIRSEP_CHAR '/'
#else
#define TD_DIRSEP_CHAR '\\'
#endif
#else
#define TD_DIRSEP_CHAR '/'
#endif

14
include/os/osDir.h
View File

@ -60,6 +60,20 @@ extern "C" {
#define TD_LOG_DIR_PATH "/var/log/taos/"
#endif // CUS_PROMPT
#elif defined(TD_ASTRA)
#ifdef CUS_NAME
#define TD_TMP_DIR_PATH "C:\\" CUS_NAME "\\Temp\\"
#define TD_CFG_DIR_PATH "C:\\" CUS_NAME "\\cfg\\"
#define TD_DATA_DIR_PATH "C:\\" CUS_NAME "\\data\\"
#define TD_LOG_DIR_PATH "C:\\" CUS_NAME "\\log\\"
#else
#define TD_TMP_DIR_PATH "C:\\TDengine\\Temp\\"
#define TD_CFG_DIR_PATH "C:\\TDengine\\cfg\\"
#define TD_DATA_DIR_PATH "C:\\TDengine\\data\\"
#define TD_LOG_DIR_PATH "C:\\TDengine\\log\\"
#endif // CUS_NAME
#else
#define TD_TMP_DIR_PATH "/tmp/"

4
include/os/osEndian.h
View File

@ -20,6 +20,10 @@
extern "C" {
#endif
#ifdef TD_ASTRA
#include <endian.h>
#endif
typedef enum { TD_LITTLE_ENDIAN = 0, TD_BIG_ENDIAN } td_endian_t;
static const int32_t endian_test_var = 1;

1
include/os/osFile.h
View File

@ -72,6 +72,7 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions);
#define TD_FILE_ACCESS_EXIST_OK 0x1
#define TD_FILE_ACCESS_READ_OK 0x2
#define TD_FILE_ACCESS_WRITE_OK 0x4
#define TD_FILE_ACCESS_EXEC_OK 0x8
#define TD_TMP_FILE_PREFIX "tdengine-"

22
include/os/osMath.h
View File

@ -29,13 +29,23 @@ extern "C" {
#define TPOW2(x) ((x) * (x))
#define TABS(x) ((x) > 0 ? (x) : -(x))
#define TSWAP(a, b) \
do { \
char *__tmp = (char*)alloca(sizeof(a)); \
(void)memcpy(__tmp, &(a), sizeof(a)); \
(void)memcpy(&(a), &(b), sizeof(a)); \
(void)memcpy(&(b), __tmp, sizeof(a)); \
#ifndef TD_ASTRA
#define TSWAP(a, b) \
do { \
char *__tmp = (char *)alloca(sizeof(a)); \
(void)memcpy(__tmp, &(a), sizeof(a)); \
(void)memcpy(&(a), &(b), sizeof(a)); \
(void)memcpy(&(b), __tmp, sizeof(a)); \
} while (0)
#else
#define TSWAP(a, b) \
do { \
char __tmp[sizeof(a)]; \
(void)memcpy(__tmp, &(a), sizeof(a)); \
(void)memcpy(&(a), &(b), sizeof(a)); \
(void)memcpy(&(b), __tmp, sizeof(a)); \
} while (0)
#endif
#ifdef WINDOWS

11
include/os/osMemPool.h
View File

@ -149,8 +149,9 @@ int32_t taosMemoryPoolInit(mpReserveFailFp, mpReserveReachFp);
} while (0)
#ifndef BUILD_TEST
extern void* gMemPoolHandle;
#if !defined(BUILD_TEST) && !defined(TD_ASTRA)
extern threadlocal void* threadPoolSession;
extern threadlocal bool threadPoolEnabled;
extern int8_t tsMemPoolFullFunc;
@ -173,10 +174,10 @@ extern int8_t tsMemPoolFullFunc;
#define taosMemoryTrim(_size, _trimed) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolTrim(gMemPoolHandle, threadPoolSession, _size, (char*)__FILE__, __LINE__, _trimed)) : (taosMemTrim(_size, _trimed)))
#define taosMemoryMallocAlign(_alignment, _size) ((threadPoolEnabled && threadPoolSession) ? (taosMemPoolMallocAlign(gMemPoolHandle, threadPoolSession, _alignment, _size, (char*)__FILE__, __LINE__)) : (taosMemMallocAlign(_alignment, _size)))
#else
#define taosEnableMemoryPoolUsage(_pool, _session)
#define taosDisableMemoryPoolUsage()
#define taosSaveDisableMemoryPoolUsage()
#define taosRestoreEnableMemoryPoolUsage()
#define taosEnableMemPoolUsage(_session)
#define taosDisableMemPoolUsage()
#define taosSaveDisableMemPoolUsage(_enable, _randErr)
#define taosRestoreEnableMemPoolUsage(_enable, _randErr)
#define taosMemoryMalloc(_size) taosMemMalloc(_size)
#define taosMemoryCalloc(_num, _size) taosMemCalloc(_num, _size)

18
include/os/osMemory.h
View File

@ -25,10 +25,22 @@ extern "C" {
#if !defined(WINDOWS)
#ifndef ALLOW_FORBID_FUNC
#define malloc MALLOC_FUNC_TAOS_FORBID
#define calloc CALLOC_FUNC_TAOS_FORBID
#ifdef malloc
#undef malloc
#endif
#define malloc MALLOC_FUNC_TAOS_FORBID
#ifdef calloc
#undef calloc
#endif
#define calloc CALLOC_FUNC_TAOS_FORBID
#ifdef realloc
#undef realloc
#endif
#define realloc REALLOC_FUNC_TAOS_FORBID
#define free FREE_FUNC_TAOS_FORBID
#ifdef free
#undef free
#endif
#define free FREE_FUNC_TAOS_FORBID
#ifdef strdup
#undef strdup
#define strdup STRDUP_FUNC_TAOS_FORBID

15
include/os/osString.h
View File

@ -20,6 +20,10 @@
extern "C" {
#endif
#ifdef TD_ASTRA
#define DISALLOW_NCHAR_WITHOUT_ICONV
#endif
typedef wchar_t TdWchar;
typedef int32_t TdUcs4;
#if !defined(DISALLOW_NCHAR_WITHOUT_ICONV)// && defined(DARWIN)
@ -101,8 +105,8 @@ void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetC
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt);
int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt);
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
int32_t taosUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
int32_t taosUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
bool taosValidateEncodec(const char *encodec);
int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len, int32_t bufSize);
int32_t taosHexDecode(const char *src, char *dst, int32_t len);
@ -131,6 +135,13 @@ int32_t taosAscii2Hex(const char *z, uint32_t n, void **data, uint32_t *size);
bool isHex(const char *z, uint32_t n);
bool isValidateHex(const char *z, uint32_t n);
#ifdef TD_ASTRA
static FORCE_INLINE size_t strnlen(const char *s, size_t maxlen) {
const char *end = (const char *)memchr(s, '\0', maxlen);
return end ? (size_t)(end - s) : maxlen;
}
#endif
#ifdef __cplusplus
}
#endif

2
include/os/osSysinfo.h
View File

@ -60,7 +60,7 @@ int32_t taosGetSystemUUIDLen(char *uid, int32_t uidlen);
char *taosGetCmdlineByPID(int32_t pid);
void taosSetCoreDump(bool enable);
#if !defined(LINUX)
#if !defined(LINUX) || defined(TD_ASTRA)
#define _UTSNAME_LENGTH 65
#define _UTSNAME_MACHINE_LENGTH _UTSNAME_LENGTH

14
include/os/osThread.h
View File

@ -35,7 +35,7 @@ extern "C" {
// #endif
#endif
#if !defined(WINDOWS) && !defined(_ALPINE)
#if !defined(WINDOWS) && !defined(_ALPINE) && !defined(TD_ASTRA)
#ifndef __USE_XOPEN2K
#define TD_USE_SPINLOCK_AS_MUTEX
typedef pthread_mutex_t pthread_spinlock_t;
@ -70,12 +70,19 @@ typedef pthread_cond_t TdThreadCond;
typedef pthread_condattr_t TdThreadCondAttr;
typedef pthread_key_t TdThreadKey;
#endif
#ifdef TD_ASTRA
#define STACK_SIZE_DEFAULT (1048576 << 1)
#define STACK_SIZE_SMALL (1048576)
#else
#define STACK_SIZE_DEFAULT (10485760)
#endif
#define taosThreadCleanupPush pthread_cleanup_push
#define taosThreadCleanupPop pthread_cleanup_pop
#if !defined(WINDOWS)
#if defined(_TD_DARWIN_64) // MACOS
#define taosThreadRwlockAttrSetKindNP(A, B) ((void)0)
#elif defined(TD_ASTRA)
#define taosThreadRwlockAttrSetKindNP(A, B) ((void)0)
#else // LINUX
#if _XOPEN_SOURCE >= 500 || _POSIX_C_SOURCE >= 200809L
#define taosThreadRwlockAttrSetKindNP(A, B) pthread_rwlockattr_setkind_np(A, B)
@ -94,7 +101,7 @@ typedef pthread_key_t TdThreadKey;
#else
#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
#endif
#if 0
// If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC
@ -200,6 +207,7 @@ typedef pthread_key_t TdThreadKey;
#define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID
#define sigwait SIGWAIT_FUNC_TAOS_FORBID
#endif
#endif
int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start)(void *), void *arg);
int32_t taosThreadAttrDestroy(TdThreadAttr *attr);

2
include/util/cus_name.h
View File

@ -20,7 +20,7 @@
// support OEM
//
#ifndef TD_PRODUCT_NAME
#ifdef TD_ENTERPRISE
#if defined(TD_ENTERPRISE) || defined(TD_ASTRA)
#define TD_PRODUCT_NAME "TDengine Enterprise Edition"
#else
#define TD_PRODUCT_NAME "TDengine Community Edition"

7
include/util/taoserror.h
View File

@ -58,6 +58,11 @@ int32_t taosGetErrSize();
#define SET_ERROR_MSG(MSG, ...) \
(void)snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__)
#ifndef TD_ASTRA_RTP
#define SET_ERRNO(_code) (errno = (_code))
#define ERRNO errno
#endif
#define TSDB_CODE_SUCCESS 0
#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error
@ -161,6 +166,8 @@ int32_t taosGetErrSize();
#define TSDB_CODE_UNSUPPORT_OS TAOS_DEF_ERROR_CODE(0, 0x013A)
#define TSDB_CODE_TIME_ERROR TAOS_DEF_ERROR_CODE(0, 0x013B)
#define TSDB_CODE_INVALID_DISK_ID TAOS_DEF_ERROR_CODE(0, 0x013C)
#define TSDB_CODE_DECIMAL_OVERFLOW TAOS_DEF_ERROR_CODE(0, 0x013D)
#define TSDB_CODE_DIVISION_BY_ZERO TAOS_DEF_ERROR_CODE(0, 0x013E)
//client
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200)

8
include/util/tcompare.h
View File

@ -60,11 +60,13 @@ int32_t setChkInBytes1(const void *pLeft, const void *pRight);
int32_t setChkInBytes2(const void *pLeft, const void *pRight);
int32_t setChkInBytes4(const void *pLeft, const void *pRight);
int32_t setChkInBytes8(const void *pLeft, const void *pRight);
int32_t setChkInDecimalHash(const void* pLeft, const void* pRight);
int32_t setChkNotInBytes1(const void *pLeft, const void *pRight);
int32_t setChkNotInBytes2(const void *pLeft, const void *pRight);
int32_t setChkNotInBytes4(const void *pLeft, const void *pRight);
int32_t setChkNotInBytes8(const void *pLeft, const void *pRight);
int32_t setChkNotInDecimalHash(const void* pLeft, const void* pRight);
int32_t compareChkInString(const void *pLeft, const void *pRight);
int32_t compareChkNotInString(const void *pLeft, const void *pRight);
@ -86,6 +88,12 @@ int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareLenBinaryVal(const void *pLeft, const void *pRight);
int32_t compareDecimal64(const void* pleft, const void* pright);
int32_t compareDecimal128(const void* pleft, const void* pright);
int32_t compareDecimal64SameScale(const void* pleft, const void* pright);
int32_t compareDecimal128SameScale(const void* pleft, const void* pright);
int32_t comparestrRegexMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight);

8
include/util/tcompression.h
View File

@ -198,6 +198,14 @@ int32_t tsCompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3
int32_t nBuf);
int32_t tsDecompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressDecimal64(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsDecompressDecimal64(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressDecimal128(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsDecompressDecimal128(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
/*************************************************************************
* STREAM COMPRESSION

4
include/util/tconv.h
View File

@ -31,8 +31,8 @@ void taosConvDestroy();
//int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt);
//int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv);
//bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt);
//int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
//int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
//int32_t taosUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
//int32_t taosUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
#ifdef __cplusplus
}

39
include/util/tdef.h
View File

@ -25,6 +25,20 @@ extern "C" {
#define TSDB__packed
#if defined(TD_ASTRA_32)
#define PACK_PUSH_MIN _Pragma("pack(push, 4)")
#elif defined(WINDOWS)
#define PACK_PUSH_MIN __pragma(pack(push, 1))
#else
#define PACK_PUSH_MIN _Pragma("pack(push, 1)")
#endif
#if defined(WINDOWS)
#define PACK_POP __pragma(pack(pop))
#else
#define PACK_POP _Pragma("pack(pop)")
#endif
#define TSKEY int64_t
#define TSKEY_MIN INT64_MIN
#define TSKEY_MAX INT64_MAX
@ -33,7 +47,7 @@ extern "C" {
#define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle
// Bytes for each type.
extern const int32_t TYPE_BYTES[21];
extern const int32_t TYPE_BYTES[22];
#define CHAR_BYTES sizeof(char)
#define SHORT_BYTES sizeof(int16_t)
@ -46,6 +60,9 @@ extern const int32_t TYPE_BYTES[21];
#define TSDB_KEYSIZE sizeof(TSKEY)
#define TSDB_NCHAR_SIZE sizeof(TdUcs4)
#define DECIMAL64_BYTES 8
#define DECIMAL128_BYTES 16
// NULL definition
#define TSDB_DATA_BOOL_NULL 0x02
#define TSDB_DATA_TINYINT_NULL 0x80
@ -339,6 +356,8 @@ typedef enum ELogicConditionType {
#define TSDB_DNODE_CONFIG_LEN 128
#define TSDB_DNODE_VALUE_LEN 256
#define TSDB_RESERVE_VALUE_LEN 256
#define TSDB_CLUSTER_VALUE_LEN 1000
#define TSDB_GRANT_LOG_COL_LEN 15600
@ -678,6 +697,24 @@ typedef enum {
#define MIN_RESERVE_MEM_SIZE 1024 // MB
// Decimal
#define TSDB_DECIMAL64_MAX_PRECISION 18
#define TSDB_DECIMAL64_MAX_SCALE TSDB_DECIMAL64_MAX_PRECISION
#define TSDB_DECIMAL128_MAX_PRECISION 38
#define TSDB_DECIMAL128_MAX_SCALE TSDB_DECIMAL128_MAX_PRECISION
#define TSDB_DECIMAL_MIN_PRECISION 1
#define TSDB_DECIMAL_MAX_PRECISION TSDB_DECIMAL128_MAX_PRECISION
#define TSDB_DECIMAL_MIN_SCALE 0
#define TSDB_DECIMAL_MAX_SCALE TSDB_DECIMAL_MAX_PRECISION
#define GET_DEICMAL_MAX_PRECISION(type) (type) == TSDB_DATA_TYPE_DECIMAL64 ? TSDB_DECIMAL64_MAX_PRECISION : TSDB_DECIMAL_MAX_SCALE
typedef uint64_t DecimalWord;
#define DECIMAL_WORD_NUM(TYPE) (sizeof(TYPE) / sizeof(DecimalWord))
#define COMPILE_TIME_ASSERT(pred) switch(0) {case 0: case pred:;}
#ifdef __cplusplus
}
#endif

2
include/util/tgeosctx.h
View File

@ -20,6 +20,7 @@
extern "C" {
#endif
#ifdef USE_GEOS
#include <geos_c.h>
#include <tpcre2.h>
@ -41,6 +42,7 @@ typedef struct SGeosContext {
SGeosContext *acquireThreadLocalGeosCtx();
int32_t getThreadLocalGeosCtx(SGeosContext **ppCtx);
const char *getGeosErrMsg(int32_t code);
#endif
#ifdef __cplusplus
}

5
include/util/tlog.h
View File

@ -34,6 +34,11 @@ typedef enum {
DEBUG_FILE = 128
} ELogLevel;
typedef enum {
LOG_MODE_TAOSC = 1,
LOG_MODE_TAOSD = 2
} ELogMode;
typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content);
extern bool tsLogEmbedded;

2
include/util/tpcre2.h
View File

@ -16,6 +16,7 @@
#ifndef _TD_ULIT_PCRE2_H_
#define _TD_ULIT_PCRE2_H_
#ifdef USE_PRCE2
#define PCRE2_CODE_UNIT_WIDTH 8
#include "pcre2.h"
@ -31,4 +32,5 @@ void destroyRegexes(pcre2_code* pWktRegex, pcre2_match_data* pWktMatchData);
}
#endif
#endif // USE_PRCE2
#endif // _TD_UTIL_PAGEDBUF_H_

184
include/util/tutil.h
View File

@ -34,7 +34,7 @@ char **strsplit(char *src, const char *delim, int32_t *num);
char *strtolower(char *dst, const char *src);
char *strntolower(char *dst, const char *src, int32_t n);
char *strntolower_s(char *dst, const char *src, int32_t n);
int64_t strnatoi(char *num, int32_t len);
int64_t strnatoi(const char *num, int32_t len);
size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize);
size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize);
@ -54,6 +54,136 @@ int32_t parseCfgReal(const char *str, float *out);
bool tIsValidFileName(const char *fileName, const char *pattern);
bool tIsValidFilePath(const char *filePath, const char *pattern);
#ifdef TD_ASTRA
static FORCE_INLINE int32_t taosStrcasecmp(const char *s1, const char *s2) {
if (s1[0] == 0 && s2[0] == 0) return 0;
return strcasecmp(s1, s2);
}
static FORCE_INLINE int32_t taosStrncasecmp(const char *s1, const char *s2, size_t n) {
if (s1[0] == 0 && s2[0] == 0) return 0;
return strncasecmp(s1, s2, n);
}
#else
#define taosStrcasecmp strcasecmp
#define taosStrncasecmp strncasecmp
#endif
#ifdef NO_UNALIGNED_ACCESS
#define CHECK_ALIGNMENT
static FORCE_INLINE int64_t taosGetInt64Aligned(int64_t *pVal) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)pVal) & 7) == 0) return *pVal;
#endif
int64_t val;
memcpy(&val, pVal, sizeof(int64_t));
return val;
}
static FORCE_INLINE uint64_t taosGetUInt64Aligned(uint64_t *pVal) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)pVal) & 7) == 0) return *pVal;
#endif
uint64_t val;
memcpy(&val, pVal, sizeof(uint64_t));
return val;
}
static FORCE_INLINE float taosGetFloatAligned(float *pVal) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)pVal) & 7) == 0) return *pVal;
#endif
float val;
memcpy(&val, pVal, sizeof(float));
return val;
}
static FORCE_INLINE double taosGetDoubleAligned(double *pVal) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)pVal) & 7) == 0) return *pVal;
#endif
double val;
memcpy(&val, pVal, sizeof(double));
return val;
}
static FORCE_INLINE void taosSetInt64Aligned(int64_t *p, int64_t val) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)p) & 7) == 0) {
*p = val;
return;
}
#endif
memcpy(p, &val, sizeof(int64_t));
}
static FORCE_INLINE void taosSetUInt64Aligned(uint64_t *p, uint64_t val) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)p) & 7) == 0) {
*p = val;
return;
}
#endif
memcpy(p, &val, sizeof(uint64_t));
}
static FORCE_INLINE void taosSetPInt64Aligned(int64_t *to, int64_t *from) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)from) & 7) == 0 && ((uintptr_t)to & 7) == 0) {
*to = *from;
return;
}
#endif
memcpy(to, from, sizeof(int64_t));
}
static FORCE_INLINE void taosSetPFloatAligned(float *to, float *from) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)from) & 7) == 0 && ((uintptr_t)to & 7) == 0) {
*to = *from;
return;
}
#endif
memcpy(to, from, sizeof(float));
}
static FORCE_INLINE void taosSetPDoubleAligned(double *to, double *from) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)from) & 7) == 0 && ((uintptr_t)to & 7) == 0) {
*to = *from;
return;
}
#endif
memcpy(to, from, sizeof(double));
}
static FORCE_INLINE void taosSetPUInt64Aligned(uint64_t *to, uint64_t *from) {
#ifdef CHECK_ALIGNMENT
if ((((uintptr_t)from) & 7) == 0 && ((uintptr_t)to & 7) == 0) {
*to = *from;
return;
}
#endif
memcpy(to, from, sizeof(uint64_t));
}
#define TAOS_SET_OBJ_ALIGNED(pTo, vFrom) memcpy((pTo), &(vFrom), sizeof(*(pTo)))
#define TAOS_SET_POBJ_ALIGNED(pTo, pFrom) memcpy((pTo), (pFrom), sizeof(*(pTo)))
#else
static FORCE_INLINE int64_t taosGetInt64Aligned(int64_t *pVal) { return *pVal; }
static FORCE_INLINE uint64_t taosGetUInt64Aligned(uint64_t *pVal) { return *pVal; }
static FORCE_INLINE float taosGetFloatAligned(float *pVal) { return *pVal; }
static FORCE_INLINE double taosGetDoubleAligned(double *pVal) { return *pVal; }
static FORCE_INLINE void taosSetInt64Aligned(int64_t *p, int64_t val) { *p = val; }
static FORCE_INLINE void taosSetUInt64Aligned(uint64_t *p, uint64_t val) { *p = val; }
static FORCE_INLINE void taosSetPInt64Aligned(int64_t *to, int64_t *from) { *to = *from; }
static FORCE_INLINE void taosSetPFloatAligned(float *to, float *from) { *to = *from; }
static FORCE_INLINE void taosSetPDoubleAligned(double *to, double *from) { *to = *from; }
static FORCE_INLINE void taosSetPUInt64Aligned(uint64_t *to, uint64_t *from) { *to = *from; }
#define TAOS_SET_OBJ_ALIGNED(pTo, vFrom) *(pTo) = (vFrom)
#define TAOS_SET_POBJ_ALIGNED(pTo, pFrom) *(pTo) = *(pFrom)
#endif
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
T_MD5_CTX context;
tMD5Init(&context);
@ -148,20 +278,20 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
#define QUERY_CHECK_CODE TSDB_CHECK_CODE
#define TSDB_CHECK_CONDITION(condition, CODE, LINO, LABEL, ERRNO) \
if (UNLIKELY(!(condition))) { \
(CODE) = (ERRNO); \
(LINO) = __LINE__; \
goto LABEL; \
#define TSDB_CHECK_CONDITION(condition, CODE, LINO, LABEL, _ERRNO) \
if (UNLIKELY(!(condition))) { \
(CODE) = (_ERRNO); \
(LINO) = __LINE__; \
goto LABEL; \
}
#define QUERY_CHECK_CONDITION TSDB_CHECK_CONDITION
#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \
if (UNLIKELY((ptr) == NULL)) { \
(CODE) = (ERRNO); \
(LINO) = __LINE__; \
goto LABEL; \
#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, _ERRNO) \
if (UNLIKELY((ptr) == NULL)) { \
(CODE) = (_ERRNO); \
(LINO) = __LINE__; \
goto LABEL; \
}
#define QUERY_CHECK_NULL TSDB_CHECK_NULL
@ -170,7 +300,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
#define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL)
#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member)))
#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member)))
#define TAOS_GET_TERRNO(code) (terrno == 0 ? code : terrno)
@ -187,13 +317,13 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
} \
} while (0)
#define TAOS_CHECK_RETURN_SET_CODE(CMD, CODE, ERRNO) \
do { \
int32_t __c = (CMD); \
if (__c != TSDB_CODE_SUCCESS) { \
(CODE) = (ERRNO); \
TAOS_RETURN(__c); \
} \
#define TAOS_CHECK_RETURN_SET_CODE(CMD, CODE, _ERRNO) \
do { \
int32_t __c = (CMD); \
if (__c != TSDB_CODE_SUCCESS) { \
(CODE) = (_ERRNO); \
TAOS_RETURN(__c); \
} \
} while (0)
#define TAOS_CHECK_RETURN_WITH_RELEASE(CMD, PTR1, PTR2) \
@ -234,14 +364,14 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
} \
} while (0)
#define TAOS_CHECK_EXIT_SET_CODE(CMD, CODE, ERRNO) \
do { \
code = (CMD); \
if (code < TSDB_CODE_SUCCESS) { \
(CODE) = (ERRNO); \
lino = __LINE__; \
goto _exit; \
} \
#define TAOS_CHECK_EXIT_SET_CODE(CMD, CODE, _ERRNO) \
do { \
code = (CMD); \
if (code < TSDB_CODE_SUCCESS) { \
(CODE) = (_ERRNO); \
lino = __LINE__; \
goto _exit; \
} \
} while (0)
#define TAOS_UNUSED(expr) (void)(expr)

8
source/client/CMakeLists.txt
View File

@ -21,10 +21,14 @@ target_include_directories(
PUBLIC "${TD_SOURCE_DIR}/include/client"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
set(TAOSD_MODULE "")
if(${TAOSD_INTEGRATED})
set(TAOSD_MODULE "taosd")
endif()
target_link_libraries(
${TAOS_LIB}
INTERFACE api
PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry ${TAOSD_MODULE} decimal
)
if(TD_WINDOWS)
@ -63,7 +67,7 @@ target_include_directories(
target_link_libraries(
${TAOS_LIB_STATIC}
INTERFACE api
PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry decimal
)
if(${BUILD_TEST})

25
source/client/inc/clientInt.h
View File

@ -119,17 +119,17 @@ typedef struct {
} SAppInstServerCFG;
struct SAppInstInfo {
int64_t numOfConns;
SCorEpSet mgmtEp;
int32_t totalDnodes;
int32_t onlineDnodes;
TdThreadMutex qnodeMutex;
SArray* pQnodeList;
SAppClusterSummary summary;
SList* pConnList; // STscObj linked list
int64_t clusterId;
SAppClusterSummary summary;
SArray* pQnodeList;
SList* pConnList; // STscObj linked list
void* pTransporter;
SAppHbMgr* pAppHbMgr;
char* instKey;
TdThreadMutex qnodeMutex;
SCorEpSet mgmtEp;
SAppInstServerCFG serverCfg;
};
@ -205,7 +205,7 @@ typedef struct SReqResultInfo {
SExecResult execRes;
const char* pRspMsg;
const char* pData;
TAOS_FIELD* fields; // todo, column names are not needed.
TAOS_FIELD_E* fields; // todo, column names are not needed.
TAOS_FIELD* userFields; // the fields info that return to user
uint32_t numOfCols;
int32_t* length;
@ -265,8 +265,9 @@ typedef struct SReqRelInfo {
typedef struct SRequestObj {
int8_t resType; // query or tmq
uint64_t requestId;
int32_t type; // request type
uint64_t requestId;
SQuery* pQuery;
STscObj* pTscObj;
char* pDb; // current database string
char* sqlstr; // sql string
@ -294,13 +295,13 @@ typedef struct SRequestObj {
uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog
uint32_t retry;
int64_t allocatorRefId;
SQuery* pQuery;
void* pPostPlan;
SReqRelInfo relation;
void* pWrapper;
SMetaData parseMeta;
char* effectiveUser;
int8_t source;
bool streamRunHistory;
} SRequestObj;
typedef struct SSyncQueryParam {
@ -312,11 +313,11 @@ typedef struct SSyncQueryParam {
void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
void doSetOneRowPtr(SReqResultInfo* pResultInfo);
void doSetOneRowPtr(SReqResultInfo* pResultInfo, bool isStmt);
void setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4);
int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool isStmt);
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4, bool isStmt);
int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols, const SExtSchema* pExtSchema, bool isStmt);
void doFreeReqResultInfo(SReqResultInfo* pResInfo);
int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
void syncCatalogFn(SMetaData* pResult, void* param, int32_t code);

3
source/client/inc/clientStmt.h
View File

@ -48,7 +48,7 @@ typedef struct SStmtTableCache {
} SStmtTableCache;
typedef struct SStmtQueryResInfo {
TAOS_FIELD *fields;
TAOS_FIELD_E *fields;
TAOS_FIELD *userFields;
uint32_t numOfCols;
int32_t precision;
@ -123,6 +123,7 @@ typedef struct SStmtStatInfo {
typedef struct SStmtQNode {
bool restoreTbCols;
STableColsData tblData;
SVCreateTbReq *pCreateTbReq;
struct SStmtQNode* next;
} SStmtQNode;

7
source/client/inc/clientStmt2.h
View File

@ -102,6 +102,8 @@ typedef struct {
SHashObj *pVgHash;
SBindInfo2 *pBindInfo;
bool bindRowFormat;
bool fixValueTags;
SVCreateTbReq *fixValueTbReq;
SStbInterlaceInfo siInfo;
} SStmtSQLInfo2;
@ -234,8 +236,9 @@ int stmtClose2(TAOS_STMT2 *stmt);
int stmtExec2(TAOS_STMT2 *stmt, int *affected_rows);
int stmtPrepare2(TAOS_STMT2 *stmt, const char *sql, unsigned long length);
int stmtSetTbName2(TAOS_STMT2 *stmt, const char *tbName);
int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags);
int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx);
int stmtSetTbTags2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *tags, SVCreateTbReq **pCreateTbReq);
int stmtCheckTags2(TAOS_STMT2 *stmt, SVCreateTbReq **pCreateTbReq);
int stmtBindBatch2(TAOS_STMT2 *stmt, TAOS_STMT2_BIND *bind, int32_t colIdx, SVCreateTbReq *pCreateTbReq);
int stmtGetStbColFields2(TAOS_STMT2 *stmt, int *nums, TAOS_FIELD_ALL **fields);
int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums);
int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert);

146
source/client/src/clientEnv.c
View File

@ -54,9 +54,9 @@
#define ENV_ERR_RET(c, info) \
do { \
int32_t _code = c; \
int32_t _code = (c); \
if (_code != TSDB_CODE_SUCCESS) { \
errno = _code; \
terrno = _code; \
tscInitRes = _code; \
tscError(info); \
return; \
@ -128,7 +128,7 @@ static void concatStrings(SArray *list, char *buf, int size) {
}
}
}
#ifdef USE_REPORT
static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration) {
cJSON *json = cJSON_CreateObject();
int32_t code = TSDB_CODE_SUCCESS;
@ -217,7 +217,7 @@ _end:
cJSON_Delete(json);
return code;
}
#endif
static bool checkSlowLogExceptDb(SRequestObj *pRequest, char *exceptDb) {
if (pRequest->pDb != NULL) {
return strcmp(pRequest->pDb, exceptDb) != 0;
@ -283,6 +283,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
}
}
#ifdef USE_REPORT
if (pTscObj->pAppInfo->serverCfg.monitorParas.tsEnableMonitor) {
if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT);
@ -308,6 +309,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
}
}
}
#endif
releaseTscObj(pTscObj->id);
}
@ -789,7 +791,7 @@ void stopAllQueries(SRequestObj *pRequest) {
}
}
}
#ifdef USE_REPORT
void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); }
static void *tscCrashReportThreadFp(void *param) {
@ -888,15 +890,15 @@ int32_t tscCrashReportInit() {
TSC_ERR_JRET(taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE));
TdThread crashReportThread;
if (taosThreadCreate(&crashReportThread, &thAttr, tscCrashReportThreadFp, NULL) != 0) {
tscError("failed to create crashReport thread since %s", strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
TSC_ERR_RET(errno);
tscError("failed to create crashReport thread since %s", strerror(ERRNO));
terrno = TAOS_SYSTEM_ERROR(ERRNO);
TSC_ERR_RET(terrno);
}
(void)taosThreadAttrDestroy(&thAttr);
_return:
if (code) {
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(ERRNO);
TSC_ERR_RET(terrno);
}
@ -921,6 +923,113 @@ void tscStopCrashReport() {
void tscWriteCrashInfo(int signum, void *sigInfo, void *context) {
writeCrashLogToFile(signum, sigInfo, CUS_PROMPT, lastClusterId, appInfo.startTime);
}
#endif
#ifdef TAOSD_INTEGRATED
typedef struct {
TdThread pid;
int32_t stat; // < 0: start failed, 0: init(not start), 1: start successfully
} SDaemonObj;
extern int dmStartDaemon(int argc, char const *argv[]);
extern void dmStopDaemon();
SDaemonObj daemonObj = {0};
typedef struct {
int32_t argc;
char **argv;
} SExecArgs;
static void *dmStartDaemonFunc(void *param) {
int32_t code = 0;
SExecArgs *pArgs = (SExecArgs *)param;
int32_t argc = pArgs->argc;
char **argv = pArgs->argv;
code = dmStartDaemon(argc, (const char **)argv);
if (code != 0) {
printf("failed to start taosd since %s\r\n", tstrerror(code));
goto _exit;
}
_exit:
if (code != 0) {
atomic_store_32(&daemonObj.stat, code);
}
return NULL;
}
static int32_t shellStartDaemon(int argc, char *argv[]) {
int32_t code = 0, lino = 0;
SExecArgs *pArgs = NULL;
int64_t startMs = taosGetTimestampMs(), endMs = startMs;
TdThreadAttr thAttr;
(void)taosThreadAttrInit(&thAttr);
(void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
#ifdef TD_COMPACT_OS
(void)taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL);
#endif
pArgs = (SExecArgs *)taosMemoryCalloc(1, sizeof(SExecArgs));
if (pArgs == NULL) {
code = terrno;
TAOS_CHECK_EXIT(code);
}
pArgs->argc = argc;
pArgs->argv = argv;
#ifndef TD_AS_LIB
tsLogEmbedded = 1;
#endif
TAOS_CHECK_EXIT(taosThreadCreate(&daemonObj.pid, &thAttr, dmStartDaemonFunc, pArgs));
while (true) {
if (atomic_load_64(&tsDndStart)) {
atomic_store_32(&daemonObj.stat, 1);
break;
}
int32_t daemonstat = atomic_load_32(&daemonObj.stat);
if (daemonstat < 0) {
code = daemonstat;
TAOS_CHECK_EXIT(code);
}
if (daemonstat > 1) {
code = TSDB_CODE_APP_ERROR;
TAOS_CHECK_EXIT(code);
}
taosMsleep(1000);
}
_exit:
endMs = taosGetTimestampMs();
(void)taosThreadAttrDestroy(&thAttr);
taosMemoryFreeClear(pArgs);
if (code) {
printf("\r\n The daemon start failed at line %d since %s, cost %" PRIi64 " ms\r\n", lino, tstrerror(code),
endMs - startMs);
} else {
printf("\r\n The daemon started successfully, cost %" PRIi64 " ms\r\n", endMs - startMs);
}
#ifndef TD_AS_LIB
tsLogEmbedded = 0;
#endif
return code;
}
void shellStopDaemon() {
#ifndef TD_AS_LIB
tsLogEmbedded = 1;
#endif
dmStopDaemon();
if (taosCheckPthreadValid(daemonObj.pid)) {
(void)taosThreadJoin(daemonObj.pid, NULL);
taosThreadClear(&daemonObj.pid);
}
}
#endif
void taos_init_imp(void) {
#if defined(LINUX)
@ -938,7 +1047,8 @@ void taos_init_imp(void) {
// In the APIs of other program language, taos_cleanup is not available yet.
// So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning.
(void)atexit(taos_cleanup);
errno = TSDB_CODE_SUCCESS;
SET_ERRNO(TSDB_CODE_SUCCESS);
terrno = TSDB_CODE_SUCCESS;
taosSeedRand(taosGetTimestampSec());
appInfo.pid = taosGetPId();
@ -956,20 +1066,26 @@ void taos_init_imp(void) {
const char *logName = CUS_PROMPT "log";
ENV_ERR_RET(taosInitLogOutput(&logName), "failed to init log output");
if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
(void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(errno), configDir);
(void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(ERRNO), configDir);
tscInitRes = terrno;
return;
}
#ifdef TAOSD_INTEGRATED
ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0), "failed to init cfg");
#else
ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg");
#endif
initQueryModuleMsgHandle();
#ifndef DISALLOW_NCHAR_WITHOUT_ICONV
if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) {
tscInitRes = terrno;
tscError("failed to init conv");
return;
}
#ifndef WINDOWS
#endif
#if !defined(WINDOWS) && !defined(TD_ASTRA)
ENV_ERR_RET(tzInit(), "failed to init timezone");
#endif
ENV_ERR_RET(monitorInit(), "failed to init monitor");
@ -997,9 +1113,13 @@ void taos_init_imp(void) {
ENV_ERR_RET(taosGetAppName(appInfo.appName, NULL), "failed to get app name");
ENV_ERR_RET(taosThreadMutexInit(&appInfo.mutex, NULL), "failed to init thread mutex");
#ifdef USE_REPORT
ENV_ERR_RET(tscCrashReportInit(), "failed to init crash report");
#endif
ENV_ERR_RET(qInitKeywordsTable(), "failed to init parser keywords table");
#ifdef TAOSD_INTEGRATED
ENV_ERR_RET(shellStartDaemon(0, NULL), "failed to start taosd daemon");
#endif
tscInfo("TAOS driver is initialized successfully");
}

13
source/client/src/clientHb.c
View File

@ -187,7 +187,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) {
for (int32_t j = 0; j < rsp->vgNum; ++j) {
SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j);
if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) {
tscError("hash push failed, errno:%d", errno);
tscError("hash push failed, terrno:%d", terrno);
code = terrno;
goto _return;
}
@ -1401,16 +1401,19 @@ static int32_t hbCreateThread() {
TdThreadAttr thAttr;
TSC_ERR_JRET(taosThreadAttrInit(&thAttr));
TSC_ERR_JRET(taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE));
#ifdef TD_COMPACT_OS
TSC_ERR_JRET(taosThreadAttrSetStackSize(&thAttr, STACK_SIZE_SMALL));
#endif
if (taosThreadCreate(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(ERRNO);
TSC_ERR_RET(terrno);
}
(void)taosThreadAttrDestroy(&thAttr);
_return:
if (code) {
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(ERRNO);
TSC_ERR_RET(terrno);
}
@ -1431,12 +1434,12 @@ static void hbStopThread() {
if (clientHbMgr.quitByKill) {
code = taosThreadKill(clientHbMgr.thread, 0);
if (TSDB_CODE_SUCCESS != code) {
tscError("taosThreadKill failed since %s", tstrerror(TAOS_SYSTEM_ERROR(code)));
tscError("taosThreadKill failed since %s", tstrerror(code));
}
} else {
code = taosThreadJoin(clientHbMgr.thread, NULL);
if (TSDB_CODE_SUCCESS != code) {
tscError("taosThreadJoin failed since %s", tstrerror(TAOS_SYSTEM_ERROR(errno)));
tscError("taosThreadJoin failed since %s", tstrerror(code));
}
}

111
source/client/src/clientImpl.c
View File

@ -28,6 +28,8 @@
#include "tref.h"
#include "tsched.h"
#include "tversion.h"
#include "decimal.h"
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo);
@ -313,7 +315,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
code = qParseSql(&cxt, pQuery);
if (TSDB_CODE_SUCCESS == code) {
if ((*pQuery)->haveResultSet) {
code = setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
code = setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols, (*pQuery)->pResExtSchema, pRequest->isStmtBind);
setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision);
}
}
@ -335,7 +337,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
int8_t biMode = atomic_load_8(&pRequest->pTscObj->biMode);
int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, biMode, pRequest->pTscObj->optionInfo.charsetCxt);
if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4, pRequest->isStmtBind);
}
return code;
@ -373,7 +375,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp,
atomic_load_8(&pRequest->pTscObj->biMode), pRequest->pTscObj->optionInfo.charsetCxt);
if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, pRequest->body.resInfo.convertUcs4, pRequest->isStmtBind);
}
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
@ -515,7 +517,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
return qCreateQueryPlan(&cxt, pPlan, pNodeList);
}
int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) {
int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols, const SExtSchema* pExtSchema, bool isStmt) {
if (pResInfo == NULL || pSchema == NULL || numOfCols <= 0) {
tscError("invalid paras, pResInfo == NULL || pSchema == NULL || numOfCols <= 0");
return TSDB_CODE_INVALID_PARA;
@ -528,7 +530,7 @@ int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32
if (pResInfo->userFields != NULL) {
taosMemoryFree(pResInfo->userFields);
}
pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD_E));
if (NULL == pResInfo->fields) return terrno;
pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
if (NULL == pResInfo->userFields) {
@ -541,17 +543,14 @@ int32_t setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32
}
for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
pResInfo->fields[i].bytes = pSchema[i].bytes;
pResInfo->fields[i].type = pSchema[i].type;
pResInfo->userFields[i].bytes = pSchema[i].bytes;
pResInfo->userFields[i].type = pSchema[i].type;
if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_VARBINARY ||
pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) {
pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
} else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
// userFields must convert to type bytes, no matter isStmt or not
pResInfo->userFields[i].bytes = calcTypeBytesFromSchemaBytes(pSchema[i].type, pSchema[i].bytes, false);
pResInfo->fields[i].bytes = calcTypeBytesFromSchemaBytes(pSchema[i].type, pSchema[i].bytes, isStmt);
if (IS_DECIMAL_TYPE(pSchema[i].type) && pExtSchema) {
decimalFromTypeMod(pExtSchema[i].typeMod, &pResInfo->fields[i].precision, &pResInfo->fields[i].scale);
}
tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name));
@ -1126,13 +1125,13 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
}
}
tscDebug("lastKey:%" PRId64 " vgId:%d, vgVer:%" PRId64, ts, *(int32_t*)pRow[1], *(int64_t*)pRow[2]);
tscInfo("[create stream with histroy] lastKey:%" PRId64 " vgId:%d, vgVer:%" PRId64, ts, *(int32_t*)pRow[1], *(int64_t*)pRow[2]);
}
(*pBlock)->info.window.ekey = lastTs;
(*pBlock)->info.rows = numOfRows;
tscDebug("lastKey:%" PRId64 " numOfRows:%d from all vgroups", lastTs, numOfRows);
tscInfo("[create stream with histroy] lastKey:%" PRId64 " numOfRows:%d from all vgroups", lastTs, numOfRows);
return TSDB_CODE_SUCCESS;
}
@ -1942,15 +1941,15 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
// return taos_connect(ipStr, userStr, passStr, dbStr, port);
// }
void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
void doSetOneRowPtr(SReqResultInfo* pResultInfo, bool isStmt) {
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
SResultColumn* pCol = &pResultInfo->pCol[i];
int32_t type = pResultInfo->fields[i].type;
int32_t bytes = pResultInfo->fields[i].bytes;
int32_t schemaBytes = calcSchemaBytesFromTypeBytes(type, pResultInfo->fields[i].bytes, isStmt);
if (IS_VAR_DATA_TYPE(type)) {
if (!IS_VAR_NULL_TYPE(type, bytes) && pCol->offset[pResultInfo->current] != -1) {
if (!IS_VAR_NULL_TYPE(type, schemaBytes) && pCol->offset[pResultInfo->current] != -1) {
char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData;
pResultInfo->length[i] = varDataLen(pStart);
@ -1961,8 +1960,8 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
}
} else {
if (!colDataIsNull_f(pCol->nullbitmap, pResultInfo->current)) {
pResultInfo->row[i] = pResultInfo->pCol[i].pData + bytes * pResultInfo->current;
pResultInfo->length[i] = bytes;
pResultInfo->row[i] = pResultInfo->pCol[i].pData + schemaBytes * pResultInfo->current;
pResultInfo->length[i] = schemaBytes;
} else {
pResultInfo->row[i] = NULL;
pResultInfo->length[i] = 0;
@ -1994,7 +1993,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
}
pRequest->code =
setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4);
setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4, pRequest->isStmtBind);
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
return NULL;
@ -2013,7 +2012,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
}
if (setupOneRowPtr) {
doSetOneRowPtr(pResultInfo);
doSetOneRowPtr(pResultInfo, pRequest->isStmtBind);
pResultInfo->current += 1;
}
@ -2060,7 +2059,7 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
return NULL;
} else {
if (setupOneRowPtr) {
doSetOneRowPtr(pResultInfo);
doSetOneRowPtr(pResultInfo, pRequest->isStmtBind);
pResultInfo->current += 1;
}
@ -2087,14 +2086,14 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
return TSDB_CODE_SUCCESS;
}
static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength, bool isStmt) {
int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx, C2M, pResultInfo->charsetCxt);
if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR;
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
int32_t type = pResultInfo->fields[i].type;
int32_t bytes = pResultInfo->fields[i].bytes;
int32_t schemaBytes = calcSchemaBytesFromTypeBytes(pResultInfo->fields[i].type, pResultInfo->fields[i].bytes, isStmt);
if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
@ -2111,11 +2110,11 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
char* pStart = pCol->offset[j] + pCol->pData;
int32_t len = taosUcs4ToMbsEx((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p), conv);
if (len < 0 || len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) {
if (len < 0 || len > schemaBytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) {
tscError(
"doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + "
"colLength[i]):%p",
len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i]));
len, schemaBytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i]));
taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
@ -2134,6 +2133,36 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
return TSDB_CODE_SUCCESS;
}
static int32_t convertDecimalType(SReqResultInfo* pResultInfo) {
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
TAOS_FIELD_E* pField = pResultInfo->fields + i;
int32_t type = pField->type;
int32_t bufLen = 0;
char* p = NULL;
if (!IS_DECIMAL_TYPE(type) || !pResultInfo->pCol[i].pData) {
continue;
} else {
bufLen = 64;
p = taosMemoryRealloc(pResultInfo->convertBuf[i], bufLen * pResultInfo->numOfRows);
pField->bytes = bufLen;
}
if (!p) return terrno;
pResultInfo->convertBuf[i] = p;
for (int32_t j = 0; j < pResultInfo->numOfRows; ++j) {
int32_t code = decimalToStr((DecimalWord*)(pResultInfo->pCol[i].pData + j * tDataTypes[type].bytes), type,
pField->precision, pField->scale, p, bufLen);
p += bufLen;
if (TSDB_CODE_SUCCESS != code) {
return code;
}
}
pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
}
return 0;
}
int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) {
return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) * 3 + sizeof(uint64_t) +
numOfCols * (sizeof(int8_t) + sizeof(int32_t));
@ -2365,7 +2394,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) {
return TSDB_CODE_SUCCESS;
}
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) {
int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4, bool isStmt) {
if (pResultInfo == NULL || pResultInfo->numOfCols <= 0 || pResultInfo->fields == NULL) {
tscError("setResultDataPtr paras error");
return TSDB_CODE_TSC_INTERNAL_ERROR;
@ -2413,7 +2442,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) {
int32_t hasColumnSeg = *(int32_t*)p;
p += sizeof(int32_t);
uint64_t groupId = *(uint64_t*)p;
uint64_t groupId = taosGetUInt64Aligned((uint64_t*)p);
p += sizeof(uint64_t);
// check fields
@ -2423,6 +2452,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) {
int32_t bytes = *(int32_t*)p;
p += sizeof(int32_t);
if (IS_DECIMAL_TYPE(type) && pResultInfo->fields[i].precision == 0) {
extractDecimalTypeInfoFromBytes(&bytes, &pResultInfo->fields[i].precision, &pResultInfo->fields[i].scale);
}
}
int32_t* colLength = (int32_t*)p;
@ -2454,7 +2487,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) {
}
pResultInfo->pCol[i].pData = pStart;
pResultInfo->length[i] = pResultInfo->fields[i].bytes;
pResultInfo->length[i] = calcSchemaBytesFromTypeBytes(pResultInfo->fields[i].type, pResultInfo->fields[i].bytes, isStmt);
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
pStart += colLength[i];
@ -2469,10 +2502,14 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, bool convertUcs4) {
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
#ifndef DISALLOW_NCHAR_WITHOUT_ICONV
if (convertUcs4) {
code = doConvertUCS4(pResultInfo, colLength);
code = doConvertUCS4(pResultInfo, colLength, isStmt);
}
#endif
if (TSDB_CODE_SUCCESS == code && convertUcs4) {
code = convertDecimalType(pResultInfo);
}
return code;
}
@ -2513,7 +2550,7 @@ void resetConnectDB(STscObj* pTscObj) {
(void)taosThreadMutexUnlock(&pTscObj->mutex);
}
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4) {
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool isStmt) {
if (pResultInfo == NULL || pRsp == NULL) {
tscError("setQueryResultFromRsp paras is null");
return TSDB_CODE_TSC_INTERNAL_ERROR;
@ -2582,7 +2619,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
// TODO handle the compressed case
pResultInfo->totalRows += pResultInfo->numOfRows;
int32_t code = setResultDataPtr(pResultInfo, convertUcs4);
int32_t code = setResultDataPtr(pResultInfo, convertUcs4, isStmt);
return code;
}
@ -3045,7 +3082,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
}
pRequest->code =
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4);
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, pRequest->isStmtBind);
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
tscError("req:0x%" PRIx64 ", fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code),
@ -3115,7 +3152,9 @@ void doRequestCallback(SRequestObj* pRequest, int32_t code) {
code = TSDB_CODE_SUCCESS;
pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
}
pRequest->body.queryFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code);
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code);
}
SRequestObj* pReq = acquireRequest(this);
if (pReq != NULL) {
pReq->inCallback = false;

71
source/client/src/clientMain.c
View File

@ -35,6 +35,10 @@
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
#ifdef TAOSD_INTEGRATED
extern void shellStopDaemon();
#endif
static int32_t sentinel = TSC_VAR_NOT_RELEASE;
static int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SSqlCallbackWrapper *pWrapper);
@ -55,7 +59,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
return ret;
}
#ifndef WINDOWS
#if !defined(WINDOWS) && !defined(TD_ASTRA)
static void freeTz(void *p) {
timezone_t tz = *(timezone_t *)p;
tzfree(tz);
@ -95,7 +99,7 @@ static timezone_t setConnnectionTz(const char *val) {
tz = tzalloc("UTC");
if (tz == NULL) {
tscError("%s set timezone UTC error", __func__);
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(ERRNO);
goto END;
}
}
@ -153,6 +157,7 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co
val = NULL;
}
#ifndef DISALLOW_NCHAR_WITHOUT_ICONV
if (option == TSDB_OPTION_CONNECTION_CHARSET || option == TSDB_OPTION_CONNECTION_CLEAR) {
if (val != NULL) {
if (!taosValidateEncodec(val)) {
@ -169,9 +174,9 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co
pObj->optionInfo.charsetCxt = NULL;
}
}
#endif
if (option == TSDB_OPTION_CONNECTION_TIMEZONE || option == TSDB_OPTION_CONNECTION_CLEAR) {
#ifndef WINDOWS
#if !defined(WINDOWS) && !defined(TD_ASTRA)
if (val != NULL) {
if (val[0] == 0) {
val = "UTC";
@ -239,7 +244,7 @@ void taos_cleanup(void) {
tscWarn("failed to cleanup task queue");
}
#ifndef WINDOWS
#if !defined(WINDOWS) && !defined(TD_ASTRA)
tzCleanup();
#endif
tmqMgmtClose();
@ -259,10 +264,14 @@ void taos_cleanup(void) {
taosConvDestroy();
DestroyRegexCache();
#ifdef TAOSD_INTEGRATED
shellStopDaemon();
#endif
tscInfo("all local resources released");
taosCleanupCfg();
#ifndef TAOSD_INTEGRATED
taosCloseLog();
#endif
}
static setConfRet taos_set_config_imp(const char *config) {
@ -599,6 +608,14 @@ TAOS_RES *taos_query_with_reqid(TAOS *taos, const char *sql, int64_t reqid) {
return taosQueryImplWithReqid(taos, sql, false, reqid);
}
TAOS_FIELD_E *taos_fetch_fields_e(TAOS_RES *res) {
if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) {
return NULL;
}
SReqResultInfo* pResInfo = tscGetCurResInfo(res);
return pResInfo->fields;
}
TAOS_ROW taos_fetch_row(TAOS_RES *res) {
if (res == NULL) {
return NULL;
@ -630,7 +647,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
}
if (pResultInfo->current < pResultInfo->numOfRows) {
doSetOneRowPtr(pResultInfo);
doSetOneRowPtr(pResultInfo, false);
pResultInfo->current += 1;
return pResultInfo->row;
} else {
@ -638,7 +655,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
return NULL;
}
doSetOneRowPtr(pResultInfo);
doSetOneRowPtr(pResultInfo, false);
pResultInfo->current += 1;
return pResultInfo->row;
}
@ -751,6 +768,14 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD
case TSDB_DATA_TYPE_BOOL:
len += tsnprintf(str + len, size - len, "%d", *((int8_t *)row[i]));
break;
case TSDB_DATA_TYPE_DECIMAL64:
case TSDB_DATA_TYPE_DECIMAL: {
uint32_t decimalLen = strlen(row[i]);
uint32_t copyLen = TMIN(size - len - 1, decimalLen);
(void)memcpy(str + len, row[i], copyLen);
len += copyLen;
} break;
default:
break;
}
@ -1260,7 +1285,7 @@ void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta
}
if (pQuery->haveResultSet) {
code = setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols);
code = setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols, pQuery->pResExtSchema, pRequest->isStmtBind);
setResPrecision(&pRequest->body.resInfo, pQuery->precision);
}
}
@ -1418,7 +1443,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SS
.parseSqlParam = pWrapper,
.setQueryFp = setQueryRequest,
.timezone = pTscObj->optionInfo.timezone,
.charsetCxt = pTscObj->optionInfo.charsetCxt};
.charsetCxt = pTscObj->optionInfo.charsetCxt,
.streamRunHistory = pRequest->streamRunHistory};
int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode);
(*pCxt)->biMode = biMode;
return TSDB_CODE_SUCCESS;
@ -2170,7 +2196,7 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col
}
STscStmt2 *pStmt = (STscStmt2 *)stmt;
if( atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum)>1) {
if (atomic_load_8((int8_t *)&pStmt->asyncBindParam.asyncBindNum) > 1) {
tscError("async bind param is still working, please try again later");
return TSDB_CODE_TSC_STMT_API_ERROR;
}
@ -2210,16 +2236,17 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col
}
}
SVCreateTbReq *pCreateTbReq = NULL;
if (bindv->tags && bindv->tags[i]) {
code = stmtSetTbTags2(stmt, bindv->tags[i]);
if (code) {
goto out;
}
} else if (pStmt->bInfo.tbType == TSDB_CHILD_TABLE && pStmt->sql.autoCreateTbl) {
code = stmtSetTbTags2(stmt, NULL);
if (code) {
return code;
}
code = stmtSetTbTags2(stmt, bindv->tags[i], &pCreateTbReq);
} else if (pStmt->sql.autoCreateTbl || pStmt->bInfo.needParse) {
code = stmtCheckTags2(stmt, &pCreateTbReq);
} else {
pStmt->sql.autoCreateTbl = false;
}
if (code) {
goto out;
}
if (bindv->bind_cols && bindv->bind_cols[i]) {
@ -2239,7 +2266,7 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col
goto out;
}
code = stmtBindBatch2(stmt, bind, col_idx);
code = stmtBindBatch2(stmt, bind, col_idx, pCreateTbReq);
if (TSDB_CODE_SUCCESS != code) {
goto out;
}
@ -2283,7 +2310,7 @@ int taos_stmt2_bind_param_a(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t c
(void)taosThreadCondSignal(&(pStmt->asyncBindParam.waitCond));
(void)atomic_sub_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1);
(void)taosThreadMutexUnlock(&(pStmt->asyncBindParam.mutex));
tscError("async bind failed, code:%d , %s", code_s, tstrerror(code_s));
tscError("async bind failed, code:%d , %s", code_s, tstrerror(code_s));
}
return code_s;

12
source/client/src/clientMonitor.c
View File

@ -669,8 +669,8 @@ static void monitorSendAllSlowLog() {
int64_t size = getFileSize(pClient->path);
if (size <= 0) {
if (size < 0) {
tscError("monitor failed to get file size:%s, err:%d", pClient->path, errno);
if (errno == ENOENT) {
tscError("monitor failed to get file size:%s, err:%d", pClient->path, ERRNO);
if (ERRNO == ENOENT) {
processFileRemoved(pClient);
}
}
@ -810,16 +810,16 @@ static void* monitorThreadFunc(void* param) {
static int32_t tscMonitortInit() {
TdThreadAttr thAttr;
if (taosThreadAttrInit(&thAttr) != 0) {
tscError("failed to init thread attr since %s", strerror(errno));
tscError("failed to init thread attr since %s", strerror(ERRNO));
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
if (taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE) != 0) {
tscError("failed to set thread attr since %s", strerror(errno));
tscError("failed to set thread attr since %s", strerror(ERRNO));
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
if (taosThreadCreate(&monitorThread, &thAttr, monitorThreadFunc, NULL) != 0) {
tscError("failed to create monitor thread since %s", strerror(errno));
tscError("failed to create monitor thread since %s", strerror(ERRNO));
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
@ -873,7 +873,7 @@ int32_t monitorInit() {
if (tsem2_init(&monitorSem, 0, 0) != 0) {
tscError("sem init error since %s", terrstr());
return TAOS_SYSTEM_ERROR(errno);
return TAOS_SYSTEM_ERROR(ERRNO);
}
code = taosOpenQueue(&monitorQueue);

122
source/client/src/clientMsgHandler.c
View File

@ -151,12 +151,14 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
POINTER_BYTES) != 0) {
tscError("failed to put appInfo into appInfo.pInstMapByClusterId");
} else {
#ifdef USE_MONITOR
MonitorSlowLogData data = {0};
data.clusterId = pTscObj->pAppInfo->clusterId;
data.type = SLOW_LOG_READ_BEGINNIG;
(void)monitorPutData2MonitorQueue(data); // ignore
monitorClientSlowQueryInit(connectRsp.clusterId);
monitorClientSQLReqInit(connectRsp.clusterId);
#endif
}
}
@ -680,7 +682,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
code = buildShowVariablesRsp(rsp.variables, &pRes);
}
if (TSDB_CODE_SUCCESS == code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, pRequest->isStmtBind);
}
if (code != 0) {
@ -835,7 +837,7 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
code = buildRetriveTableRspForCompactDb(&rsp, &pRes);
}
if (TSDB_CODE_SUCCESS == code) {
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false);
code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, pRequest->isStmtBind);
}
if (code != 0) {
@ -856,6 +858,119 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
return code;
}
static int32_t setCreateStreamFailedRsp(void* param, SDataBuf* pMsg, int32_t code) {
if (pMsg) {
taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pMsg->pData);
}
if (code != 0){
tscError("setCreateStreamFailedRsp since %s", tstrerror(code));
} else{
tscInfo("setCreateStreamFailedRsp success");
}
return code;
}
void sendCreateStreamFailedMsg(SRequestObj* pRequest, char* streamName){
int32_t code = 0;
tscInfo("send failed stream name to mgmt: %s", streamName);
int32_t size = INT_BYTES + strlen(streamName);
void *buf = taosMemoryMalloc(size);
if (buf == NULL) {
tscError("failed to strdup stream name: %s", terrstr());
return;
}
*(int32_t*)buf = pRequest->code;
memcpy(POINTER_SHIFT(buf, INT_BYTES), streamName, strlen(streamName));
SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
taosMemoryFree(buf);
tscError("failed to calloc msgSendInfo: %s", terrstr());
return;
}
sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = size, .handle = NULL};
sendInfo->requestId = generateRequestId();
sendInfo->requestObjRefId = 0;
sendInfo->msgType = TDMT_MND_FAILED_STREAM;
sendInfo->fp = setCreateStreamFailedRsp;
SEpSet epSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
code = asyncSendMsgToServer(pRequest->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo);
if (code != 0) {
tscError("failed to send failed stream name to mgmt since %s", tstrerror(code));
}
}
static void processCreateStreamSecondPhaseRsp(void* param, void* res, int32_t code) {
SRequestObj* pRequest = res;
if (code != 0 && param != NULL){
sendCreateStreamFailedMsg(pRequest, param);
}
taosMemoryFree(param);
destroyRequest(pRequest);
}
static char* getStreamName(SRequestObj* pRequest){
SCreateStreamStmt* pStmt = (SCreateStreamStmt*)(pRequest->pQuery->pRoot);
SName name;
int32_t code = tNameSetDbName(&name, pRequest->pTscObj->acctId, pStmt->streamName, strlen(pStmt->streamName));
if (TSDB_CODE_SUCCESS != code) {
tscError("failed to set db name for stream since %s", tstrerror(code));
return NULL;
} else{
char *streamName = taosMemoryCalloc(1, TSDB_STREAM_FNAME_LEN);
(void)tNameGetFullDbName(&name, streamName);
return streamName;
}
}
void processCreateStreamSecondPhase(SRequestObj* pRequest){
tscInfo("[create stream with histroy] create in second phase");
char *streamName = getStreamName(pRequest);
size_t sqlLen = strlen(pRequest->sqlstr);
SRequestObj* pRequestNew = NULL;
int32_t code = buildRequest(pRequest->pTscObj->id, pRequest->sqlstr, sqlLen, streamName, false, &pRequestNew, 0);
if (code != TSDB_CODE_SUCCESS) {
tscError("[create stream with histroy] create in second phase, build request failed since %s", tstrerror(code));
return;
}
pRequestNew->source = pRequest->source;
pRequestNew->body.queryFp = processCreateStreamSecondPhaseRsp;
pRequestNew->streamRunHistory = true;
doAsyncQuery(pRequestNew, false);
}
int32_t processCreateStreamFirstPhaseRsp(void* param, SDataBuf* pMsg, int32_t code) {
SRequestObj* pRequest = param;
if (code != TSDB_CODE_SUCCESS) {
setErrno(pRequest, code);
}
if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
if (removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)) != 0) {
tscError("failed to remove meta data for table");
}
}
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
if (code == 0 && !pRequest->streamRunHistory && tsStreamRunHistoryAsync){
processCreateStreamSecondPhase(pRequest);
}
if (pRequest->body.queryFp != NULL) {
pRequest->body.queryFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code);
} else {
if (tsem_post(&pRequest->body.rspSem) != 0) {
tscError("failed to post semaphore");
}
}
return code;
}
__async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
switch (msgType) {
case TDMT_MND_CONNECT:
@ -872,9 +987,12 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
return processAlterStbRsp;
case TDMT_MND_SHOW_VARIABLES:
return processShowVariablesRsp;
case TDMT_MND_CREATE_STREAM:
return processCreateStreamFirstPhaseRsp;
case TDMT_MND_COMPACT_DB:
return processCompactDbRsp;
default:
return genericRspCallback;
}
}

8
source/client/src/clientRawBlockWrite.c
View File

@ -516,7 +516,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(tvalue);
} else {
double val = 0;
GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64);
GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64, 0); // currently tag type can't be decimal, so pass 0 as typeMod
tvalue = cJSON_CreateNumber(val);
RAW_NULL_CHECK(tvalue);
}
@ -1060,6 +1060,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, uint32_t metaLen) {
SColCmpr* pCmp = &req.colCmpr.pColCmpr[i];
field.compress = pCmp->alg;
}
if (req.pExtSchemas) field.typeMod = req.pExtSchemas[i].typeMod;
RAW_NULL_CHECK(taosArrayPush(pReq.pColumns, &field));
}
pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField));
@ -1974,10 +1975,11 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe
int j = 0;
for (; j < pTableMeta->tableInfo.numOfColumns; j++) {
SSchema* pColSchema = &pTableMeta->schema[j];
SSchemaExt* pColExtSchema = &pTableMeta->schemaExt[j];
char* fieldName = pSW->pSchema[i].name;
if (strcmp(pColSchema->name, fieldName) == 0) {
if (checkSchema(pColSchema, fields, NULL, 0) != 0){
if (checkSchema(pColSchema, pColExtSchema, fields, NULL, 0) != 0){
return true;
}
break;
@ -2714,4 +2716,4 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, uint32_t metaLe
end:
tDeleteMqBatchMetaRsp(&rsp);
return code;
}
}

10
source/client/src/clientSml.c
View File

@ -37,9 +37,9 @@
kvVal->f = (float)result;
#define SET_BIGINT \
errno = 0; \
SET_ERRNO(0); \
int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \
if (errno == ERANGE) { \
if (ERRNO == ERANGE) { \
smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
return false; \
} \
@ -63,9 +63,9 @@
kvVal->i = result;
#define SET_UBIGINT \
errno = 0; \
SET_ERRNO(0); \
uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \
if (errno == ERANGE || result < 0) { \
if (ERRNO == ERANGE || result < 0) { \
smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
return false; \
} \
@ -1247,7 +1247,9 @@ void freeSSmlKv(void *data) {
SSmlKv *kv = (SSmlKv *)data;
if (kv->keyEscaped) taosMemoryFreeClear(kv->key);
if (kv->valueEscaped) taosMemoryFreeClear(kv->value);
#ifdef USE_GEOS
if (kv->type == TSDB_DATA_TYPE_GEOMETRY) geosFreeBuffer((void *)(kv->value));
#endif
if (kv->type == TSDB_DATA_TYPE_VARBINARY) taosMemoryFreeClear(kv->value);
}

10
source/client/src/clientSmlLine.c
View File

@ -97,13 +97,14 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
}
if (pVal->value[0] == 'g' || pVal->value[0] == 'G') { // geometry
#ifdef USE_GEOS
if (pVal->length >= NCHAR_ADD_LEN && pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"') {
int32_t code = initCtxGeomFromText();
if (code != TSDB_CODE_SUCCESS) {
return code;
}
char* tmp = taosMemoryCalloc(pVal->length, 1);
if (tmp == NULL){
char *tmp = taosMemoryCalloc(pVal->length, 1);
if (tmp == NULL) {
return terrno;
}
(void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN);
@ -115,12 +116,15 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
pVal->type = TSDB_DATA_TYPE_GEOMETRY;
if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
geosFreeBuffer((void*)(pVal->value));
geosFreeBuffer((void *)(pVal->value));
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_TSC_INVALID_VALUE;
#else
return TSDB_CODE_OPS_NOT_SUPPORT;
#endif
}
if (pVal->value[0] == 'b' || pVal->value[0] == 'B') { // varbinary

11
source/client/src/clientStmt.c
View File

@ -213,7 +213,7 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) {
int32_t stmtRestoreQueryFields(STscStmt* pStmt) {
SStmtQueryResInfo* pRes = &pStmt->sql.queryRes;
int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD);
int32_t size = pRes->numOfCols * sizeof(TAOS_FIELD_E);
pStmt->exec.pRequest->body.resInfo.numOfCols = pRes->numOfCols;
pStmt->exec.pRequest->body.resInfo.precision = pRes->precision;
@ -762,7 +762,7 @@ int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) {
atomic_store_8((int8_t*)&pStmt->sql.siInfo.tableColsReady, true);
} else {
STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock,
&pStmt->sql.siInfo));
&pStmt->sql.siInfo, NULL));
// taosMemoryFree(pParam->pTbData);
@ -809,7 +809,7 @@ int32_t stmtStartBindThread(STscStmt* pStmt) {
}
if (taosThreadCreate(&pStmt->bindThread, &thAttr, stmtBindThreadFunc, pStmt) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(ERRNO);
STMT_ERR_RET(terrno);
}
@ -1270,8 +1270,9 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
if (pStmt->sql.pQuery->haveResultSet) {
STMT_ERR_RET(setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema,
pStmt->sql.pQuery->numOfResCols));
pStmt->sql.pQuery->numOfResCols, pStmt->sql.pQuery->pResExtSchema, true));
taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema);
taosMemoryFreeClear(pStmt->sql.pQuery->pResExtSchema);
setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision);
}
@ -1861,7 +1862,7 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
}
*type = pField[idx].type;
*bytes = pField[idx].bytes;
*bytes = calcSchemaBytesFromTypeBytes(pField[idx].type, pField[idx].bytes, true);
_return:

199
source/client/src/clientStmt2.c
View File

@ -183,7 +183,7 @@ static int32_t stmtGetTbName(TAOS_STMT2* stmt, char** tbName) {
pStmt->sql.type = STMT_TYPE_MULTI_INSERT;
if ('\0' == pStmt->bInfo.tbName[0]) {
tscError("no table name set");
tscWarn("no table name set, OK if it is a stmt get fields");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR);
}
@ -240,9 +240,6 @@ static int32_t stmtUpdateInfo(TAOS_STMT2* stmt, STableMeta* pTableMeta, void* ta
STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash));
pStmt->sql.autoCreateTbl = autoCreateTbl;
if (pStmt->sql.autoCreateTbl) {
pStmt->sql.stbInterlaceMode = false;
}
return TSDB_CODE_SUCCESS;
}
@ -294,18 +291,18 @@ static int32_t stmtParseSql(STscStmt2* pStmt) {
}
STableDataCxt* pTableCtx = *pSrc;
if (pStmt->sql.stbInterlaceMode) {
int16_t lastIdx = -1;
// if (pStmt->sql.stbInterlaceMode) {
// int16_t lastIdx = -1;
for (int32_t i = 0; i < pTableCtx->boundColsInfo.numOfBound; ++i) {
if (pTableCtx->boundColsInfo.pColIndex[i] < lastIdx) {
pStmt->sql.stbInterlaceMode = false;
break;
}
// for (int32_t i = 0; i < pTableCtx->boundColsInfo.numOfBound; ++i) {
// if (pTableCtx->boundColsInfo.pColIndex[i] < lastIdx) {
// pStmt->sql.stbInterlaceMode = false;
// break;
// }
lastIdx = pTableCtx->boundColsInfo.pColIndex[i];
}
}
// lastIdx = pTableCtx->boundColsInfo.pColIndex[i];
// }
// }
if (NULL == pStmt->sql.pBindInfo) {
pStmt->sql.pBindInfo = taosMemoryMalloc(pTableCtx->boundColsInfo.numOfBound * sizeof(*pStmt->sql.pBindInfo));
@ -319,7 +316,6 @@ static int32_t stmtParseSql(STscStmt2* pStmt) {
static int32_t stmtCleanBindInfo(STscStmt2* pStmt) {
pStmt->bInfo.tbUid = 0;
pStmt->bInfo.tbSuid = 0;
pStmt->bInfo.tbVgId = -1;
pStmt->bInfo.tbType = 0;
pStmt->bInfo.needParse = true;
@ -331,7 +327,10 @@ static int32_t stmtCleanBindInfo(STscStmt2* pStmt) {
qDestroyBoundColInfo(pStmt->bInfo.boundTags);
taosMemoryFreeClear(pStmt->bInfo.boundTags);
}
pStmt->bInfo.stbFName[0] = 0;
if (!pStmt->sql.autoCreateTbl) {
pStmt->bInfo.stbFName[0] = 0;
pStmt->bInfo.tbSuid = 0;
}
return TSDB_CODE_SUCCESS;
}
@ -439,6 +438,9 @@ static int32_t stmtCleanSQLInfo(STscStmt2* pStmt) {
taosArrayDestroy(pStmt->sql.nodeList);
taosHashCleanup(pStmt->sql.pVgHash);
pStmt->sql.pVgHash = NULL;
if (pStmt->sql.fixValueTags) {
tdDestroySVCreateTbReq(pStmt->sql.fixValueTbReq);
}
void* pIter = taosHashIterate(pStmt->sql.pTableCache, NULL);
while (pIter) {
@ -691,12 +693,11 @@ static int32_t stmtAsyncOutput(STscStmt2* pStmt, void* param) {
atomic_store_8((int8_t*)&pStmt->sql.siInfo.tableColsReady, true);
} else {
STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock,
&pStmt->sql.siInfo));
int code = qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock,
&pStmt->sql.siInfo, pParam->pCreateTbReq);
// taosMemoryFree(pParam->pTbData);
(void)atomic_sub_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1);
STMT_ERR_RET(code);
}
return TSDB_CODE_SUCCESS;
}
@ -738,7 +739,7 @@ static int32_t stmtStartBindThread(STscStmt2* pStmt) {
}
if (taosThreadCreate(&pStmt->bindThread, &thAttr, stmtBindThreadFunc, pStmt) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(ERRNO);
STMT_ERR_RET(terrno);
}
@ -852,7 +853,7 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) {
pStmt->sql.siInfo.tableColsReady = true;
if (pStmt->options.asyncExecFn) {
if (tsem_init(&pStmt->asyncExecSem, 0, 1) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
terrno = TAOS_SYSTEM_ERROR(ERRNO);
(void)stmtClose2(pStmt);
return NULL;
}
@ -1019,7 +1020,7 @@ int stmtSetTbName2(TAOS_STMT2* stmt, const char* tbName) {
return TSDB_CODE_SUCCESS;
}
int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) {
int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags, SVCreateTbReq** pCreateTbReq) {
STscStmt2* pStmt = (STscStmt2*)stmt;
STMT_DLOG_E("start to set tbTags");
@ -1048,22 +1049,125 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) {
// tscWarn("no tags or cols bound in sql, will not bound tags");
// return TSDB_CODE_SUCCESS;
// }
STableDataCxt** pDataBlock =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERR_RET(TSDB_CODE_APP_ERROR);
if (pStmt->sql.autoCreateTbl && pStmt->sql.stbInterlaceMode) {
STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, pStmt->bInfo.tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb,
pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
STMT_ERR_RET(tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName));
}
STableDataCxt** pDataBlock = NULL;
if (pStmt->exec.pCurrBlock) {
pDataBlock = &pStmt->exec.pCurrBlock;
} else {
pDataBlock =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERR_RET(TSDB_CODE_TSC_STMT_CACHE_ERROR);
}
// pStmt->exec.pCurrBlock = *pDataBlock;
// if (pStmt->sql.stbInterlaceMode) {
// taosArrayDestroy(pStmt->exec.pCurrBlock->pData->aCol);
// (*pDataBlock)->pData->aCol = NULL;
// }
}
if (pStmt->bInfo.inExecCache && !pStmt->sql.autoCreateTbl) {
return TSDB_CODE_SUCCESS;
}
tscDebug("start to bind stmt tag values");
STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName,
void* boundTags = NULL;
if (pStmt->sql.stbInterlaceMode) {
boundTags = pStmt->sql.siInfo.boundTags;
*pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
if (NULL == pCreateTbReq) {
return terrno;
}
int32_t vgId = -1;
STMT_ERR_RET(stmtTryAddTableVgroupInfo(pStmt, &vgId));
(*pCreateTbReq)->uid = vgId;
} else {
boundTags = pStmt->bInfo.boundTags;
}
STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName,
pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt));
pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt, *pCreateTbReq));
return TSDB_CODE_SUCCESS;
}
int stmtCheckTags2(TAOS_STMT2* stmt, SVCreateTbReq** pCreateTbReq) {
STscStmt2* pStmt = (STscStmt2*)stmt;
STMT_DLOG_E("start to set tbTags");
if (pStmt->errCode != TSDB_CODE_SUCCESS) {
return pStmt->errCode;
}
if (!pStmt->sql.stbInterlaceMode) {
return TSDB_CODE_SUCCESS;
}
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
pStmt->bInfo.needParse = false;
}
STMT_ERR_RET(stmtCreateRequest(pStmt));
if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt));
if (!pStmt->sql.autoCreateTbl) {
return TSDB_CODE_SUCCESS;
}
}
if (pStmt->sql.stbInterlaceMode && NULL == pStmt->sql.siInfo.pDataCtx) {
STMT_ERR_RET(stmtInitStbInterlaceTableInfo(pStmt));
}
STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, pStmt->bInfo.tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb,
pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
STMT_ERR_RET(tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName));
STableDataCxt** pDataBlock = NULL;
if (pStmt->exec.pCurrBlock) {
pDataBlock = &pStmt->exec.pCurrBlock;
} else {
pDataBlock =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERR_RET(TSDB_CODE_TSC_STMT_CACHE_ERROR);
}
}
if (!((*pDataBlock)->pData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE)) {
return TSDB_CODE_SUCCESS;
}
if (pStmt->sql.fixValueTags) {
STMT_ERR_RET(cloneSVreateTbReq(pStmt->sql.fixValueTbReq, pCreateTbReq));
if ((*pCreateTbReq)->name) {
taosMemoryFree((*pCreateTbReq)->name);
}
(*pCreateTbReq)->name = taosStrdup(pStmt->bInfo.tbName);
int32_t vgId = -1;
STMT_ERR_RET(stmtTryAddTableVgroupInfo(pStmt, &vgId));
(*pCreateTbReq)->uid = vgId;
return TSDB_CODE_SUCCESS;
}
if ((*pDataBlock)->pData->pCreateTbReq) {
pStmt->sql.fixValueTags = true;
STMT_ERR_RET(cloneSVreateTbReq((*pDataBlock)->pData->pCreateTbReq, &pStmt->sql.fixValueTbReq));
STMT_ERR_RET(cloneSVreateTbReq(pStmt->sql.fixValueTbReq, pCreateTbReq));
(*pCreateTbReq)->uid = (*pDataBlock)->pMeta->vgId;
}
return TSDB_CODE_SUCCESS;
}
@ -1110,22 +1214,27 @@ static int stmtFetchStbColFields2(STscStmt2* pStmt, int32_t* fieldNum, TAOS_FIEL
}
STableDataCxt** pDataBlock = NULL;
bool cleanStb = false;
if (pStmt->sql.stbInterlaceMode) {
if (pStmt->sql.stbInterlaceMode && pStmt->sql.siInfo.pDataCtx != NULL) {
pDataBlock = &pStmt->sql.siInfo.pDataCtx;
} else {
cleanStb = true;
pDataBlock =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERRI_JRET(TSDB_CODE_APP_ERROR);
}
}
STMT_ERRI_JRET(qBuildStmtStbColFields(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.preCtbname, fieldNum, fields));
if (pStmt->bInfo.tbType == TSDB_SUPER_TABLE) {
if (NULL == pDataBlock || NULL == *pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERRI_JRET(TSDB_CODE_APP_ERROR);
}
STMT_ERRI_JRET(
qBuildStmtStbColFields(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.preCtbname, fieldNum, fields));
if (pStmt->bInfo.tbType == TSDB_SUPER_TABLE && cleanStb) {
pStmt->bInfo.needParse = true;
qDestroyStmtDataBlock(*pDataBlock);
*pDataBlock = NULL;
if (taosHashRemove(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)) != 0) {
tscError("get fileds %s remove exec blockHash fail", pStmt->bInfo.tbFName);
STMT_ERRI_JRET(TSDB_CODE_APP_ERROR);
@ -1281,6 +1390,10 @@ static int stmtAddBatch2(TAOS_STMT2* stmt) {
param->restoreTbCols = true;
param->next = NULL;
if (pStmt->sql.autoCreateTbl) {
pStmt->bInfo.tagsCached = true;
}
stmtEnqueue(pStmt, param);
return TSDB_CODE_SUCCESS;
@ -1334,7 +1447,7 @@ static int32_t stmtRestoreQueryFields(STscStmt2* pStmt) {
return TSDB_CODE_SUCCESS;
}
*/
int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx, SVCreateTbReq* pCreateTbReq) {
STscStmt2* pStmt = (STscStmt2*)stmt;
int32_t code = 0;
@ -1385,8 +1498,9 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
if (pStmt->sql.pQuery->haveResultSet) {
STMT_ERR_RET(setResSchemaInfo(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->pResSchema,
pStmt->sql.pQuery->numOfResCols));
pStmt->sql.pQuery->numOfResCols, pStmt->sql.pQuery->pResExtSchema, true));
taosMemoryFreeClear(pStmt->sql.pQuery->pResSchema);
taosMemoryFreeClear(pStmt->sql.pQuery->pResExtSchema);
setResPrecision(&pStmt->exec.pRequest->body.resInfo, pStmt->sql.pQuery->precision);
}
@ -1443,6 +1557,8 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
param->restoreTbCols = false;
tstrncpy(param->tblData.tbName, pStmt->bInfo.tbName, TSDB_TABLE_NAME_LEN);
param->pCreateTbReq = pCreateTbReq;
}
int64_t startUs3 = taosGetTimestampUs();
@ -1452,7 +1568,8 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
if (colIdx < 0) {
if (pStmt->sql.stbInterlaceMode) {
(*pDataBlock)->pData->flags = 0;
// (*pDataBlock)->pData->flags = 0;
(*pDataBlock)->pData->flags &= ~SUBMIT_REQ_COLUMN_DATA_FORMAT;
code = qBindStmtStbColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo,
pStmt->taos->optionInfo.charsetCxt);

12
source/client/src/clientTmq.c
View File

@ -1346,13 +1346,13 @@ static void doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
if (pParam == NULL) {
goto FAIL;
goto _ERR;
}
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
if (tmq == NULL) {
code = TSDB_CODE_TMQ_CONSUMER_CLOSED;
goto FAIL;
goto _ERR;
}
if (code != TSDB_CODE_SUCCESS) {
@ -1408,7 +1408,7 @@ static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
}
}
FAIL:
_ERR:
if (pParam && pParam->sync) {
SAskEpInfo* pInfo = pParam->pParam;
if (pInfo) {
@ -1760,7 +1760,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
// init semaphore
if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) {
tqErrorC("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId,
tstrerror(TAOS_SYSTEM_ERROR(errno)), pTmq->groupId);
tstrerror(TAOS_SYSTEM_ERROR(ERRNO)), pTmq->groupId);
SET_ERROR_MSG_TMQ("init t_sem failed")
goto _failed;
}
@ -3012,7 +3012,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
doFreeReqResultInfo(&pRspObj->resInfo);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(data->blockSchema, pRspObj->resIter);
if (pSW) {
TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols));
TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols, NULL, false));
}
}
@ -3028,7 +3028,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
pRspObj->resInfo.precision = precision;
pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows;
int32_t code = setResultDataPtr(&pRspObj->resInfo, convertUcs4);
int32_t code = setResultDataPtr(&pRspObj->resInfo, convertUcs4, false);
if (code != 0) {
return code;
}

102
source/client/test/stmt2Test.cpp
View File

@ -43,8 +43,7 @@ void checkError(TAOS_STMT2* stmt, int code) {
if (pStmt == nullptr || pStmt->sql.sqlStr == nullptr || pStmt->exec.pRequest == nullptr) {
printf("stmt api error\n stats : %d\n errstr : %s\n", pStmt->sql.status, taos_stmt_errstr(stmt));
} else {
printf("stmt api error\n sql : %s\n stats : %d\n errstr : %s\n", pStmt->sql.sqlStr, pStmt->sql.status,
taos_stmt_errstr(stmt));
printf("stmt api error\n sql : %s\n stats : %d\n", pStmt->sql.sqlStr, pStmt->sql.status);
}
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
}
@ -133,9 +132,9 @@ void do_query(TAOS* taos, const char* sql) {
taos_free_result(result);
}
void do_stmt(TAOS* taos, TAOS_STMT2_OPTION* option, const char* sql, int CTB_NUMS, int ROW_NUMS, int CYC_NUMS,
bool hastags, bool createTable) {
printf("test sql : %s\n", sql);
void do_stmt(const char* msg, TAOS* taos, TAOS_STMT2_OPTION* option, const char* sql, int CTB_NUMS, int ROW_NUMS,
int CYC_NUMS, bool hastags, bool createTable) {
printf("stmt2 [%s] : %s\n", msg, sql);
do_query(taos, "drop database if exists stmt2_testdb_1");
do_query(taos, "create database IF NOT EXISTS stmt2_testdb_1");
do_query(taos, "create stable stmt2_testdb_1.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))");
@ -903,14 +902,14 @@ TEST(stmt2Case, stmt2_stb_insert) {
TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0);
ASSERT_NE(taos, nullptr);
// normal
TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL};
TAOS_STMT2_OPTION option = {0, false, true, NULL, NULL};
{
do_stmt(taos, &option, "insert into `stmt2_testdb_1`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true,
true);
do_stmt("no-interlcace", taos, &option, "insert into `stmt2_testdb_1`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)",
3, 3, 3, true, true);
}
{
do_stmt(taos, &option, "insert into `stmt2_testdb_1`.? using `stmt2_testdb_1`.`stb` tags(?,?) values(?,?)", 3, 3, 3,
true, true);
do_stmt("no-interlcace", taos, &option,
"insert into `stmt2_testdb_1`.? using `stmt2_testdb_1`.`stb` tags(?,?) values(?,?)", 3, 3, 3, true, true);
}
// async
@ -918,28 +917,61 @@ TEST(stmt2Case, stmt2_stb_insert) {
aa->async_affected_rows = 0;
ASSERT_EQ(tsem_init(&aa->sem, 0, 0), TSDB_CODE_SUCCESS);
void* param = aa;
option = {0, true, true, stmtAsyncQueryCb, param};
option = {0, false, true, stmtAsyncQueryCb, param};
{
do_stmt(taos, &option, "insert into stmt2_testdb_1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, true);
do_stmt("no-interlcace & aync exec", taos, &option,
"insert into stmt2_testdb_1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, true);
}
{
do_stmt(taos, &option, "insert into stmt2_testdb_1.? using stmt2_testdb_1.stb (t1,t2)tags(?,?) (ts,b)values(?,?)",
3, 3, 3, true, true);
do_stmt("no-interlcace & aync exec", taos, &option,
"insert into stmt2_testdb_1.? using stmt2_testdb_1.stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3, 3, true,
true);
}
// { do_stmt(taos, &option, "insert into db.? values(?,?)", 3, 3, 3, false, true); }
// interlace = 0 & use db]
do_query(taos, "use stmt2_testdb_1");
option = {0, false, false, NULL, NULL};
{ do_stmt(taos, &option, "insert into stb (tbname,ts,b) values(?,?,?)", 3, 3, 3, false, true); }
{ do_stmt(taos, &option, "insert into ? using stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3, 3, true, true); }
{ do_stmt(taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); }
{
do_stmt("no-interlcace & no-db", taos, &option, "insert into stb (tbname,ts,b) values(?,?,?)", 3, 3, 3, false,
true);
}
{
do_stmt("no-interlcace & no-db", taos, &option, "insert into ? using stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3,
3, true, true);
}
{ do_stmt("no-interlcace & no-db", taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); }
// interlace = 1
option = {0, true, true, stmtAsyncQueryCb, param};
{ do_stmt(taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); }
{ do_stmt("interlcace & preCreateTB", taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); }
option = {0, true, true, NULL, NULL};
{ do_stmt(taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); }
{ do_stmt("interlcace & preCreateTB", taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); }
// auto create table
// interlace = 1
option = {0, true, true, NULL, NULL};
{
do_stmt("interlcace & no-preCreateTB", taos, &option, "insert into ? using stb (t1,t2)tags(?,?) (ts,b)values(?,?)",
3, 3, 3, true, false);
}
{
do_stmt("interlcace & no-preCreateTB", taos, &option,
"insert into stmt2_testdb_1.? using stb (t1,t2)tags(1,'abc') (ts,b)values(?,?)", 3, 3, 3, false, false);
}
{
do_stmt("interlcace & no-preCreateTB", taos, &option,
"insert into stmt2_testdb_1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, false);
}
// interlace = 0
option = {0, false, false, NULL, NULL};
{
do_stmt("no-interlcace & no-preCreateTB", taos, &option,
"insert into ? using stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3, 3, true, false);
}
{
do_stmt("no-interlcace & no-preCreateTB", taos, &option,
"insert into stmt2_testdb_1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, false);
}
do_query(taos, "drop database if exists stmt2_testdb_1");
(void)tsem_destroy(&aa->sem);
@ -967,7 +999,8 @@ TEST(stmt2Case, stmt2_insert_non_statndard) {
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
ASSERT_NE(stmt, nullptr);
const char* sql = "INSERT INTO stmt2_testdb_6.? using stmt2_testdb_6.stb1 (int_tag)tags(1) (ts) VALUES (?)";
int code = taos_stmt2_prepare(stmt, sql, 0);
printf("stmt2 [%s] : %s\n", "less params", sql);
int code = taos_stmt2_prepare(stmt, sql, 0);
checkError(stmt, code);
int total_affect_rows = 0;
@ -987,7 +1020,7 @@ TEST(stmt2Case, stmt2_insert_non_statndard) {
TAOS_STMT2_BIND* tagv[2] = {&tags1, &tags2};
TAOS_STMT2_BIND* paramv[2] = {&params1, &params2};
char* tbname[2] = {"tb1", "tb2"};
TAOS_STMT2_BINDV bindv = {2, &tbname[0], NULL, &paramv[0]};
TAOS_STMT2_BINDV bindv = {2, &tbname[0], tagv, &paramv[0]};
code = taos_stmt2_bind_param(stmt, &bindv, -1);
checkError(stmt, code);
@ -1004,10 +1037,12 @@ TEST(stmt2Case, stmt2_insert_non_statndard) {
// less cols and tags
{
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
TAOS_STMT2_OPTION option = {0, false, false, NULL, NULL};
ASSERT_NE(stmt, nullptr);
const char* sql = "INSERT INTO stmt2_testdb_6.stb1 (ts,int_tag,tbname) VALUES (?,?,?)";
int code = taos_stmt2_prepare(stmt, sql, 0);
printf("stmt2 [%s] : %s\n", "less params", sql);
int code = taos_stmt2_prepare(stmt, sql, 0);
checkError(stmt, code);
int total_affect_rows = 0;
@ -1026,7 +1061,7 @@ TEST(stmt2Case, stmt2_insert_non_statndard) {
TAOS_STMT2_BIND* tagv[2] = {&tags1, &tags2};
TAOS_STMT2_BIND* paramv[2] = {&params1, &params2};
char* tbname[2] = {"tb3", "tb4"};
char* tbname[2] = {"tb1", "tb2"};
TAOS_STMT2_BINDV bindv = {2, &tbname[0], &tagv[0], &paramv[0]};
code = taos_stmt2_bind_param(stmt, &bindv, -1);
checkError(stmt, code);
@ -1044,10 +1079,15 @@ TEST(stmt2Case, stmt2_insert_non_statndard) {
// disorder cols and tags
{
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
TAOS_STMT2_OPTION option = {0, false, false, NULL, NULL};
TAOS_STMT2* stmt = taos_stmt2_init(taos, &option);
ASSERT_NE(stmt, nullptr);
do_query(taos,
"INSERT INTO stmt2_testdb_6.stb1 (ts, int_tag, tbname) VALUES (1591060627000, 5, 'tb5')(1591060627000, "
"6,'tb6')");
const char* sql = "INSERT INTO stmt2_testdb_6.stb1 (binary_tag,int_col,tbname,ts,int_tag) VALUES (?,?,?,?,?)";
int code = taos_stmt2_prepare(stmt, sql, 0);
printf("stmt2 [%s] : %s\n", "disorder params", sql);
int code = taos_stmt2_prepare(stmt, sql, 0);
checkError(stmt, code);
int tag_i = 0;
@ -1096,6 +1136,7 @@ TEST(stmt2Case, stmt2_insert_non_statndard) {
ASSERT_NE(stmt, nullptr);
const char* sql =
"INSERT INTO stmt2_testdb_6.? using stmt2_testdb_6.stb1 (int_tag)tags(1) (int_col,ts)VALUES (?,?)";
printf("stmt2 [%s] : %s\n", "PK error", sql);
int code = taos_stmt2_prepare(stmt, sql, 0);
checkError(stmt, code);
@ -1131,11 +1172,10 @@ TEST(stmt2Case, stmt2_insert_db) {
ASSERT_NE(taos, nullptr);
do_query(taos, "drop database if exists stmt2_testdb_12");
do_query(taos, "create database IF NOT EXISTS stmt2_testdb_12");
do_query(taos, "create stable `stmt2_testdb_12`.`stb1`(ts timestamp, int_col int) tags(int_tag int)");
do_query(taos,
"create stable `stmt2_testdb_12`.`stb1` (ts timestamp, int_col int,long_col bigint,double_col "
"double,bool_col bool,binary_col binary(20),nchar_col nchar(20),varbinary_col varbinary(20),geometry_col "
"geometry(200)) tags(int_tag int,long_tag bigint,double_tag double,bool_tag bool,binary_tag "
"binary(20),nchar_tag nchar(20),varbinary_tag varbinary(20),geometry_tag geometry(200));");
"INSERT INTO `stmt2_testdb_12`.`stb1` (ts,int_tag,tbname) VALUES "
"(1591060627000,1,'tb1')(1591060627000,2,'tb2')");
TAOS_STMT2_OPTION option = {0, false, false, NULL, NULL};

2
source/common/src/cos_cp.c
View File

@ -217,7 +217,7 @@ static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
}
if (taosFsyncFile(fp) < 0) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(ERRNO), &lino, _exit);
}
_exit:

167
source/common/src/msg/tmsg.c
View File

@ -673,6 +673,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->bytes));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pField->name));
TAOS_CHECK_EXIT(tEncodeU32(&encoder, pField->compress));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->typeMod));
}
for (int32_t i = 0; i < pReq->numOfTags; ++i) {
@ -699,6 +700,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
}
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->deleteMark1));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->deleteMark2));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->keep));
ENCODESQL();
@ -759,6 +761,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.bytes));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, field.name));
TAOS_CHECK_EXIT(tDecodeU32(&decoder, &field.compress));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &field.typeMod));
if (taosArrayPush(pReq->pColumns, &field) == NULL) {
TAOS_CHECK_EXIT(terrno);
}
@ -809,6 +812,9 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->deleteMark1));
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->deleteMark2));
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->keep));
}
DECODESQL();
@ -916,7 +922,21 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq)
if (pReq->commentLen > 0) {
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->comment));
}
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->keep));
ENCODESQL();
if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN ||
pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
if (taosArrayGetSize(pReq->pTypeMods) > 0) {
int8_t hasTypeMod = 1;
TAOS_CHECK_EXIT(tEncodeI8(&encoder, hasTypeMod));
for (int32_t i = 0; i < pReq->pTypeMods->size; ++i) {
const STypeMod *pTypeMod = taosArrayGet(pReq->pTypeMods, i);
TAOS_CHECK_ERRNO(tEncodeI32(&encoder, *pTypeMod));
}
} else {
TAOS_CHECK_EXIT(tEncodeI8(&encoder, 0));
}
}
tEndEncode(&encoder);
_exit:
@ -978,9 +998,28 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
}
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->comment));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->keep));
}
DECODESQL();
if (!tDecodeIsEnd(&decoder) && (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN ||
pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION)) {
int8_t hasTypeMod = 0;
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &hasTypeMod));
if (hasTypeMod == 1) {
pReq->pTypeMods = taosArrayInit(pReq->numOfFields, sizeof(STypeMod));
if (!pReq->pTypeMods) {
TAOS_CHECK_EXIT(terrno);
}
for (int32_t i = 0; i < pReq->numOfFields; ++i) {
STypeMod typeMod = 0;
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &typeMod));
if (taosArrayPush(pReq->pTypeMods, &typeMod) == NULL) {
TAOS_CHECK_EXIT(terrno);
}
}
}
}
tEndDecode(&decoder);
_exit:
@ -993,6 +1032,7 @@ void tFreeSMAltertbReq(SMAlterStbReq *pReq) {
pReq->pFields = NULL;
taosMemoryFreeClear(pReq->comment);
FREESQL();
taosArrayDestroy(pReq->pTypeMods);
}
int32_t tSerializeSEpSet(void *buf, int32_t bufLen, const SEpSet *pEpset) {
@ -3907,7 +3947,7 @@ int32_t tSerializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp) {
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->tagsLen));
TAOS_CHECK_EXIT(tEncodeBinary(&encoder, pRsp->pTags, pRsp->tagsLen));
if (useCompress(pRsp->tableType)) {
if (withExtSchema(pRsp->tableType)) {
for (int32_t i = 0; i < pRsp->numOfColumns; ++i) {
SSchemaExt *pSchemaExt = &pRsp->pSchemaExt[i];
TAOS_CHECK_EXIT(tEncodeSSchemaExt(&encoder, pSchemaExt));
@ -3983,7 +4023,7 @@ int32_t tDeserializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp)
TAOS_CHECK_EXIT(tDecodeBinaryAlloc(&decoder, (void **)&pRsp->pTags, NULL));
if (!tDecodeIsEnd(&decoder)) {
if (useCompress(pRsp->tableType) && pRsp->numOfColumns > 0) {
if (withExtSchema(pRsp->tableType) && pRsp->numOfColumns > 0) {
pRsp->pSchemaExt = taosMemoryMalloc(sizeof(SSchemaExt) * pRsp->numOfColumns);
if (pRsp->pSchemaExt == NULL) {
TAOS_CHECK_EXIT(terrno);
@ -6078,7 +6118,7 @@ static int32_t tEncodeSTableMetaRsp(SEncoder *pEncoder, STableMetaRsp *pRsp) {
TAOS_CHECK_RETURN(tEncodeSSchema(pEncoder, pSchema));
}
if (useCompress(pRsp->tableType)) {
if (withExtSchema(pRsp->tableType)) {
for (int32_t i = 0; i < pRsp->numOfColumns; ++i) {
SSchemaExt *pSchemaExt = &pRsp->pSchemaExt[i];
TAOS_CHECK_RETURN(tEncodeSSchemaExt(pEncoder, pSchemaExt));
@ -6119,7 +6159,7 @@ static int32_t tDecodeSTableMetaRsp(SDecoder *pDecoder, STableMetaRsp *pRsp) {
}
if (!tDecodeIsEnd(pDecoder)) {
if (useCompress(pRsp->tableType) && pRsp->numOfColumns > 0) {
if (withExtSchema(pRsp->tableType) && pRsp->numOfColumns > 0) {
pRsp->pSchemaExt = taosMemoryMalloc(sizeof(SSchemaExt) * pRsp->numOfColumns);
if (pRsp->pSchemaExt == NULL) {
TAOS_CHECK_RETURN(terrno);
@ -10117,7 +10157,13 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
SFieldWithOptions *pField = taosArrayGet(pReq->pCols, i);
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pField->type));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pField->flags));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pField->bytes));
int32_t bytes = pField->bytes;
if (IS_DECIMAL_TYPE(pField->type)) {
uint8_t prec = 0, scale = 0;
extractTypeFromTypeMod(pField->type, pField->typeMod, &prec, &scale, NULL);
fillBytesForDecimalType(&bytes, pField->type, prec, scale);
}
TAOS_CHECK_EXIT(tEncodeI32(&encoder, bytes));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pField->name));
}
@ -10132,6 +10178,12 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->notifyEventTypes));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->notifyErrorHandle));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->notifyHistory));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->recalculateInterval));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pWstartName));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pWendName));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pGroupIdName));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->pIsWindowFilledName));
tEndEncode(&encoder);
_exit:
@ -10290,6 +10342,14 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->notifyHistory));
}
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pReq->recalculateInterval));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pWstartName));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pWendName));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pGroupIdName));
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->pIsWindowFilledName));
}
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
@ -10430,6 +10490,44 @@ _exit:
return code;
}
static int32_t tEncodeSExtSchema(SEncoder* pCoder, const SExtSchema* pExtSchema) {
int32_t code = 0, lino;
TAOS_CHECK_EXIT(tEncodeI32v(pCoder, pExtSchema->typeMod));
_exit:
return code;
}
int32_t tDecodeSExtSchema(SDecoder* pCoder, SExtSchema* pExtSchema) {
int32_t code = 0, lino;
TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &pExtSchema->typeMod));
_exit:
return code;
}
static int32_t tEncodeSExtSchemas(SEncoder* pCoder, const SExtSchema* pExtSchemas, int32_t nCol) {
int32_t code = 0, lino;
for (int32_t i = 0; i < nCol; ++i) {
TAOS_CHECK_EXIT(tEncodeSExtSchema(pCoder, pExtSchemas + i));
}
_exit:
return code;
}
static int32_t tDecodeSExtSchemas(SDecoder* pCoder, SExtSchema** ppExtSchema, int32_t nCol) {
int32_t code = 0, lino;
*ppExtSchema = tDecoderMalloc(pCoder, sizeof(SExtSchema) * nCol);
if (!*ppExtSchema) TAOS_CHECK_EXIT(terrno);
for (int32_t i = 0; i < nCol; ++i) {
TAOS_CHECK_EXIT(tDecodeSExtSchema(pCoder, (*ppExtSchema) + i));
}
_exit:
return code;
}
int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
int32_t code = 0;
int32_t lino;
@ -10453,6 +10551,13 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, pReq->colCmpred));
TAOS_CHECK_EXIT(tEncodeSColCmprWrapper(pCoder, &pReq->colCmpr));
TAOS_CHECK_EXIT(tEncodeI64(pCoder, pReq->keep));
if (pReq->pExtSchemas) {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, 1));
TAOS_CHECK_EXIT(tEncodeSExtSchemas(pCoder, pReq->pExtSchemas, pReq->schemaRow.nCols));
} else {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, 0));
}
tEndEncode(pCoder);
_exit:
@ -10487,6 +10592,16 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
if (!tDecodeIsEnd(pCoder)) {
TAOS_CHECK_EXIT(tDecodeSColCmprWrapperEx(pCoder, &pReq->colCmpr));
}
if (!tDecodeIsEnd(pCoder)) {
TAOS_CHECK_EXIT(tDecodeI64(pCoder, &pReq->keep));
}
if (!tDecodeIsEnd(pCoder)) {
int8_t hasExtSchema = 0;
TAOS_CHECK_EXIT(tDecodeI8(pCoder, &hasExtSchema));
if (hasExtSchema) {
TAOS_CHECK_EXIT(tDecodeSExtSchemas(pCoder, &pReq->pExtSchemas, pReq->schemaRow.nCols));
}
}
}
tEndDecode(pCoder);
@ -10536,6 +10651,12 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
// Encode Column Options: encode compress level
if (pReq->type == TSDB_SUPER_TABLE || pReq->type == TSDB_NORMAL_TABLE) {
TAOS_CHECK_EXIT(tEncodeSColCmprWrapper(pCoder, &pReq->colCmpr));
if (pReq->pExtSchemas) {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, 1));
TAOS_CHECK_EXIT(tEncodeSExtSchemas(pCoder, pReq->pExtSchemas, pReq->ntb.schemaRow.nCols));
} else {
TAOS_CHECK_EXIT(tEncodeI8(pCoder, 0));
}
}
tEndEncode(pCoder);
@ -10600,6 +10721,14 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
if (!tDecodeIsEnd(pCoder)) {
TAOS_CHECK_EXIT(tDecodeSColCmprWrapperEx(pCoder, &pReq->colCmpr));
}
if (!tDecodeIsEnd(pCoder)) {
int8_t hasExtSchema = 0;
TAOS_CHECK_EXIT(tDecodeI8(pCoder, &hasExtSchema));
if (hasExtSchema) {
TAOS_CHECK_EXIT(tDecodeSExtSchemas(pCoder, &pReq->pExtSchemas, pReq->ntb.schemaRow.nCols));
}
}
}
tEndDecode(pCoder);
@ -10957,6 +11086,9 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
}
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->ctimeMs));
TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pReq->source));
if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION || pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN) {
TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->typeMod));
}
tEndEncode(pEncoder);
_exit:
@ -11060,6 +11192,11 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
if (!tDecodeIsEnd(pDecoder)) {
TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pReq->source));
}
if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) {
if (!tDecodeIsEnd(pDecoder)) {
TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->typeMod));
}
}
tEndDecode(pDecoder);
_exit:
@ -11239,7 +11376,7 @@ int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal)
if (IS_VAR_DATA_TYPE(pOffsetVal->primaryKey.type)) {
TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, pOffsetVal->primaryKey.pData, pOffsetVal->primaryKey.nData));
} else {
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pOffsetVal->primaryKey.val));
TAOS_CHECK_EXIT(tEncodeI64(pEncoder, VALUE_GET_TRIVIAL_DATUM(&pOffsetVal->primaryKey)));
}
} else if (pOffsetVal->type == TMQ_OFFSET__LOG) {
@ -11270,7 +11407,7 @@ int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) {
TAOS_CHECK_EXIT(
tDecodeBinaryAlloc32(pDecoder, (void **)&pOffsetVal->primaryKey.pData, &pOffsetVal->primaryKey.nData));
} else {
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &pOffsetVal->primaryKey.val));
TAOS_CHECK_EXIT(tDecodeI64(pDecoder, &VALUE_GET_TRIVIAL_DATUM(&pOffsetVal->primaryKey)));
}
}
} else if (pOffsetVal->type == TMQ_OFFSET__LOG) {
@ -11301,7 +11438,7 @@ void tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
taosMemoryFree(tmp);
} else {
(void)snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%" PRId64, pVal->uid, pVal->ts,
pVal->primaryKey.type, pVal->primaryKey.val);
pVal->primaryKey.type, VALUE_GET_TRIVIAL_DATUM(&pVal->primaryKey));
}
}
}
@ -11683,7 +11820,7 @@ void tDeleteMqDataRsp(SMqDataRsp *rsp) { tDeleteMqDataRspCommon(rsp); }
int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
int32_t code = 0;
int32_t lino;
int32_t lino = 0;
TAOS_CHECK_EXIT(tEncodeMqDataRspCommon(pEncoder, pRsp));
TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pRsp->createTableNum));
@ -11725,7 +11862,6 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
}
}
}
_exit:
return code;
}
@ -11866,6 +12002,7 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm
// auto create table
if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) {
if (!(pSubmitTbData->pCreateTbReq)) {
uError("auto create table but request is NULL");
return TSDB_CODE_INVALID_MSG;
}
TAOS_CHECK_EXIT(tEncodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq));
@ -12005,7 +12142,7 @@ int32_t tEncodeSubmitReq(SEncoder *pCoder, const SSubmitReq2 *pReq) {
for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) {
SSubmitTbData *pSubmitTbData = taosArrayGet(pReq->aSubmitTbData, i);
if ((pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) && pSubmitTbData->pCreateTbReq == NULL) {
pSubmitTbData->flags = 0;
pSubmitTbData->flags &= ~SUBMIT_REQ_AUTO_CREATE_TABLE;
}
TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, pSubmitTbData));
}
@ -13391,3 +13528,7 @@ void tDeleteMqBatchMetaRsp(SMqBatchMetaRsp *pRsp) {
pRsp->batchMetaReq = NULL;
pRsp->batchMetaLen = NULL;
}
bool hasExtSchema(const SExtSchema *pExtSchema) {
return pExtSchema->typeMod != 0;
}

Some files were not shown because too many files have changed in this diff Show More