diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index e7802b3d0d..0000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: 1.0.{build} -image: - - Visual Studio 2015 - - macos -environment: - matrix: - - ARCH: amd64 - - ARCH: x86 -matrix: - exclude: - - image: macos - ARCH: x86 -for: - - - matrix: - only: - - image: Visual Studio 2015 - clone_folder: c:\dev\TDengine - clone_depth: 1 - - init: - - call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %ARCH% - - before_build: - - cd c:\dev\TDengine - - md build - - build_script: - - cd build - - cmake -G "NMake Makefiles" .. -DBUILD_JDBC=false - - nmake install - - - matrix: - only: - - image: macos - clone_depth: 1 - - build_script: - - mkdir debug - - cd debug - - cmake .. > /dev/null - - make > /dev/null -notifications: -- provider: Email - to: - - sangshuduo@gmail.com - on_build_success: true - on_build_failure: true - on_build_status_changed: true diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 6f98693add..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,13 +0,0 @@ -# Use the latest 2.1 version of CircleCI pipeline process engine. See: https://circleci.com/docs/2.0/configuration-reference -version: 2.1 -# Use a package of configuration called an orb. -orbs: - # Declare a dependency on the welcome-orb - welcome: circleci/welcome-orb@0.4.1 -# Orchestrate or schedule a set of jobs -workflows: - # Name the workflow "welcome" - welcome: - # Run the welcome/run job in its own container - jobs: - - welcome/run diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index d35c104830..0000000000 --- a/.drone.yml +++ /dev/null @@ -1,266 +0,0 @@ ---- -kind: pipeline -name: test_amd64 - -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: gcc - commands: - - apt-get update - - apt-get install -y cmake build-essential - - mkdir debug - - cd debug - - cmake .. - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 ---- -kind: pipeline -name: test_arm64_bionic - -platform: - os: linux - arch: arm64 -steps: -- name: build - image: arm64v8/ubuntu:bionic - commands: - - apt-get update - - apt-get install -y cmake build-essential - - mkdir debug - - cd debug - - cmake .. -DCPUTYPE=aarch64 > /dev/null - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 ---- -kind: pipeline -name: test_arm64_focal - -platform: - os: linux - arch: arm64 - -steps: -- name: build - image: arm64v8/ubuntu:focal - commands: - - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections - - apt-get update - - apt-get install -y -qq cmake build-essential - - mkdir debug - - cd debug - - cmake .. -DCPUTYPE=aarch64 > /dev/null - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 ---- -kind: pipeline -name: test_arm64_centos7 - -platform: - os: linux - arch: arm64 - -steps: -- name: build - image: arm64v8/centos:7 - commands: - - yum install -y gcc gcc-c++ make cmake git - - mkdir debug - - cd debug - - cmake .. -DCPUTYPE=aarch64 > /dev/null - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 ---- -kind: pipeline -name: test_arm64_centos8 - -platform: - os: linux - arch: arm64 - -steps: -- name: build - image: arm64v8/centos:8 - commands: - - dnf install -y gcc gcc-c++ make cmake epel-release git libarchive - - mkdir debug - - cd debug - - cmake .. -DCPUTYPE=aarch64 > /dev/null - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 ---- -kind: pipeline -name: test_arm_bionic - -platform: - os: linux - arch: arm - -steps: -- name: build - image: arm32v7/ubuntu:bionic - commands: - - apt-get update - - apt-get install -y cmake build-essential - - mkdir debug - - cd debug - - cmake .. -DCPUTYPE=aarch32 > /dev/null - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 ---- -kind: pipeline -name: build_trusty - -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: ubuntu:trusty - commands: - - apt-get update - - apt-get install -y gcc cmake3 build-essential git binutils-2.26 - - - mkdir debug - - cd debug - - cmake .. - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 ---- -kind: pipeline -name: build_xenial - -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: ubuntu:xenial - commands: - - apt-get update - - apt-get install -y gcc cmake build-essential - - mkdir debug - - cd debug - - cmake .. - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 ---- -kind: pipeline -name: build_bionic -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: ubuntu:bionic - commands: - - apt-get update - - apt-get install -y gcc cmake build-essential - - mkdir debug - - cd debug - - cmake .. - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 ---- -kind: pipeline -name: build_centos7 -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: ansible/centos7-ansible - commands: - - yum install -y gcc gcc-c++ make cmake - - mkdir debug - - cd debug - - cmake .. - - make -j4 - trigger: - event: - - pull_request - when: - branch: - - develop - - master - - 2.0 - - 3.0 \ No newline at end of file diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml new file mode 100644 index 0000000000..cd5f1eeeae --- /dev/null +++ b/.github/workflows/taosd-ci-build.yml @@ -0,0 +1,71 @@ +name: TDengine Build + +on: + pull_request: + branches: + - 'main' + - '3.0' + - '3.1' + paths-ignore: + - 'docs/**' + - 'packaging/**' + - 'tests/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + name: Build and test + + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.18 + + - name: Install system dependencies + run: | + sudo apt update -y + sudo apt install -y build-essential cmake \ + libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev \ + zlib1g pkg-config libssl-dev gawk + + - name: Build and install TDengine + run: | + mkdir debug && cd debug + cmake .. -DBUILD_TOOLS=true \ + -DBUILD_KEEPER=true \ + -DBUILD_HTTP=false \ + -DBUILD_TEST=false \ + -DBUILD_DEPENDENCY_TESTS=false + make -j 4 + sudo make install + which taosd + which taosadapter + which taoskeeper + + - name: Start taosd + run: | + cp /etc/taos/taos.cfg ./ + sudo echo "supportVnodes 256" >> taos.cfg + nohup sudo taosd -c taos.cfg & + + - name: Start taosadapter + run: nohup sudo taosadapter & + + - name: Run tests with taosBenchmark + run: | + taosBenchmark -t 10 -n 10 -y + taos -s "select count(*) from test.meters" + + - name: Clean up + if: always() + run: | + if pgrep taosd; then sudo pkill taosd; fi + if pgrep taosadapter; then sudo pkill taosadapter; fi diff --git a/.github/workflows/taoskeeper-ci.yml b/.github/workflows/taoskeeper-ci.yml index fbc662ffb2..7f84eaa401 100644 --- a/.github/workflows/taoskeeper-ci.yml +++ b/.github/workflows/taoskeeper-ci.yml @@ -1,4 +1,4 @@ -name: TaosKeeper CI +name: taosKeeper CI on: push: diff --git a/.lgtm.yml b/.lgtm.yml deleted file mode 100644 index fbcedead43..0000000000 --- a/.lgtm.yml +++ /dev/null @@ -1,402 +0,0 @@ -########################################################################################## -# Customize file classifications. # -# Results from files under any classifier will be excluded from LGTM # -# statistics. # -########################################################################################## - -########################################################################################## -# Use the `path_classifiers` block to define changes to the default classification of # -# files. # -########################################################################################## - -path_classifiers: - # docs: - # Identify the top-level file called `generate_javadoc.py` as documentation-related. - test: - # Override LGTM's default classification of test files by excluding all files. - - exclude: / - # Classify all files in the top-level directories tests/ and testsuites/ as test code. - - tests - # - testsuites - # Classify all files with suffix `.test` as test code. - # Note: use only forward slash / as a path separator. - # Use ** to indicate an arbitrary parent path. - # Use * to indicate any sequence of characters excluding /. - # Always enclose the expression in double quotes if it includes *. - # - "**/*.test" - # Refine the classifications above by excluding files in test/util/. - # - exclude: test/util - # The default behavior is to tag all files created during the - # build as `generated`. Results are hidden for generated code. You can tag - # further files as being generated by adding them to the `generated` section. - generated: - # Exclude all `*.c` files under the `ui/` directory from classification as - # generated code. - # - exclude: ui/**/*.c - # By default, all files not checked into the repository are considered to be - # 'generated'. - # The default behavior is to tag library code as `library`. Results are hidden - # for library code. You can tag further files as being library code by adding them - # to the `library` section. - library: - - exclude: deps/ - # The default behavior is to tag template files as `template`. Results are hidden - # for template files. You can tag further files as being template files by adding - # them to the `template` section. - template: - #- exclude: path/to/template/code/**/*.c - # Define your own category, for example: 'some_custom_category'. - some_custom_category: - # Classify all files in the top-level directory tools/ (or the top-level file - # called tools). - # - tools - -######################################################################################### -# Use the `queries` block to change the default display of query results. # -######################################################################################### - - # queries: - # Start by hiding the results of all queries. - # - exclude: "*" - # Then include all queries tagged 'security' and 'correctness', and with a severity of - # 'error'. - # - include: - # tags: - # - "security" - # - "correctness" - # severity: "error" - # Specifically hide the results of two queries. - # - exclude: cpp/use-of-goto - # - exclude: java/equals-on-unrelated-types - # Refine by including the `java/command-line-injection` query. - # - include: java/command-line-injection - -######################################################################################### -# Define changes to the default code extraction process. # -# Each block configures the extraction of a single language, and modifies actions in a # -# named step. Every named step includes automatic default actions, # -# except for the 'prepare' step. The steps are performed in the following sequence: # -# prepare # -# after_prepare # -# configure (C/C++ only) # -# python_setup (Python only) # -# before_index # -# index # -########################################################################################## - -######################################################################################### -# Environment variables available to the steps: # -######################################################################################### - -# LGTM_SRC -# The root of the source tree. -# LGTM_WORKSPACE -# An existing (initially empty) folder outside the source tree. -# Used for temporary download and setup commands. - -######################################################################################### -# Use the extraction block to define changes to the default code extraction process # -# for one or more languages. The settings for each language are defined in a child # -# block, with one or more steps. # -######################################################################################### - -extraction: - # Define settings for C/C++ analysis - ##################################### - cpp: - # The `prepare` step exists for customization on LGTM.com only. - prepare: - # # The `packages` section is valid for LGTM.com only. It names Ubuntu packages to - # # be installed. - packages: - - cmake - # Add an `after-prepare` step if you need to run commands after the prepare step. - # Each command should be listed on a separate line. - # This step is useful for C/C++ analysis where you want to prepare the environment - # for the `configure` step without changing the default behavior for that step. - # after_prepare: - #- export GNU_MAKE=make - #- export GIT=true - # The `configure` step generates build configuration files which the `index` step - # then uses to build the codebase. - configure: - command: - - mkdir build - - cd build - - cmake .. - # - ./prepare_deps - # Optional step. You should add a `before_index` step if you need to run commands - # before the `index` step. - # before_index: - # - export BOOST_DIR=$LGTM_SRC/boost - # - export GTEST_DIR=$LGTM_SRC/googletest - # - export HUNSPELL_DIR=$LGTM_SRC/hunspell - # - export CRYPTOPP_DIR=$LGTM_SRC/cryptopp - # The `index` step builds the code and extracts information during the build - # process. - index: - # Override the autobuild process by specifying a list of custom build commands - # to use instead. - build_command: - - cd build - - make - # - $GNU_MAKE -j2 -s - # Specify that all project or solution files should be used for extraction. - # Default: false. - # all_solutions: true - # Specify a list of one or more project or solution files for extraction. - # Default: LGTM chooses the file closest to the root of the repository (this may - # fail if there are multiple candidates). - # solution: - # - myProject.sln - # Specify MSBuild settings - # msbuild: - # Specify a list of additional arguments to MSBuild. Default: empty. - # arguments: /p:Platform=x64 /p:Configuration=Release - # Specify the MSBuild configuration to use, for example, debug or release. - # Default: read from the solution file or files. - # configuration: - # Specify the platform to target, for example: x86, x64, or Any CPU. - # Default: read from the solution file or files. - # platform: - # Specify the MSBuild target. Default: rebuild. - # target: - # Specify whether or not to perform a NuGet restore for extraction. Default: true. - # nuget_restore: false - # Specify a version of Microsoft Visual Studio to use for MSBuild or any custom - # build commands (build_command). For example: - # 10 for Visual Studio 2010 - # 12 for Visual Studio 2012 - # 14 for Visual Studio 2015 - # 15 for Visual Studio 2017 - # Default: read from project files. - # vstools_version: 10 - - # Define settings for C# analysis - ################################## - # csharp: - # The `prepare` step exists for customization on LGTM.com only. - # prepare: - # packages: - # - example_package - # Add an `after-prepare` step if you need to run commands after the `prepare` step. - # Each command should be listed on a separate line. - # after_prepare: - # - export PATH=$LGTM_WORKSPACE/tools:$PATH - # The `index` step builds the code and extracts information during the build - # process. - #index: - # Specify that all project or solution files should be used for extraction. - # Default: false. - # all_solutions: true - # Specify a list of one or more project or solution files for extraction. - # Default: LGTM chooses the file closest to the root of the repository (this may - # fail if there are multiple candidates). - # solution: - # - myProject.sln - # Override the autobuild process by specifying a list of custom build commands - # to use instead. - # build_command: - # - ./example-compile-all.sh - # By default, LGTM analyzes the code by building it. You can override this, - # and tell LGTM not to build the code. Beware that this can lead - # to less accurate results. - # buildless: true - # Specify .NET Core settings. - # dotnet: - # Specify additional arguments to `dotnet build`. - # Default: empty. - # arguments: "example_arg" - # Specify the version of .NET Core SDK to use. - # Default: The version installed on the build machine. - # version: 2.1 - # Specify MSBuild settings. - # msbuild: - # Specify a list of additional arguments to MSBuild. Default: empty. - # arguments: /P:WarningLevel=2 - # Specify the MSBuild configuration to use, for example, debug or release. - # Default: read from the solution file or files. - # configuration: release - # Specify the platform to target, for example: x86, x64, or Any CPU. - # Default: read from the solution file or files. - # platform: x86 - # Specify the MSBuild target. Default: rebuild. - # target: notest - # Specify whether or not to perform a NuGet restore for extraction. Default: true. - # nuget_restore: false - # Specify a version of Microsoft Visual Studio to use for MSBuild or any custom - # build commands (build_command). For example: - # 10 for Visual Studio 2010 - # 12 for Visual Studio 2012 - # 14 for Visual Studio 2015 - # 15 for Visual Studio 2017 - # Default: read from project files - # vstools_version: 10 - # Specify additional options for the extractor, - # for example --fast to perform a faster extraction that produces a smaller - # database. - # extractor: "--fast" - - # Define settings for Go analysis - ################################## - # go: - # The `prepare` step exists for customization on LGTM.com only. - # prepare: - # packages: - # - example_package - # Add an `after-prepare` step if you need to run commands after the `prepare` step. - # Each command should be listed on a separate line. - # after_prepare: - # - export PATH=$LGTM_WORKSPACE/tools:$PATH - # The `index` step builds the code and extracts information during the build - # process. - # index: - # Override the autobuild process by specifying a list of custom build commands - # to use instead. - # build_command: - # - ./compile-all.sh - - # Define settings for Java analysis - #################################### - # java: - # The `prepare` step exists for customization on LGTM.com only. - # prepare: - # packages: - # - example_package - # Add an `after-prepare` step if you need to run commands after the prepare step. - # Each command should be listed on a separate line. - # after_prepare: - # - export PATH=$LGTM_WORKSPACE/tools:$PATH - # The `index` step extracts information from the files in the codebase. - # index: - # Specify Gradle settings. - # gradle: - # Specify the required Gradle version. - # Default: determined automatically. - # version: 4.4 - # Override the autobuild process by specifying a list of custom build commands - # to use instead. - # build_command: ./compile-all.sh - # Specify the Java version required to build the project. - # java_version: 11 - # Specify whether to extract Java .properties files - # Default: false - # properties_files: true - # Specify Maven settings. - # maven: - # Specify the path (absolute or relative) of a Maven settings file to use. - # Default: Maven uses a settings file in the default location, if it exists. - # settings_file: /opt/share/settings.xml - # Specify the path of a Maven toolchains file. - # Default: Maven uses a toolchains file in the default location, if it exists. - # toolchains_file: /opt/share/toolchains.xml - # Specify the required Maven version. - # Default: the Maven version is determined automatically, where feasible. - # version: 3.5.2 - # Specify how XML files should be extracted: - # all = extract all XML files. - # default = only extract XML files named `AndroidManifest.xml`, `pom.xml`, and `web.xml`. - # disabled = do not extract any XML files. - # xml_mode: all - - # Define settings for JavaScript analysis - ########################################## - # javascript: - # The `prepare` step exists for customization on LGTM.com only. - # prepare: - # packages: - # - example_package - # Add an `after-prepare` step if you need to run commands after the prepare step. - # Each command should be listed on a separate line. - # after_prepare: - # - export PATH=$LGTM_WORKSPACE/tools:$PATH - # The `index` step extracts information from the files in the codebase. - # index: - # Specify a list of files and folders to extract. - # Default: The project root directory. - # include: - # - src/js - # Specify a list of files and folders to exclude from extraction. - # exclude: - # - thirdparty/lib - # You can add additional file types for LGTM to extract, by mapping file - # extensions (including the leading dot) to file types. The usual - # include/exclude patterns apply, so, for example, `.jsm` files under - # `thirdparty/lib` will not be extracted. - # filetypes: - # ".jsm": "js" - # ".tmpl": "html" - # Specify a list of glob patterns to include/exclude files from extraction; this - # is applied on top of the include/exclude paths from above; patterns are - # processed in the same way as for path classifiers above. - # Default: include all files with known extensions (such as .js, .ts and .html), - # but exclude files ending in `-min.js` or `.min.js` and folders named `node_modules` - # or `bower_components` - # filters: - # exclude any *.ts files anywhere. - # - exclude: "**/*.ts" - # but include *.ts files under src/js/typescript. - # - include: "src/js/typescript/**/*.ts" - # Specify how TypeScript files should be extracted: - # none = exclude all TypeScript files. - # basic = extract syntactic information from TypeScript files. - # full = extract syntactic and type information from TypeScript files. - # Default: full. - # typescript: basic - # By default, LGTM doesn't extract any XML files. You can override this by - # using the `xml_mode` property and setting it to `all`. - # xml_mode: all - - # Define settings for Python analysis - ###################################### - # python: - # # The `prepare` step exists for customization on LGTM.com only. - # # prepare: - # # # The `packages` section is valid for LGTM.com only. It names packages to - # # # be installed. - # # packages: libpng-dev - # # This step is useful for Python analysis where you want to prepare the - # # environment for the `python_setup` step without changing the default behavior - # # for that step. - # after_prepare: - # - export PATH=$LGTM_WORKSPACE/tools:$PATH - # # This sets up the Python interpreter and virtual environment, ready for the - # # `index` step to extract the codebase. - # python_setup: - # # Specify packages that should NOT be installed despite being mentioned in the - # # requirements.txt file. - # # Default: no package marked for exclusion. - # exclude_requirements: - # - pywin32 - # # Specify a list of pip packages to install. - # # If any of these packages cannot be installed, the extraction will fail. - # requirements: - # - Pillow - # # Specify a list of requirements text files to use to set up the environment, - # # or false for none. Default: any requirements.txt, test-requirements.txt, - # # and similarly named files identified in the codebase are used. - # requirements_files: - # - required-packages.txt - # # Specify a setup.py file to use to set up the environment, or false for none. - # # Default: any setup.py files identified in the codebase are used in preference - # # to any requirements text files. - # setup_py: new-setup.py - # # Override the version of the Python interpreter used for setup and extraction - # # Default: Python 3. - # version: 2 - # # Optional step. You should add a `before_index` step if you need to run commands - # # before the `index` step. - # before_index: - # - antlr4 -Dlanguage=Python3 Grammar.g4 - # # The `index` step extracts information from the files in the codebase. - # index: - # # Specify a list of files and folders to exclude from extraction. - # # Default: Git submodules and Subversion externals. - # exclude: - # - legacy-implementation - # - thirdparty/libs - # filters: - # - exclude: "**/documentation/examples/snippets/*.py" - # - include: "**/documentation/examples/test_application/*" - # include: - # - example/to/include diff --git a/README.md b/README.md index f827c38975..e4814cee67 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -

-

+[![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/taosdata/tdengine/taosd-ci-build.yml)](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=3.0)](https://coveralls.io/github/taosdata/TDengine?branch=3.0) +![GitHub commit activity](https://img.shields.io/github/commit-activity/m/taosdata/tdengine) +
+![GitHub Release](https://img.shields.io/github/v/release/taosdata/tdengine) +![GitHub License](https://img.shields.io/github/license/taosdata/tdengine) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
[![Twitter Follow](https://img.shields.io/twitter/follow/tdenginedb?label=TDengine&style=social)](https://twitter.com/tdenginedb) diff --git a/docs/en/07-develop/01-connect.md b/docs/en/07-develop/01-connect.md index af5f171f8c..c14eed311a 100644 --- a/docs/en/07-develop/01-connect.md +++ b/docs/en/07-develop/01-connect.md @@ -109,7 +109,7 @@ If you are using Maven to manage your project, simply add the following dependen com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 ``` diff --git a/docs/en/07-develop/05-stmt.md b/docs/en/07-develop/05-stmt.md index 4503bb8bd3..11b055bcf9 100644 --- a/docs/en/07-develop/05-stmt.md +++ b/docs/en/07-develop/05-stmt.md @@ -28,8 +28,15 @@ Next, we continue to use smart meters as an example to demonstrate the efficient + +There are two kinds of interfaces for parameter binding: one is the standard JDBC interface, and the other is an extended interface. The extended interface offers better performance. + ```java -{{#include docs/examples/java/src/main/java/com/taos/example/WSParameterBindingBasicDemo.java:para_bind}} +{{#include docs/examples/java/src/main/java/com/taos/example/WSParameterBindingStdInterfaceDemo.java:para_bind}} +``` + +```java +{{#include docs/examples/java/src/main/java/com/taos/example/WSParameterBindingExtendInterfaceDemo.java:para_bind}} ``` This is a [more detailed parameter binding example](https://github.com/taosdata/TDengine/blob/main/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingFullDemo.java) @@ -91,14 +98,24 @@ This is a [more detailed parameter binding example](https://github.com/taosdata/ ```python -{{#include docs/examples/python/stmt_native.py}} +{{#include docs/examples/python/stmt2_native.py}} ``` + +The example code for binding parameters with stmt2 (Go connector v3.6.0 and above, TDengine v3.3.5.0 and above) is as follows: + +```go +{{#include docs/examples/go/stmt2/native/main.go}} +``` + +The example code for binding parameters with stmt is as follows: + ```go {{#include docs/examples/go/stmt/native/main.go}} ``` + diff --git a/docs/en/10-third-party/01-collection/flink.md b/docs/en/10-third-party/01-collection/flink.md new file mode 100644 index 0000000000..dea8fedc05 --- /dev/null +++ b/docs/en/10-third-party/01-collection/flink.md @@ -0,0 +1,375 @@ +--- +sidebar_label: Flink +title: TDengine Flink Connector +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Apache Flink is an open-source distributed stream batch integrated processing framework supported by the Apache Software Foundation, which can be used for many big data processing scenarios such as stream processing, batch processing, complex event processing, real-time data warehouse construction, and providing real-time data support for machine learning. At the same time, Flink has a wealth of connectors and various tools that can interface with numerous different types of data sources to achieve data reading and writing. In the process of data processing, Flink also provides a series of reliable fault-tolerant mechanisms, effectively ensuring that tasks can run stably and continuously even in the event of unexpected situations. + +With the help of TDengine's Flink connector, Apache Flink can seamlessly integrate with the TDengine database. On the one hand, it can accurately store the results obtained after complex calculations and deep analysis into the TDengine database, achieving efficient storage and management of data; On the other hand, it is also possible to quickly and stably read massive amounts of data from the TDengine database, and conduct comprehensive and in-depth analysis and processing on this basis, fully tapping into the potential value of the data, providing strong data support and scientific basis for enterprise decision-making, greatly improving the efficiency and quality of data processing, and enhancing the competitiveness and innovation ability of enterprises in the digital age. + +## Prerequisites + +Prepare the following environment: + +- TDengine cluster has been deployed and is running normally (both enterprise and community versions are available) +- TaosAdapter can run normally. +- Apache Flink v1.19.0 or above is installed. Please refer to the installation of Apache Flink [Official documents](https://flink.apache.org/) + +## Supported platforms + +Flink Connector supports all platforms that can run Flink 1.19 and above versions. + +## Version History + +| Flink Connector Version | Major Changes | TDengine Version| +|-------------------------| ------------------------------------ | ---------------- | +| 2.0.0 | 1.Support SQL queries on data in TDengine database
2. Support CDC subscription to data in TDengine database
3. Supports reading and writing to TDengine database using Table SQL | 3.3.5.0 and above versions| +| 1.0.0 | Support Sink function to write data from other sources to TDengine in the future| 3.3.2.0 and above versions| + +## Exception and error codes + +After the task execution fails, check the Flink task execution log to confirm the reason for the failure +Please refer to: + +| Error Code | Description | Suggested Actions | +| ---------------- |------------------------------------------------------- | -------------------- | +|0xa000 | connection param error | Connector parameter error. +|0xa001 | the groupid parameter of CDC is incorrect | The groupid parameter of CDC is incorrect.| +|0xa002 | wrong topic parameter for CDC | The topic parameter for CDC is incorrect.| +|0xa010 | database name configuration error | database name configuration error.| +|0xa011 | table name configuration error | Table name configuration error.| +|0xa012 | no data was obtained from the data source | Failed to retrieve data from the data source.| +|0xa013 | value.deserializer parameter not set | No serialization method set.| +|0xa014 | list of column names set incorrectly | List of column names for target table not set. | +|0x2301 | connection already closed | The connection has been closed. Check the connection status or create a new connection to execute the relevant instructions.| +|0x2302 | this operation is NOT supported currently | The current interface is not supported, you can switch to other connection methods.| +|0x2303 | invalid variables | The parameter is invalid. Please check the corresponding interface specification and adjust the parameter type and size.| +|0x2304 | statement is closed | Statement has already been closed. Please check if the statement is closed and reused, or if the connection is working properly.| +|0x2305 | resultSet is closed | The ResultSet has been released. Please check if the ResultSet has been released and used again.| +|0x230d | parameter index out of range | parameter out of range, please check the reasonable range of the parameter.| +|0x230e | connection already closed | The connection has been closed. Please check if the connection is closed and used again, or if the connection is working properly.| +|0x230f | unknown SQL type in TDengine | Please check the Data Type types supported by TDengine.| +|0x2315 | unknown tao type in TDengine | Did the correct TDengine data type be specified when converting TDengine data type to JDBC data type.| +|0x2319 | user is required | Username information is missing when creating a connection.| +|0x231a | password is required | Password information is missing when creating a connection.| +|0x231d | can't create connection with server within | Increase connection time by adding the parameter httpConnectTimeout, or check the connection status with taosAdapter.| +|0x231e | failed to complete the task within the specified time | Increase execution time by adding the parameter messageWaitTimeout, or check the connection with taosAdapter.| +|0x2352 | unsupported encoding | An unsupported character encoding set was specified under the local connection.| +|0x2353 | internal error of database, Please see taoslog for more details | An error occurred while executing prepareStatement on the local connection. Please check the taoslog for problem localization.| +|0x2354 | connection is NULL | Connection has already been closed while executing the command on the local connection. Please check the connection with TDengine.| +|0x2355 | result set is NULL | Local connection to obtain result set, result set exception, please check connection status and retry.| +|0x2356 | invalid num of fields | The meta information obtained from the local connection result set does not match.| +|0x2357 | empty SQL string | Fill in the correct SQL for execution.| +|0x2371 | consumer properties must not be null | When creating a subscription, the parameter is empty. Please fill in the correct parameter.| +|0x2375 | topic reference has been destroyed | During the process of creating a data subscription, the topic reference was released. Please check the connection with TDengine.| +|0x2376 | failed to set consumer topic, Topic name is empty | During the process of creating a data subscription, the subscription topic name is empty. Please check if the specified topic name is filled in correctly.| +|0x2377 | consumer reference has been destroyed | The subscription data transmission channel has been closed, please check the connection with TDengine.| +|0x2378 | consumer create error | Failed to create data subscription. Please check the taos log based on the error message to locate the problem.| +|0x237a | vGroup not found in result set VGroup | Not assigned to the current consumer, due to the Rebalance mechanism, the relationship between Consumer and VGroup is not bound.| + +## Data type mapping + +TDengine currently supports timestamp, number, character, and boolean types, and the corresponding type conversions with Flink RowData Type are as follows: + +| TDengine DataType | Flink RowDataType | +| ----------------- | ------------------ | +| TIMESTAMP | TimestampData | +| INT | Integer | +| BIGINT | Long | +| FLOAT | Float | +| DOUBLE | Double | +| SMALLINT | Short | +| TINYINT | Byte | +| BOOL | Boolean | +| BINARY | byte[] | +| NCHAR | StringData | +| JSON | StringData | +| VARBINARY | byte[] | +| GEOMETRY | byte[] | + +## Instructions for use + +### Flink Semantic Selection Instructions + +The semantic reason for using At Least One (at least once) is: + +- TDengine currently does not support transactions and cannot perform frequent checkpoint operations and complex transaction coordination. +- Due to TDengine's use of timestamps as primary keys, downstream operators of duplicate data can perform filtering operations to avoid duplicate calculations. +- Using At Least One (at least once) to ensure high data processing performance and low data latency, the setting method is as follows: + +Instructions: + +```java +StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); +env.enableCheckpointing(5000); +env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.AT_LEAST_ONCE); +``` + +If using Maven to manage a project, simply add the following dependencies in pom.xml. + +```xml + + com.taosdata.flink + flink-connector-tdengine + 2.0.0 + +``` + +The parameters for establishing a connection include URL and Properties. +The URL specification format is: + +`jdbc: TAOS-WS://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&timezone={timezone}]` + +Parameter description: + +- User: Login TDengine username, default value is' root '. +- Password: User login password, default value 'taosdata'. +- database_name: database name。 +- timezone: time zone。 +- HttpConnectTimeout: The connection timeout time, measured in milliseconds, with a default value of 60000. +- MessageWaitTimeout: The timeout period for a message, measured in milliseconds, with a default value of 60000. +- UseSSL: Whether SSL is used in the connection. + +### Source + +Source retrieves data from the TDengine database, converts it into a format and type that Flink can handle internally, and reads and distributes it in parallel, providing efficient input for subsequent data processing. +By setting the parallelism of the data source, multiple threads can read data from the data source in parallel, improving the efficiency and throughput of data reading, and fully utilizing cluster resources for large-scale data processing capabilities. + +#### Source Properties + +The configuration parameters in Properties are as follows: + +- TDengineConfigParams.PROPERTY_KEY_USER: Login to TDengine username, default value is 'root '. +- TDengineConfigParams.PROPERTY_KEY_PASSWORD: User login password, default value 'taosdata'. +- TDengineConfigParams.VALUE_DESERIALIZER: The downstream operator receives the result set deserialization method. If the received result set type is `RowData` of `Flink`, it only needs to be set to `RowData`. It is also possible to inherit `TDengineRecordDeserialization` and implement `convert` and `getProducedType` methods, customizing the deserialization method based on `ResultSet` of `SQL`. +- TDengineConfigParams.TD_BATCH_MODE: This parameter is used to batch push data to downstream operators. If set to True, when creating the `TDengine Source` object, it is necessary to specify the data type as a `Template` form of the `SourceRecords` type. +- TDengineConfigParams.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: Message timeout time, in milliseconds, default value is 60000. +- TDengineConfigParams.PROPERTY_KEY_ENABLE_COMPRESSION: Is compression enabled during the transmission process. true: Enable, false: Not enabled. The default is false. +- TDengineConfigParams.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: Whether to enable automatic reconnection. true: Enable, false: Not enabled. The default is false. +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_INTERVAL_MS: Automatic reconnection retry interval, in milliseconds, default value 2000. It only takes effect when `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` is true. +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_RETRY_COUNT: The default value for automatic reconnection retry is 3, which only takes effect when `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` is true. +- TDengineConfigParams.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: Turn off SSL certificate verification. true: Enable, false: Not enabled. The default is false. + + +#### Split by time + +Users can split the SQL query into multiple subtasks based on time, entering: start time, end time, split interval, time field name. The system will split and obtain data in parallel according to the set interval (time left closed and right open). + +```java +{{#include docs/examples/flink/Main.java:time_interval}} +``` + +Splitting by Super Table TAG + +Users can split the query SQL into multiple query conditions based on the TAG field of the super table, and the system will split them into subtasks corresponding to each query condition, thereby obtaining data in parallel. + +```java +{{#include docs/examples/flink/Main.java:tag_split}} +``` + +Classify by table + +Support sharding by inputting multiple super tables or regular tables with the same table structure. The system will split them according to the method of one table, one task, and then obtain data in parallel. + +```java +{{#include docs/examples/flink/Main.java:table_split}} +``` + +Use Source connector + +The query result is RowData data type example: + +

+RowData Source +```java +{{#include docs/examples/flink/Main.java:source_test}} +``` +
+ +Example of batch query results: + +
+Batch Source +```java +{{#include docs/examples/flink/Main.java:source_batch_test}} +``` +
+ +Example of custom data type query result: + +
+Custom Type Source +```java +{{#include docs/examples/flink/Main.java:source_custom_type_test}} +``` +
+ +- ResultBean is a custom inner class used to define the data type of the Source query results. +- ResultSoureDeserialization is a custom inner class that inherits `TDengine` RecordDesrialization and implements convert and getProducedType methods. + +### CDC Data Subscription + +Flink CDC is mainly used to provide data subscription functionality, which can monitor real-time changes in TDengine database data and transmit these changes in the form of data streams to Flink for processing, while ensuring data consistency and integrity. + +Parameter Description + +- TDengineCdcParams.BOOTSTRAP_SERVERS: `ip:port` of the TDengine server, if using WebSocket connection, then it is the `ip:port` where taosAdapter is located. +- TDengineCdcParams.CONNECT_USER: Login to TDengine username, default value is 'root '. +- TDengineCdcParams.CONNECT_PASS: User login password, default value 'taosdata'. +- TDengineCdcParams.POLL_INTERVAL_MS: Pull data interval, default 500ms. +- TDengineCdcParams. VALUE_DESERIALIZER: Result set deserialization method, If the received result set type is `RowData` of `Flink`, simply set it to 'RowData'. You can inherit `com.taosdata.jdbc.tmq.ReferenceDeserializer`, specify the result set bean, and implement deserialization. You can also inherit `com.taosdata.jdbc.tmq.Deserializer` and customize the deserialization method based on the SQL resultSet. +- TDengineCdcParams.TMQ_BATCH_MODE: This parameter is used to batch push data to downstream operators. If set to True, when creating the `TDengineCdcSource` object, it is necessary to specify the data type as a template form of the `ConsumerRecords` type. +- TDengineCdcParams.GROUP_ID: Consumer group ID, the same consumer group shares consumption progress。Maximum length: 192. +- TDengineCdcParams.AUTO_OFFSET_RESET: Initial position of the consumer group subscription （ `earliest` subscribe from the beginning, `latest` subscribe from the latest data, default `latest`）。 +- TDengineCdcParams.ENABLE_AUTO_COMMIT: Whether to enable automatic consumption point submission，true: automatic submission；false：submit based on the `checkpoint` time, default to false. +> **Note**：The automatic submission mode of the reader automatically submits data after obtaining it, regardless of whether the downstream operator has processed the data correctly. There is a risk of data loss, and it is mainly used for efficient stateless operator scenarios or scenarios with low data consistency requirements. + +- TDengineCdcParams.AUTO_COMMIT_INTERVAL_MS: Time interval for automatically submitting consumption records, in milliseconds, default 5000. This parameter takes effect when `ENABLE_AUTO_COMMIT` is set to true. +- TDengineConfigParams.PROPERTY_KEY_ENABLE_COMPRESSION: Is compression enabled during the transmission process. true: Enable, false: Not enabled. The default is false. +- TDengineConfigParams.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: Whether to enable automatic reconnection. true: Enable, false: Not enabled. The default is false. +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_INTERVAL_MS: Automatic reconnection retry interval, in milliseconds, default value 2000. It only takes effect when `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` is true. +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_RETRY_COUNT: The default value for automatic reconnection retry is 3, which only takes effect when `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` is true. +- TDengineCdcParams.TMQ_SESSION_TIMEOUT_MS: Timeout after consumer heartbeat is lost, after which rebalance logic is triggered, and upon success, that consumer will be removed (supported from version 3.3.3.0)，Default is 12000, range [6000, 1800000]. +- TDengineCdcParams.TMQ_MAX_POLL_INTERVAL_MS: The longest time interval for consumer poll data fetching, exceeding this time will be considered as the consumer being offline, triggering rebalance logic, and upon success, that consumer will be removed (supported from version 3.3.3.0) Default is 300000, range [1000, INT32_MAX]. + +#### Use CDC connector + +The CDC connector will create consumers based on the parallelism set by the user, so the user should set the parallelism reasonably according to the resource situation. +The subscription result is RowData data type example: + +
+CDC Source +```java +{{#include docs/examples/flink/Main.java:cdc_source}} +``` +
+ +Example of batch query results: + +
+CDC Batch Source +```java +{{#include docs/examples/flink/Main.java:cdc_batch_source}} +``` +
+ +Example of custom data type query result: + +
+CDC Custom Type +```java +{{#include docs/examples/flink/Main.java:cdc_custom_type_test}} +``` +
+ +- ResultBean is a custom inner class whose field names and data types correspond one-to-one with column names and data types. This allows the deserialization class corresponding to the value.ddeserializer property to deserialize objects of ResultBean type. + +### Sink + +The core function of Sink is to efficiently and accurately write Flink processed data from different data sources or operators into TDengine. In this process, the efficient write mechanism possessed by TDengine played a crucial role, effectively ensuring the fast and stable storage of data. + +Sink Properties + +- TDengineConfigParams.PROPERTY_KEY_USER: Login to TDengine username, default value is 'root '. +- TDengineConfigParams.PROPERTY_KEY_PASSWORD: User login password, default value 'taosdata'. +- TDengineConfigParams.PROPERTY_KEY_DBNAME: The database name. +- TDengineConfigParams.TD_SUPERTABLE_NAME:The name of the super table. The received data must have a tbname field to determine which sub table to write to. +- TDengineConfigParams.TD_TABLE_NAME: The table name of a sub table or a normal table. This parameter only needs to be set together with `TD_SUPERTABLE_NAME`. +- TDengineConfigParams.VALUE_DESERIALIZER: The deserialization method for receiving result sets. If the type of the received result set is RowData of Flink, it only needs to be set to RowData. It is also possible to inherit 'TDengine SinkRecordSequencer' and implement the 'serialize' method, customizing the deserialization method based on the received data type. +- TDengineConfigParams.TD_BATCH_SIZE: Set the batch size for writing to the `TDengine` database once | Writing will be triggered when the number of batches is reached, or when a checkpoint is set. +- TDengineConfigParams.TD_BATCH_MODE: When set to True for receiving batch data, if the data source is `TDengine Source` , use the `SourceRecords Template` type to create a `TDengineSink` object; If the source is `TDengine CDC`, use the `ConsumerRecords Template` to create a `TDengineSink` object. +- TDengineConfigParams.TD_SOURCE_TYPE: Set the data source. When the data source is `TDengine Source`, it is set to 'tdengine_stource', and when the source is `TDengine CDC`, it is set to 'tdengine_cdc'. When the configuration of `TD_BATCH_MODE` is set to True, it takes effect. +- TDengineConfigParams.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: Message timeout time, in milliseconds, default value is 60000. +- TDengineConfigParams.PROPERTY_KEY_ENABLE_COMPRESSION: Is compression enabled during the transmission process. true: Enable, false: Not enabled. The default is false. +- TDengineConfigParams.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: Whether to enable automatic reconnection. true: Enable, false: Not enabled. The default is false. +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_INTERVAL_MS: Automatic reconnection retry interval, in milliseconds, default value 2000. It only takes effect when `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` is true. +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_RETRY_COUNT: The default value for automatic reconnection retry is 3, which only takes effect when `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` is true. +- TDengineConfigParams.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: Turn off SSL certificate verification. true: Enable, false: Not enabled. The default is false. + +Usage example: + +Write the sub table data of the meters table in the power database into the corresponding sub table of the sink_meters super table in the power_stink database. + +
+Sink RowData +```java +{{#include docs/examples/flink/Main.java:RowDataToSink}} +``` +
+ +Usage example: + +Subscribe to the sub table data of the meters super table in the power database and write it to the corresponding sub table of the sink_meters super table in the power_stink database. + +
+Cdc Sink +```java +{{#include docs/examples/flink/Main.java:CdcRowDataToSink}} +``` +
+ +### Table SQL + +Extract data from multiple different data source databases (such as TDengine, MySQL, Oracle, etc.) using Table SQL, perform custom operator operations (such as data cleaning, format conversion, associating data from different tables, etc.), and then load the processed results into the target data source (such as TDengine, MySQL, etc.). + +#### Source connector + +Parameter configuration instructions: + +| Parameter Name | Type | Parameter Description | +|-----------------------| :-----: | ------------ | +| connector | string | connector identifier, set `tdengine-connector`| +| td.jdbc.url | string | url of the connection | +| td.jdbc.mode | strng | connector type: `source`, `sink`| +| table.name | string | original or target table name | +| scan.query | string | SQL statement to retrieve data| +| sink.db.name | string | target database name| +| sink.supertable.name | string | name of the supertable| +| sink.batch.size | integer| batch size written| +| sink.table.name | string | the table name of a sub table or a normal table | + +Usage example: + +Write the sub table data of the meters table in the power database into the corresponding sub table of the sink_meters super table in the power_stink database. + +
+Table Source +```java +{{#include docs/examples/flink/Main.java:source_table}} +``` +
+ +#### Table CDC connector + +Parameter configuration instructions: + +| Parameter Name | Type | Parameter Description | +|-------------------| :-----: |--------------------------------------------------------------------------------------| +| connector | string | connector identifier, set `tdengine-connector` | +| user | string | username, default root | +| password | string | password, default taosdata | +| bootstrap. servers| string | server address | +| topic | string | subscribe to topic | +| td.jdbc.mode | strng | connector type: `cdc`, `sink` | +| group.id | string | consumption group ID, sharing consumption progress within the same consumption group | +| auto.offset.reset | string | initial position for consumer group subscription.
`earliest`: subscribe from the beginning
`latest` subscribe from the latest data
default `latest`| +| poll.interval_mas | integer | pull data interval, default 500ms | +| sink.db.name | string | target database name | +| sink.supertable.name | string | name of the supertable | +| sink.batch.size | integer | batch size written | +| sink.table.name | string | the table name of a sub table or a normal table | + +Usage example: + +Subscribe to the sub table data of the meters super table in the power database and write it to the corresponding sub table of the sink_meters super table in the power_stink database. + +
+Table CDC +```java +{{#include docs/examples/flink/Main.java:cdc_table}} +``` +
+ diff --git a/docs/en/14-reference/01-components/01-taosd.md b/docs/en/14-reference/01-components/01-taosd.md index 1b7f63510b..c86b631df4 100644 --- a/docs/en/14-reference/01-components/01-taosd.md +++ b/docs/en/14-reference/01-components/01-taosd.md @@ -190,7 +190,8 @@ The effective value of charset is UTF-8. |Parameter Name |Supported Version |Dynamic Modification|Description| |-----------------------|-------------------------|--------------------|------------| |supportVnodes | |Supported, effective immediately |Maximum number of vnodes supported by a dnode, range 0-4096, default value is twice the number of CPU cores + 5| -|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 0-1024, default value 4| +|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 1-1024, default value 4| +|numOfCompactThreads | |Supported, effective after restart|Maximum number of commit threads, range 1-16, default value 2| |numOfMnodeReadThreads | |Supported, effective after restart|Number of Read threads for mnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| |numOfVnodeQueryThreads | |Supported, effective after restart|Number of Query threads for vnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| |numOfVnodeFetchThreads | |Supported, effective after restart|Number of Fetch threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| diff --git a/docs/en/14-reference/05-connector/10-cpp.md b/docs/en/14-reference/05-connector/10-cpp.md index fe7574d416..940d4c359e 100644 --- a/docs/en/14-reference/05-connector/10-cpp.md +++ b/docs/en/14-reference/05-connector/10-cpp.md @@ -1,4 +1,5 @@ --- +toc_max_heading_level: 4 sidebar_label: C/C++ title: C/C++ Client Library slug: /tdengine-reference/client-libraries/cpp diff --git a/docs/en/14-reference/05-connector/14-java.md b/docs/en/14-reference/05-connector/14-java.md index 10e4ec6d42..c28702440a 100644 --- a/docs/en/14-reference/05-connector/14-java.md +++ b/docs/en/14-reference/05-connector/14-java.md @@ -1,4 +1,5 @@ --- +toc_max_heading_level: 4 sidebar_label: Java title: Java Client Library slug: /tdengine-reference/client-libraries/java @@ -30,33 +31,35 @@ The JDBC driver implementation for TDengine strives to be consistent with relati ## Version History -| taos-jdbcdriver Version | Major Changes | TDengine Version | -| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | -| 3.4.0 | 1. Replaced fastjson library with jackson.
2. WebSocket uses a separate protocol identifier.
3. Optimized background thread usage to avoid user misuse leading to timeouts. | - | -| 3.3.4 | Fixed getInt error when data type is float. | - | -| 3.3.3 | Fixed memory leak caused by closing WebSocket statement. | - | -| 3.3.2 | 1. Optimized parameter binding performance under WebSocket connection.
2. Improved support for mybatis. | - | -| 3.3.0 | 1. Optimized data transmission performance under WebSocket connection.
2. Supports skipping SSL verification, off by default. | 3.3.2.0 and higher | -| 3.2.11 | Fixed a bug in closing result set in Native connection. | - | -| 3.2.10 | 1. REST/WebSocket connections support data compression during transmission.
2. WebSocket automatic reconnection mechanism, off by default.
3. Connection class provides methods for schemaless writing.
4. Optimized data fetching performance for native connections.
5. Fixed some known issues.
6. Metadata retrieval functions can return a list of supported functions. | - | -| 3.2.9 | Fixed bug in closing WebSocket prepareStatement. | - | -| 3.2.8 | 1. Optimized auto-commit.
2. Fixed manual commit bug in WebSocket.
3. Optimized WebSocket prepareStatement using a single connection.
4. Metadata supports views. | - | -| 3.2.7 | 1. Supports VARBINARY and GEOMETRY types.
2. Added timezone setting support for native connections.
3. Added WebSocket automatic reconnection feature. | 3.2.0.0 and higher | -| 3.2.5 | Data subscription adds committed() and assignment() methods. | 3.1.0.3 and higher | -| 3.2.4 | Data subscription adds enable.auto.commit parameter under WebSocket connection, as well as unsubscribe() method. | - | -| 3.2.3 | Fixed ResultSet data parsing failure in some cases. | - | -| 3.2.2 | New feature: Data subscription supports seek function. | 3.0.5.0 and higher | -| 3.2.1 | 1. WebSocket connection supports schemaless and prepareStatement writing.
2. Consumer poll returns result set as ConsumerRecord, which can be accessed through value() method. | 3.0.3.0 and higher | -| 3.2.0 | Connection issues, not recommended for use. | - | -| 3.1.0 | WebSocket connection supports subscription function. | - | -| 3.0.1 - 3.0.4 | Fixed data parsing errors in result sets under some conditions. 3.0.1 compiled in JDK 11 environment, other versions recommended for JDK 8. | - | -| 3.0.0 | Supports TDengine 3.0 | 3.0.0.0 and higher | -| 2.0.42 | Fixed wasNull interface return value in WebSocket connection. | - | -| 2.0.41 | Fixed username and password encoding method in REST connection. | - | -| 2.0.39 - 2.0.40 | Added REST connection/request timeout settings. | - | -| 2.0.38 | JDBC REST connection adds batch fetching function. | - | -| 2.0.37 | Added support for json tag. | - | -| 2.0.36 | Added support for schemaless writing. | - | +| taos-jdbcdriver Version | Major Changes | TDengine Version | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | +| 3.5.1 | Fixed the getObject issue in data subscription. | - | +| 3.5.0 | 1. Optimized the performance of WebSocket connection parameter binding, supporting parameter binding queries using binary data.
2. Optimized the performance of small queries in WebSocket connection.
3. Added support for setting time zone and app info on WebSocket connection. | 3.3.5.0 and higher | +| 3.4.0 | 1. Replaced fastjson library with jackson.
2. WebSocket uses a separate protocol identifier.
3. Optimized background thread usage to avoid user misuse leading to timeouts. | - | +| 3.3.4 | Fixed getInt error when data type is float. | - | +| 3.3.3 | Fixed memory leak caused by closing WebSocket statement. | - | +| 3.3.2 | 1. Optimized parameter binding performance under WebSocket connection.
2. Improved support for mybatis. | - | +| 3.3.0 | 1. Optimized data transmission performance under WebSocket connection.
2. Supports skipping SSL verification, off by default. | 3.3.2.0 and higher | +| 3.2.11 | Fixed a bug in closing result set in Native connection. | - | +| 3.2.10 | 1. REST/WebSocket connections support data compression during transmission.
2. WebSocket automatic reconnection mechanism, off by default.
3. Connection class provides methods for schemaless writing.
4. Optimized data fetching performance for native connections.
5. Fixed some known issues.
6. Metadata retrieval functions can return a list of supported functions. | - | +| 3.2.9 | Fixed bug in closing WebSocket prepareStatement. | - | +| 3.2.8 | 1. Optimized auto-commit.
2. Fixed manual commit bug in WebSocket.
3. Optimized WebSocket prepareStatement using a single connection.
4. Metadata supports views. | - | +| 3.2.7 | 1. Supports VARBINARY and GEOMETRY types.
2. Added timezone setting support for native connections.
3. Added WebSocket automatic reconnection feature. | 3.2.0.0 and higher | +| 3.2.5 | Data subscription adds committed() and assignment() methods. | 3.1.0.3 and higher | +| 3.2.4 | Data subscription adds enable.auto.commit parameter under WebSocket connection, as well as unsubscribe() method. | - | +| 3.2.3 | Fixed ResultSet data parsing failure in some cases. | - | +| 3.2.2 | New feature: Data subscription supports seek function. | 3.0.5.0 and higher | +| 3.2.1 | 1. WebSocket connection supports schemaless and prepareStatement writing.
2. Consumer poll returns result set as ConsumerRecord, which can be accessed through value() method. | 3.0.3.0 and higher | +| 3.2.0 | Connection issues, not recommended for use. | - | +| 3.1.0 | WebSocket connection supports subscription function. | - | +| 3.0.1 - 3.0.4 | Fixed data parsing errors in result sets under some conditions. 3.0.1 compiled in JDK 11 environment, other versions recommended for JDK 8. | - | +| 3.0.0 | Supports TDengine 3.0 | 3.0.0.0 and higher | +| 2.0.42 | Fixed wasNull interface return value in WebSocket connection. | - | +| 2.0.41 | Fixed username and password encoding method in REST connection. | - | +| 2.0.39 - 2.0.40 | Added REST connection/request timeout settings. | - | +| 2.0.38 | JDBC REST connection adds batch fetching function. | - | +| 2.0.37 | Added support for json tag. | - | +| 2.0.36 | Added support for schemaless writing. | - | ## Exceptions and Error Codes @@ -75,47 +78,47 @@ The error codes that the JDBC connector may report include 4 types: Please refer to the specific error codes: -| Error Code | Description | Suggested Actions | -| ---------- | --------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | -| 0x2301 | connection already closed | The connection is already closed, check the connection status, or recreate the connection to execute related commands. | -| 0x2302 | this operation is NOT supported currently! | The current interface is not supported, consider switching to another connection method. | -| 0x2303 | invalid variables | Invalid parameters, please check the interface specifications and adjust the parameter types and sizes. | -| 0x2304 | statement is closed | The statement is already closed, check if the statement was used after being closed, or if the connection is normal. | -| 0x2305 | resultSet is closed | The resultSet has been released, check if the resultSet was used after being released. | -| 0x2306 | Batch is empty! | Add parameters to prepareStatement before executing executeBatch. | -| 0x2307 | Can not issue data manipulation statements with executeQuery() | Use executeUpdate() for update operations, not executeQuery(). | -| 0x2308 | Can not issue SELECT via executeUpdate() | Use executeQuery() for query operations, not executeUpdate(). | -| 0x230d | parameter index out of range | Parameter out of bounds, check the reasonable range of parameters. | -| 0x230e | connection already closed | The connection is already closed, check if the Connection was used after being closed, or if the connection is normal. | -| 0x230f | unknown sql type in tdengine | Check the Data Type types supported by TDengine. | -| 0x2310 | can't register JDBC-JNI driver | Cannot register JNI driver, check if the url is correctly filled. | -| 0x2312 | url is not set | Check if the REST connection url is correctly filled. | -| 0x2314 | numeric value out of range | Check if the correct interface was used for numeric types in the result set. | -| 0x2315 | unknown taos type in tdengine | When converting TDengine data types to JDBC data types, check if the correct TDengine data type was specified. | -| 0x2317 | | Incorrect request type used in REST connection. | -| 0x2318 | | Data transmission error occurred in REST connection, check the network situation and retry. | -| 0x2319 | user is required | Username information is missing when creating a connection. | -| 0x231a | password is required | Password information is missing when creating a connection. | -| 0x231c | httpEntity is null, sql: | An exception occurred in REST connection execution. | -| 0x231d | can't create connection with server within | Increase the httpConnectTimeout parameter to extend the connection time, or check the connection with taosAdapter. | -| 0x231e | failed to complete the task within the specified time | Increase the messageWaitTimeout parameter to extend the execution time, or check the connection with taosAdapter. | -| 0x2350 | unknown error | Unknown exception, please provide feedback to the developers on github. | -| 0x2352 | Unsupported encoding | An unsupported character encoding set was specified in the local connection. | -| 0x2353 | internal error of database, please see taoslog for more details | An error occurred while executing prepareStatement in local connection, check taos log for troubleshooting. | -| 0x2354 | JNI connection is NULL | The Connection was already closed when executing commands in local connection. Check the connection with TDengine. | -| 0x2355 | JNI result set is NULL | The result set is abnormal in local connection, check the connection and retry. | -| 0x2356 | invalid num of fields | The meta information of the result set obtained in local connection does not match. | -| 0x2357 | empty sql string | Fill in the correct SQL for execution. | -| 0x2359 | JNI alloc memory failed, please see taoslog for more details | Memory allocation error in local connection, check taos log for troubleshooting. | -| 0x2371 | consumer properties must not be null! | Parameters are null when creating a subscription, fill in the correct parameters. | -| 0x2372 | configs contain empty key, failed to set consumer property | The parameter key contains empty values, fill in the correct parameters. | -| 0x2373 | failed to set consumer property, | The parameter value contains empty values, fill in the correct parameters. | -| 0x2375 | topic reference has been destroyed | During the data subscription process, the topic reference was released. Check the connection with TDengine. | +| Error Code | Description | Suggested Actions | +| ---------- | --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| 0x2301 | connection already closed | The connection is already closed, check the connection status, or recreate the connection to execute related commands. | +| 0x2302 | this operation is NOT supported currently! | The current interface is not supported, consider switching to another connection method. | +| 0x2303 | invalid variables | Invalid parameters, please check the interface specifications and adjust the parameter types and sizes. | +| 0x2304 | statement is closed | The statement is already closed, check if the statement was used after being closed, or if the connection is normal. | +| 0x2305 | resultSet is closed | The resultSet has been released, check if the resultSet was used after being released. | +| 0x2306 | Batch is empty! | Add parameters to prepareStatement before executing executeBatch. | +| 0x2307 | Can not issue data manipulation statements with executeQuery() | Use executeUpdate() for update operations, not executeQuery(). | +| 0x2308 | Can not issue SELECT via executeUpdate() | Use executeQuery() for query operations, not executeUpdate(). | +| 0x230d | parameter index out of range | Parameter out of bounds, check the reasonable range of parameters. | +| 0x230e | connection already closed | The connection is already closed, check if the Connection was used after being closed, or if the connection is normal. | +| 0x230f | unknown sql type in tdengine | Check the Data Type types supported by TDengine. | +| 0x2310 | can't register JDBC-JNI driver | Cannot register JNI driver, check if the url is correctly filled. | +| 0x2312 | url is not set | Check if the REST connection url is correctly filled. | +| 0x2314 | numeric value out of range | Check if the correct interface was used for numeric types in the result set. | +| 0x2315 | unknown taos type in tdengine | When converting TDengine data types to JDBC data types, check if the correct TDengine data type was specified. | +| 0x2317 | | Incorrect request type used in REST connection. | +| 0x2318 | | Data transmission error occurred in REST connection, check the network situation and retry. | +| 0x2319 | user is required | Username information is missing when creating a connection. | +| 0x231a | password is required | Password information is missing when creating a connection. | +| 0x231c | httpEntity is null, sql: | An exception occurred in REST connection execution. | +| 0x231d | can't create connection with server within | Increase the httpConnectTimeout parameter to extend the connection time, or check the connection with taosAdapter. | +| 0x231e | failed to complete the task within the specified time | Increase the messageWaitTimeout parameter to extend the execution time, or check the connection with taosAdapter. | +| 0x2350 | unknown error | Unknown exception, please provide feedback to the developers on github. | +| 0x2352 | Unsupported encoding | An unsupported character encoding set was specified in the local connection. | +| 0x2353 | internal error of database, please see taoslog for more details | An error occurred while executing prepareStatement in local connection, check taos log for troubleshooting. | +| 0x2354 | JNI connection is NULL | The Connection was already closed when executing commands in local connection. Check the connection with TDengine. | +| 0x2355 | JNI result set is NULL | The result set is abnormal in local connection, check the connection and retry. | +| 0x2356 | invalid num of fields | The meta information of the result set obtained in local connection does not match. | +| 0x2357 | empty sql string | Fill in the correct SQL for execution. | +| 0x2359 | JNI alloc memory failed, please see taoslog for more details | Memory allocation error in local connection, check taos log for troubleshooting. | +| 0x2371 | consumer properties must not be null! | Parameters are null when creating a subscription, fill in the correct parameters. | +| 0x2372 | configs contain empty key, failed to set consumer property | The parameter key contains empty values, fill in the correct parameters. | +| 0x2373 | failed to set consumer property, | The parameter value contains empty values, fill in the correct parameters. | +| 0x2375 | topic reference has been destroyed | During the data subscription process, the topic reference was released. Check the connection with TDengine. | | 0x2376 | failed to set consumer topic, topic name is empty | During the data subscription process, the subscription topic name is empty. Check if the specified topic name is correctly filled. | -| 0x2377 | consumer reference has been destroyed | The data transmission channel for the subscription has been closed, check the connection with TDengine. | -| 0x2378 | consumer create error | Data subscription creation failed, check the error information and taos log for troubleshooting. | -| 0x2379 | seek offset must not be a negative number | The seek interface parameter must not be negative, use the correct parameters. | -| 0x237a | vGroup not found in result set | VGroup not assigned to the current consumer, due to the Rebalance mechanism causing the Consumer and VGroup to be unbound. | +| 0x2377 | consumer reference has been destroyed | The data transmission channel for the subscription has been closed, check the connection with TDengine. | +| 0x2378 | consumer create error | Data subscription creation failed, check the error information and taos log for troubleshooting. | +| 0x2379 | seek offset must not be a negative number | The seek interface parameter must not be negative, use the correct parameters. | +| 0x237a | vGroup not found in result set | VGroup not assigned to the current consumer, due to the Rebalance mechanism causing the Consumer and VGroup to be unbound. | - [TDengine Java Connector Error Code](https://github.com/taosdata/taos-connector-jdbc/blob/main/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java) @@ -244,13 +247,13 @@ For WebSocket connections, the configuration parameters in the URL are as follow - user: Login username for TDengine, default value 'root'. - password: User login password, default value 'taosdata'. -- charset: Specifies the character set for parsing string data when batch fetching is enabled. - batchErrorIgnore: true: Continues executing the following SQL if one SQL fails during the execution of Statement's executeBatch. false: Does not execute any statements after a failed SQL. Default value: false. - httpConnectTimeout: Connection timeout in ms, default value 60000. - messageWaitTimeout: Message timeout in ms, default value 60000. - useSSL: Whether SSL is used in the connection. +- timezone: Client timezone, default is the system current timezone. Recommended not to set, using the system time zone provides better performance. -**Note**: Some configuration items (such as: locale, timezone) do not take effect in WebSocket connections. +**Note**: Some configuration items (such as: locale, charset) do not take effect in WebSocket connections. **REST Connection** Using JDBC REST connection does not depend on the client driver. Compared to native JDBC connections, you only need to: @@ -263,14 +266,13 @@ For REST connections, the configuration parameters in the URL are as follows: - user: Login username for TDengine, default value 'root'. - password: User login password, default value 'taosdata'. -- charset: Specifies the character set for parsing string data when batch fetching is enabled. - batchErrorIgnore: true: Continues executing the following SQL if one SQL fails during the execution of Statement's executeBatch. false: Does not execute any statements after a failed SQL. Default value: false. - httpConnectTimeout: Connection timeout in ms, default value 60000. - httpSocketTimeout: Socket timeout in ms, default value 60000. - useSSL: Whether SSL is used in the connection. - httpPoolSize: REST concurrent request size, default 20. -**Note**: Some configuration items (such as: locale, timezone) do not take effect in REST connections. +**Note**: Some configuration items (such as: locale, charset and timezone) do not take effect in REST connections. :::note @@ -294,7 +296,9 @@ The configuration parameters in properties are as follows: - TSDBDriver.PROPERTY_KEY_CONFIG_DIR: Effective only when using native JDBC connections. Client configuration file directory path, default value on Linux OS is `/etc/taos`, on Windows OS is `C:/TDengine/cfg`. - TSDBDriver.PROPERTY_KEY_CHARSET: Character set used by the client, default value is the system character set. - TSDBDriver.PROPERTY_KEY_LOCALE: Effective only when using native JDBC connections. Client locale, default value is the current system locale. -- TSDBDriver.PROPERTY_KEY_TIME_ZONE: Effective only when using native JDBC connections. Client time zone, default value is the current system time zone. Due to historical reasons, we only support part of the POSIX standard, such as UTC-8 (representing Shanghai, China), GMT-8, Asia/Shanghai. +- TSDBDriver.PROPERTY_KEY_TIME_ZONE: + - Native connections: Client time zone, default value is the current system time zone. Effective globally. Due to historical reasons, we only support part of the POSIX standard, such as UTC-8 (representing Shanghai, China), GMT-8, Asia/Shanghai. + - WebSocket connections. Client time zone, default value is the current system time zone. Effective on the connection. Only IANA time zones are supported, such as Asia/Shanghai. It is recommended not to set this parameter, as using the system time zone provides better performance. - TSDBDriver.HTTP_CONNECT_TIMEOUT: Connection timeout, in ms, default value is 60000. Effective only in REST connections. - TSDBDriver.HTTP_SOCKET_TIMEOUT: Socket timeout, in ms, default value is 60000. Effective only in REST connections and when batchfetch is set to false. - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: Message timeout, in ms, default value is 60000. Effective only under WebSocket connections. @@ -303,12 +307,14 @@ The configuration parameters in properties are as follows: - TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: Whether to enable compression during transmission. Effective only when using REST/WebSocket connections. true: enabled, false: not enabled. Default is false. - TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: Whether to enable auto-reconnect. Effective only when using WebSocket connections. true: enabled, false: not enabled. Default is false. -> **Note**: Enabling auto-reconnect is only effective for simple SQL execution, schema-less writing, and data subscription. It is ineffective for parameter binding. Auto-reconnect is only effective for connections established through parameters specifying the database, and ineffective for later `use db` statements to switch databases. + > **Note**: Enabling auto-reconnect is only effective for simple SQL execution, schema-less writing, and data subscription. It is ineffective for parameter binding. Auto-reconnect is only effective for connections established through parameters specifying the database, and ineffective for later `use db` statements to switch databases. - TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS: Auto-reconnect retry interval, in milliseconds, default value 2000. Effective only when PROPERTY_KEY_ENABLE_AUTO_RECONNECT is true. - TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT: Auto-reconnect retry count, default value 3, effective only when PROPERTY_KEY_ENABLE_AUTO_RECONNECT is true. - TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: Disable SSL certificate validation. Effective only when using WebSocket connections. true: enabled, false: not enabled. Default is false. - +- TSDBDriver.PROPERTY_KEY_APP_NAME: App name, can be used for display in the `show connections` query result. Effective only when using WebSocket connections. Default value is java. +- TSDBDriver.PROPERTY_KEY_APP_IP: App IP, can be used for display in the `show connections` query result. Effective only when using WebSocket connections. Default value is empty. + Additionally, for native JDBC connections, other parameters such as log level and SQL length can be specified by specifying the URL and Properties. **Priority of Configuration Parameters** @@ -489,16 +495,16 @@ For example: if the password is specified as taosdata in the URL and as taosdemo List of interface methods that return `true` for supported features, others not explicitly mentioned return `false`. -| Interface Method | Description | -|--------------------------------------------------------|-----------------------------------------------------| -| `boolean nullsAreSortedAtStart()` | Determines if `NULL` values are sorted at the start | -| `boolean storesLowerCaseIdentifiers()` | Determines if the database stores identifiers in lowercase | -| `boolean supportsAlterTableWithAddColumn()` | Determines if the database supports adding columns with `ALTER TABLE` | -| `boolean supportsAlterTableWithDropColumn()` | Determines if the database supports dropping columns with `ALTER TABLE` | -| `boolean supportsColumnAliasing()` | Determines if the database supports column aliasing | -| `boolean supportsGroupBy()` | Determines if the database supports `GROUP BY` statements | -| `boolean isCatalogAtStart()` | Determines if the catalog name appears at the start of the fully qualified name in the database | -| `boolean supportsCatalogsInDataManipulation()` | Determines if the database supports catalog names in data manipulation statements | +| Interface Method | Description | +| ---------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| `boolean nullsAreSortedAtStart()` | Determines if `NULL` values are sorted at the start | +| `boolean storesLowerCaseIdentifiers()` | Determines if the database stores identifiers in lowercase | +| `boolean supportsAlterTableWithAddColumn()` | Determines if the database supports adding columns with `ALTER TABLE` | +| `boolean supportsAlterTableWithDropColumn()` | Determines if the database supports dropping columns with `ALTER TABLE` | +| `boolean supportsColumnAliasing()` | Determines if the database supports column aliasing | +| `boolean supportsGroupBy()` | Determines if the database supports `GROUP BY` statements | +| `boolean isCatalogAtStart()` | Determines if the catalog name appears at the start of the fully qualified name in the database | +| `boolean supportsCatalogsInDataManipulation()` | Determines if the database supports catalog names in data manipulation statements | ### Connection Features diff --git a/docs/en/14-reference/05-connector/20-go.md b/docs/en/14-reference/05-connector/20-go.md index dd32df2c5b..578150f0fa 100644 --- a/docs/en/14-reference/05-connector/20-go.md +++ b/docs/en/14-reference/05-connector/20-go.md @@ -1,4 +1,5 @@ --- +toc_max_heading_level: 4 sidebar_label: Go title: Go Client Library slug: /tdengine-reference/client-libraries/go @@ -21,24 +22,25 @@ Supports Go 1.14 and above. ## Version History -| driver-go Version | Major Changes | TDengine Version | -|------------------|------------------------------------------------------------------|-------------------| -| v3.5.8 | Fixed null pointer exception. | - | -| v3.5.7 | taosWS and taosRestful support passing request id. | - | -| v3.5.6 | Improved websocket query and insert performance. | 3.3.2.0 and higher | -| v3.5.5 | Restful supports skipping SSL certificate check. | - | -| v3.5.4 | Compatible with TDengine 3.3.0.0 tmq raw data. | - | -| v3.5.3 | Refactored taosWS. | - | -| v3.5.2 | Websocket compression and optimized tmq subscription performance. | 3.2.3.0 and higher | -| v3.5.1 | Native stmt query and geometry type support. | 3.2.1.0 and higher | -| v3.5.0 | Support tmq get assignment and seek offset. | 3.0.5.0 and higher | -| v3.3.1 | Schemaless protocol insert based on websocket. | 3.0.4.1 and higher | -| v3.1.0 | Provided Kafka-like subscription API. | - | -| v3.0.4 | Added request id related interfaces. | 3.0.2.2 and higher | -| v3.0.3 | Websocket-based statement insert. | - | -| v3.0.2 | Websocket-based data query and insert. | 3.0.1.5 and higher | -| v3.0.1 | Websocket-based message subscription. | - | -| v3.0.0 | Adapted to TDengine 3.0 query and insert. | 3.0.0.0 and higher | +| driver-go Version | Major Changes | TDengine Version | +|-------------------|-------------------------------------------------------------------------------------------------|--------------------| +| v3.6.0 | stmt2 native interface, DSN supports passwords containing special characters (url.QueryEscape). | 3.3.5.0 and higher | +| v3.5.8 | Fixed null pointer exception. | - | +| v3.5.7 | taosWS and taosRestful support passing request id. | - | +| v3.5.6 | Improved websocket query and insert performance. | 3.3.2.0 and higher | +| v3.5.5 | Restful supports skipping SSL certificate check. | - | +| v3.5.4 | Compatible with TDengine 3.3.0.0 tmq raw data. | - | +| v3.5.3 | Refactored taosWS. | - | +| v3.5.2 | Websocket compression and optimized tmq subscription performance. | 3.2.3.0 and higher | +| v3.5.1 | Native stmt query and geometry type support. | 3.2.1.0 and higher | +| v3.5.0 | Support tmq get assignment and seek offset. | 3.0.5.0 and higher | +| v3.3.1 | Schemaless protocol insert based on websocket. | 3.0.4.1 and higher | +| v3.1.0 | Provided Kafka-like subscription API. | - | +| v3.0.4 | Added request id related interfaces. | 3.0.2.2 and higher | +| v3.0.3 | Websocket-based statement insert. | - | +| v3.0.2 | Websocket-based data query and insert. | 3.0.1.5 and higher | +| v3.0.1 | Websocket-based message subscription. | - | +| v3.0.0 | Adapted to TDengine 3.0 query and insert. | 3.0.0.0 and higher | ## Exceptions and Error Codes @@ -136,6 +138,8 @@ Full form of DSN: username:password@protocol(address)/dbname?param=value ``` +When the password contains special characters, it needs to be escaped using url.QueryEscape. + ##### Native Connection Import the driver: @@ -493,6 +497,43 @@ The `af` package provides more interfaces using native connections for parameter * **Interface Description**: Closes the statement. * **Return Value**: Error information. +From version 3.6.0, the `stmt2` interface for binding parameters is provided. + +* `func (conn *Connector) Stmt2(reqID int64, singleTableBindOnce bool) *Stmt2` + * **Interface Description**: Returns a Stmt2 object bound to this connection. + * **Parameter Description**: + * `reqID`: Request ID. + * `singleTableBindOnce`: Indicates whether a single child table is bound only once during a single execution. + * **Return Value**: Stmt2 object. + +* `func (s *Stmt2) Prepare(sql string) error` + * **Interface Description**: Prepares an SQL. + * **Parameter Description**: + * `sql`: The statement for parameter binding. + * **Return Value**: Error information. + +* `func (s *Stmt2) Bind(params []*stmt.TaosStmt2BindData) error` + * **Interface Description**: Binds data to the prepared statement. + * **Parameter Description**: + * `params`: The data to bind. + * **Return Value**: Error information. + +* `func (s *Stmt2) Execute() error` + * **Interface Description**: Executes the batch. + * **Return Value**: Error information. + +* `func (s *Stmt2) GetAffectedRows() int` + * **Interface Description**: Gets the number of affected rows (only valid for insert statements). + * **Return Value**: Number of affected rows. + +* `func (s *Stmt2) UseResult() (driver.Rows, error)` + * **Interface Description**: Retrieves the result set (only valid for query statements). + * **Return Value**: Result set Rows object, error information. + +* `func (s *Stmt2) Close() error` + * **Interface Description**: Closes the statement. + * **Return Value**: Error information. + The `ws/stmt` package provides interfaces for parameter binding via WebSocket * `func (c *Connector) Init() (*Stmt, error)` diff --git a/docs/en/14-reference/05-connector/26-rust.md b/docs/en/14-reference/05-connector/26-rust.md index 637d009b8c..8de5d628de 100644 --- a/docs/en/14-reference/05-connector/26-rust.md +++ b/docs/en/14-reference/05-connector/26-rust.md @@ -1,4 +1,5 @@ --- +toc_max_heading_level: 4 sidebar_label: Rust title: Rust Client Library slug: /tdengine-reference/client-libraries/rust diff --git a/docs/en/14-reference/05-connector/30-python.md b/docs/en/14-reference/05-connector/30-python.md index 58cff75c90..19247e5364 100644 --- a/docs/en/14-reference/05-connector/30-python.md +++ b/docs/en/14-reference/05-connector/30-python.md @@ -1,4 +1,5 @@ --- +toc_max_heading_level: 4 sidebar_label: Python title: Python Client Library slug: /tdengine-reference/client-libraries/python @@ -55,6 +56,8 @@ Python Connector historical versions (it is recommended to use the latest versio |Python Connector Version | Major Changes | TDengine Version| | -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | +|2.7.21 | Native supports STMT2 writing | - | +|2.7.19 | Support Apache Superset connection to TDengine Cloud data source | - | |2.7.18 | Support Apache SuperSet BI Tools. | - | |2.7.16 | Add subscription configuration (session. timeout. ms, Max. roll. interval. ms). | - | |2.7.15 | Added support for VARBINRY and GEOMETRY types. | - | @@ -136,7 +139,7 @@ TDengine currently supports timestamp, numeric, character, boolean types, and th | [tmq_consumer.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq_consumer.py) | tmq subscription | | [native_all_type_query.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/native_all_type_query.py) | Example supporting all types | | [native_all_type_stmt.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/native_all_type_stmt.py) | Parameter binding example supporting all types | - +| [test_stmt2.py](https://github.com/taosdata/taos-connector-python/blob/main/tests/test_stmt2.py) | Example of STMT2 writing | Example program source code can be found at: 1. [More native example programs](https://github.com/taosdata/taos-connector-python/tree/main/examples) @@ -429,51 +432,40 @@ TaosResult object can be iterated over to retrieve queried data. - **Exceptions**: Throws `SchemalessError` if operation fails. #### Parameter Binding - -- `def statement(self, sql=None)` - - **Interface Description**: Creates a stmt object using the connection object, if sql is not empty it will call prepare. - - `sql`: Precompiled SQL statement. - - **Return Value**: stmt object. - - **Exception**: Throws `StatementError` exception on failure. +- `def statement2(self, sql=None, option=None)` + - **Interface Description**：Creating an STMT2 object using a connection object + - **Parameter Description** + - `sql`: The bound SQL statement will call the `prepare` function if it is not empty + - `option` Pass in `TaoStmt2Option` class instance + - **Return Value**：STMT2 object + - **Exception**：Throws `ConnectionError` on failure - `def prepare(self, sql)` - - **Interface Description**: Binds a precompiled sql statement. - - **Parameter Description**: - - `sql`: Precompiled SQL statement. - - **Exception**: Throws `StatementError` exception on failure. -- `def set_tbname(self, name)` - - **Interface Description**: Sets the table name for data to be written to. - - **Parameter Description**: - - `name`: Table name, if you need to specify a database, for example: `db_name.table_name`. - - **Exception**: Throws `StatementError` exception on failure. -- `def set_tbname_tags(self, name, tags):` - - **Interface Description**: Sets the table and Tags data, used for automatic table creation. - - **Parameter Description**: - - `name`: Table name, if you need to specify a database, for example: `db_name.table_name`. - - `tags`: Tags data. - - **Exception**: Throws `StatementError` exception on failure. -- `def bind_param(self, params, add_batch=True)` - - **Interface Description**: Binds a set of data and submits. - - **Parameter Description**: - - `params`: Data to bind. - - `add_batch`: Whether to submit the bound data. - - **Exception**: Throws `StatementError` exception on failure. -- `def bind_param_batch(self, binds, add_batch=True)` - - **Interface Description**: Binds multiple sets of data and submits. - - **Parameter Description**: - - `binds`: Data to bind. - - `add_batch`: Whether to submit the bound data. - - **Exception**: Throws `StatementError` exception on failure. -- `def add_batch(self)` - - **Interface Description**: Submits the bound data. - - **Exception**: Throws `StatementError` exception on failure. -- `def execute(self)` - - **Interface Description**: Executes and writes all the bound data. - - **Exception**: Throws `StatementError` exception on failure. -- `def affected_rows(self)` - - **Interface Description**: Gets the number of rows written. - - **Return Value**: Number of rows written. -- `def close(&self)` - - **Interface Description**: Closes the stmt object. + - **Interface Description**：Bind a precompiled SQL statement + - **Parameter Description**： + - `sql`: Precompiled SQL statement + - **Exception**：Throws `StatementError` on failure +- `def bind_param(self, tbnames, tags, datas)` + - **Interface Description**：Binding Data as an Independent Array + - **Parameter Description**： + - `tbnames`:Bind table name array, data type is list + - `tags`: Bind tag column value array, data type is list + - `datas`: Bind data column value array, data type of list + - **Exception**：Throws `StatementError` on failure +- `def bind_param_with_tables(self, tables)` + - **Interface Description**：Bind data in an independent table format. Independent tables are organized by table units, with table name, TAG value, and data column attributes in table object + - **Parameter Description**： + - `tables`: `BindTable` Independent table object array + - **Exception**：Throws `StatementError` on failure +- `def execute(self) -> int:` + - **Interface Description**：Execute to write all bound data + - **Return Value**：Affects the number of rows + - **Exception**：Throws `QueryError` on failure +- `def result(self)` + - **Interface Description**：Get parameter binding query result set + - **Return Value**：Returns the TaosResult object +- `def close(self)` + - **Interface Description**： close the STMT2 object + #### Data Subscription diff --git a/docs/en/14-reference/05-connector/35-node.md b/docs/en/14-reference/05-connector/35-node.md index 19dae0357f..49b1d200cf 100644 --- a/docs/en/14-reference/05-connector/35-node.md +++ b/docs/en/14-reference/05-connector/35-node.md @@ -1,4 +1,5 @@ --- +toc_max_heading_level: 4 sidebar_label: Node.js title: Node.js Client Library slug: /tdengine-reference/client-libraries/node diff --git a/docs/en/14-reference/05-connector/40-csharp.md b/docs/en/14-reference/05-connector/40-csharp.md index 8e51cb319b..c9c9f95228 100644 --- a/docs/en/14-reference/05-connector/40-csharp.md +++ b/docs/en/14-reference/05-connector/40-csharp.md @@ -1,4 +1,5 @@ --- +toc_max_heading_level: 4 sidebar_label: C# title: C# Client Library slug: /tdengine-reference/client-libraries/csharp diff --git a/docs/en/27-train-faq/index.md b/docs/en/27-train-faq/index.md index ca6cd91714..aa85729102 100644 --- a/docs/en/27-train-faq/index.md +++ b/docs/en/27-train-faq/index.md @@ -297,3 +297,22 @@ Reporting this error indicates that the first connection to the cluster was succ Therefore, first, check whether all ports on the server and cluster (default 6030 for native connections and 6041 for HTTP connections) are open; Next, check if the client's hosts file has configured the fqdn and IP information for all dnodes in the cluster. If the issue still cannot be resolved, it is necessary to contact Taos technical personnel for support. + +### 32 Why is the original database lost and the cluster ID changed when the data directory dataDir of the database remains unchanged on the same server? +Background: When the TDengine server process (taosd) starts, if there are no valid data file subdirectories (such as mnode, dnode, and vnode) under the data directory (dataDir, which is specified in the configuration file taos.cfg), these directories will be created automatically.When a new mnode directory is created, a new cluster ID will be allocated to generate a new cluster. + +Cause analysis: The data directory dataDir of taosd can point to multiple different mount points.If these mount points are not configured for automatic mounting in the fstab file, after the server restarts, dataDir will only exist as a normal directory of the local disk, and it will not point to the mounted disk as expected.At this point, if the taosd service is started, it will create a new directory under dataDir to generate a new cluster. + +Impact of the problem: After the server is restarted, the original database is lost (note: it is not really lost, but the original data disk is not attached and cannot be seen for the time being) and the cluster ID changes, resulting in the inability to access the original database. For enterprise users, if they have been authorized for the cluster ID, they will also find that the machine code of the cluster server has not changed, but the original authorization has expired.If the problem is not monitored or found and handled in time, the user will not notice that the original database has been lost, resulting in losses and increased operation and maintenance costs. + +Problem solving: You should configure the automatic mount of the dataDir directory in the fstab file to ensure that the dataDir always points to the expected mount point and directory. At this point, restarting the server will retrieve the original database and cluster. In the subsequent version, we will develop a function to enable taosd to exit in the startup phase when it detects that the dataDir changes before and after startup, and provide corresponding error prompts. + +### 33 How to solve MVCP1400.DLL loss when running TDengine on Windows platform? +1. Reinstall Microsoft Visual C++ Redistributable: As msvcp140.dll is part of Microsoft Visual C++Redistributable, reinstalling this package usually resolves most issues. You can download the corresponding version from the official Microsoft website for installation +2. Manually download and replace the msvcp140.dll file online: You can download the msvcp140.dll file from a reliable source and copy it to the corresponding directory in the system. Ensure that the downloaded files match your system architecture (32-bit or 64 bit) and ensure the security of the source + +### 34 Which fast query data from super table with TAG filter or child table ? +Directly querying from child table is fast. The query from super table with TAG filter is designed to meet the convenience of querying. It can filter data from multiple child tables at the same time. If the goal is to pursue performance and the child table has been clearly queried, directly querying from the sub table can achieve higher performance + +### 35 How to view data compression ratio indicators? +Currently, TDengine only provides compression ratios based on tables, not databases or the entire system. To view the compression ratios, execute the `SHOW TABLE DISTRIBUTED table_name;` command in the client taos-CLI. The table_name can be a super table, regular table, or subtable. For details [Click Here](https://docs.tdengine.com/tdengine-reference/sql-manual/show-commands/#show-table-distributed) \ No newline at end of file diff --git a/docs/examples/JDBC/JDBCDemo/pom.xml b/docs/examples/JDBC/JDBCDemo/pom.xml index 4b3e1ab675..78262712e9 100644 --- a/docs/examples/JDBC/JDBCDemo/pom.xml +++ b/docs/examples/JDBC/JDBCDemo/pom.xml @@ -19,7 +19,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 org.locationtech.jts diff --git a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml index 34719dc135..7ff4a72f5e 100644 --- a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 diff --git a/docs/examples/JDBC/connectionPools/pom.xml b/docs/examples/JDBC/connectionPools/pom.xml index e3ef30d2f8..70be6ed527 100644 --- a/docs/examples/JDBC/connectionPools/pom.xml +++ b/docs/examples/JDBC/connectionPools/pom.xml @@ -18,7 +18,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 diff --git a/docs/examples/JDBC/consumer-demo/pom.xml b/docs/examples/JDBC/consumer-demo/pom.xml index 709f87d9c1..c9537a93bf 100644 --- a/docs/examples/JDBC/consumer-demo/pom.xml +++ b/docs/examples/JDBC/consumer-demo/pom.xml @@ -17,7 +17,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 com.google.guava diff --git a/docs/examples/JDBC/mybatisplus-demo/pom.xml b/docs/examples/JDBC/mybatisplus-demo/pom.xml index 2077e31d8d..effb13cfe8 100644 --- a/docs/examples/JDBC/mybatisplus-demo/pom.xml +++ b/docs/examples/JDBC/mybatisplus-demo/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 diff --git a/docs/examples/JDBC/springbootdemo/pom.xml b/docs/examples/JDBC/springbootdemo/pom.xml index df8a3f5d61..25b503b0e6 100644 --- a/docs/examples/JDBC/springbootdemo/pom.xml +++ b/docs/examples/JDBC/springbootdemo/pom.xml @@ -70,7 +70,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 diff --git a/docs/examples/JDBC/taosdemo/pom.xml b/docs/examples/JDBC/taosdemo/pom.xml index c36973947b..a80deeff94 100644 --- a/docs/examples/JDBC/taosdemo/pom.xml +++ b/docs/examples/JDBC/taosdemo/pom.xml @@ -67,7 +67,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 diff --git a/docs/examples/flink/Main.java b/docs/examples/flink/Main.java new file mode 100644 index 0000000000..12d79126cf --- /dev/null +++ b/docs/examples/flink/Main.java @@ -0,0 +1,579 @@ +package com.taosdata.flink.example; + +import com.taosdata.flink.cdc.TDengineCdcSource; +import com.taosdata.flink.common.TDengineCdcParams; +import com.taosdata.flink.common.TDengineConfigParams; +import com.taosdata.flink.sink.TDengineSink; +import com.taosdata.flink.source.TDengineSource; +import com.taosdata.flink.source.entity.SourceSplitSql; +import com.taosdata.flink.source.entity.SplitType; +import com.taosdata.flink.source.entity.TimestampSplitInfo; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.tmq.ConsumerRecords; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.shaded.curator5.com.google.common.base.Strings; +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.data.RowData; +import org.junit.Assert; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.SimpleDateFormat; +import java.time.Duration; + +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import javax.xml.transform.Source; + +import org.apache.flink.streaming.api.CheckpointingMode; + + + +public class Main { + static String jdbcUrl = "jdbc:TAOS-WS://localhost:6041?user=root&password=taosdata"; + static void prepare() throws ClassNotFoundException, SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + String insertQuery = "INSERT INTO " + + "power.d1001 USING power.meters TAGS('California.SanFrancisco', 1) " + + "VALUES " + + "('2024-12-19 19:12:45.642', 50.30000, 201, 0.31000) " + + "('2024-12-19 19:12:46.642', 82.60000, 202, 0.33000) " + + "('2024-12-19 19:12:47.642', 92.30000, 203, 0.31000) " + + "('2024-12-19 18:12:45.642', 50.30000, 201, 0.31000) " + + "('2024-12-19 18:12:46.642', 82.60000, 202, 0.33000) " + + "('2024-12-19 18:12:47.642', 92.30000, 203, 0.31000) " + + "('2024-12-19 17:12:45.642', 50.30000, 201, 0.31000) " + + "('2024-12-19 17:12:46.642', 82.60000, 202, 0.33000) " + + "('2024-12-19 17:12:47.642', 92.30000, 203, 0.31000) " + + "power.d1002 USING power.meters TAGS('Alabama.Montgomery', 2) " + + "VALUES " + + "('2024-12-19 19:12:45.642', 50.30000, 204, 0.25000) " + + "('2024-12-19 19:12:46.642', 62.60000, 205, 0.33000) " + + "('2024-12-19 19:12:47.642', 72.30000, 206, 0.31000) " + + "('2024-12-19 18:12:45.642', 50.30000, 204, 0.25000) " + + "('2024-12-19 18:12:46.642', 62.60000, 205, 0.33000) " + + "('2024-12-19 18:12:47.642', 72.30000, 206, 0.31000) " + + "('2024-12-19 17:12:45.642', 50.30000, 204, 0.25000) " + + "('2024-12-19 17:12:46.642', 62.60000, 205, 0.33000) " + + "('2024-12-19 17:12:47.642', 72.30000, 206, 0.31000) "; + + Class.forName("com.taosdata.jdbc.ws.WebSocketDriver"); + try (Connection connection = DriverManager.getConnection(jdbcUrl, properties); + Statement stmt = connection.createStatement()) { + + stmt.executeUpdate("DROP TOPIC IF EXISTS topic_meters"); + + stmt.executeUpdate("DROP database IF EXISTS power"); + // create database + int rowsAffected = stmt.executeUpdate("CREATE DATABASE IF NOT EXISTS power vgroups 5"); + + stmt.executeUpdate("use power"); + // you can check rowsAffected here + System.out.println("Create database power successfully, rowsAffected: " + rowsAffected); + // create table + rowsAffected = stmt.executeUpdate("CREATE STABLE IF NOT EXISTS meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);"); + // you can check rowsAffected here + System.out.println("Create stable power.meters successfully, rowsAffected: " + rowsAffected); + + stmt.executeUpdate("CREATE TOPIC topic_meters as SELECT ts, `current`, voltage, phase, location, groupid, tbname FROM meters"); + + int affectedRows = stmt.executeUpdate(insertQuery); + // you can check affectedRows here + System.out.println("Successfully inserted " + affectedRows + " rows to power.meters."); + + stmt.executeUpdate("DROP database IF EXISTS power_sink"); + // create database + stmt.executeUpdate("CREATE DATABASE IF NOT EXISTS power_sink vgroups 5"); + + stmt.executeUpdate("use power_sink"); + // you can check rowsAffected here + System.out.println("Create database power successfully, rowsAffected: " + rowsAffected); + // create table + stmt.executeUpdate("CREATE STABLE IF NOT EXISTS sink_meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);"); + // you can check rowsAffected here + + stmt.executeUpdate("CREATE TABLE IF NOT EXISTS sink_normal (ts timestamp, current float, voltage int, phase float);"); + // you can check rowsAffected here + + + } catch (Exception ex) { + // please refer to the JDBC specifications for detailed exceptions info + System.out.printf("Failed to create database power or stable meters, %sErrMessage: %s%n", + ex instanceof SQLException ? "ErrCode: " + ((SQLException) ex).getErrorCode() + ", " : "", + ex.getMessage()); + // Print stack trace for context in examples. Use logging in production. + throw ex; + } + } + + public static void main(String[] args) throws Exception { + prepare(); + if (args != null && args.length > 0 && args[0].equals("source")) { + testSource(); + } else if (args != null && args.length > 0 && args[0].equals("table")) { + testTableToSink(); + } else if (args != null && args.length > 0 && args[0].equals("cdc")) { + testCustomTypeCdc(); + }else if (args != null && args.length > 0 && args[0].equals("table-cdc")) { + testCdcTableToSink(); + } + } + + static SourceSplitSql getTimeSplit() { +// ANCHOR: time_interval +SourceSplitSql splitSql = new SourceSplitSql(); +splitSql.setSql("select ts, `current`, voltage, phase, groupid, location, tbname from meters") +.setSplitType(SplitType.SPLIT_TYPE_TIMESTAMP) +.setTimestampSplitInfo(new TimestampSplitInfo( + "2024-12-19 16:12:48.000", + "2024-12-19 19:12:48.000", + "ts", + Duration.ofHours(1), + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"), + ZoneId.of("Asia/Shanghai"))); +// ANCHOR_END: time_interval +return splitSql; + } + + static SourceSplitSql getTagSplit() throws Exception { +// ANCHOR: tag_split +SourceSplitSql splitSql = new SourceSplitSql(); +splitSql.setSql("select ts, current, voltage, phase, groupid, location from meters where voltage > 100") + .setTagList(Arrays.asList("groupid >100 and location = 'Shanghai'", + "groupid >50 and groupid < 100 and location = 'Guangzhou'", + "groupid >0 and groupid < 50 and location = 'Beijing'")) + .setSplitType(SplitType.SPLIT_TYPE_TAG); +// ANCHOR_END: tag_split +return splitSql; + } + + static SourceSplitSql getTableSqlit() { +// ANCHOR: table_split +SourceSplitSql splitSql = new SourceSplitSql(); +splitSql.setSelect("ts, current, voltage, phase, groupid, location") + .setTableList(Arrays.asList("d1001", "d1002")) + .setOther("order by ts limit 100") + .setSplitType(SplitType.SPLIT_TYPE_TABLE); +// ANCHOR_END: table_split + } + + //ANCHOR: source_test + static void testSource() throws Exception { + Properties connProps = new Properties(); + connProps.setProperty(TDengineConfigParams.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true"); + connProps.setProperty(TDengineConfigParams.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connProps.setProperty(TDengineConfigParams.VALUE_DESERIALIZER, "RowData"); + connProps.setProperty(TDengineConfigParams.TD_JDBC_URL, "jdbc:TAOS-WS://localhost:6041/power?user=root&password=taosdata"); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(3); + + splitSql.setSql("select ts, `current`, voltage, phase, groupid, location, tbname from meters") + .setSplitType(SplitType.SPLIT_TYPE_TIMESTAMP) + .setTimestampSplitInfo(new TimestampSplitInfo( + "2024-12-19 16:12:48.000", + "2024-12-19 19:12:48.000", + "ts", + Duration.ofHours(1), + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"), + ZoneId.of("Asia/Shanghai"))); + + TDengineSource source = new TDengineSource<>(connProps, sql, RowData.class); + DataStreamSource input = env.fromSource(source, WatermarkStrategy.noWatermarks(), "tdengine-source"); + DataStream resultStream = input.map((MapFunction) rowData -> { + StringBuilder sb = new StringBuilder(); + sb.append("ts: " + rowData.getTimestamp(0, 0) + + ", current: " + rowData.getFloat(1) + + ", voltage: " + rowData.getInt(2) + + ", phase: " + rowData.getFloat(3) + + ", location: " + new String(rowData.getBinary(4))); + sb.append("\n"); + return sb.toString(); + }); + resultStream.print(); + env.execute("tdengine flink source"); + + } + //ANCHOR_END: source_test + + //ANCHOR: source_custom_type_test + void testCustomTypeSource() throws Exception { + System.out.println("testTDengineSourceByTimeSplit start！"); + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connProps.setProperty(TDengineConfigParams.VALUE_DESERIALIZER, "com.taosdata.flink.entity.ResultSoureDeserialization"); + connProps.setProperty(TDengineConfigParams.TD_JDBC_URL, "jdbc:TAOS-WS://localhost:6041/power?user=root&password=taosdata"); + SourceSplitSql splitSql = new SourceSplitSql(); + splitSql.setSql("select ts, `current`, voltage, phase, groupid, location, tbname from meters") + .setSplitType(SplitType.SPLIT_TYPE_TIMESTAMP) + //按照时间分片 + .setTimestampSplitInfo(new TimestampSplitInfo( + "2024-12-19 16:12:48.000", + "2024-12-19 19:12:48.000", + "ts", + Duration.ofHours(1), + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"), + ZoneId.of("Asia/Shanghai"))); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(3); + TDengineSource source = new TDengineSource<>(connProps, splitSql, ResultBean.class); + DataStreamSource input = env.fromSource(source, WatermarkStrategy.noWatermarks(), "tdengine-source"); + DataStream resultStream = input.map((MapFunction) rowData -> { + StringBuilder sb = new StringBuilder(); + sb.append("ts: " + rowData.getTs() + + ", current: " + rowData.getCurrent() + + ", voltage: " + rowData.getVoltage() + + ", phase: " + rowData.getPhase() + + ", groupid: " + rowData.getGroupid() + + ", location" + rowData.getLocation() + + ", tbname: " + rowData.getTbname()); + sb.append("\n"); + totalVoltage.addAndGet(rowData.getVoltage()); + return sb.toString(); + }); + resultStream.print(); + env.execute("flink tdengine source"); + } + //ANCHOR_END: source_custom_type_test + + //ANCHOR: source_batch_test + void testBatchSource() throws Exception { + Properties connProps = new Properties(); + connProps.setProperty(TDengineConfigParams.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true"); + connProps.setProperty(TDengineConfigParams.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connProps.setProperty(TDengineConfigParams.VALUE_DESERIALIZER, "RowData"); + connProps.setProperty(TDengineConfigParams.TD_BATCH_MODE, "true"); + connProps.setProperty(TDengineConfigParams.TD_JDBC_URL, "jdbc:TAOS-WS://localhost:6041/power?user=root&password=taosdata"); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(3); + Class> typeClass = (Class>) (Class) SourceRecords.class; + SourceSplitSql sql = new SourceSplitSql("select ts, `current`, voltage, phase, tbname from meters"); + TDengineSource> source = new TDengineSource<>(connProps, sql, typeClass); + DataStreamSource> input = env.fromSource(source, WatermarkStrategy.noWatermarks(), "kafka-source"); + DataStream resultStream = input.map((MapFunction, String>) records -> { + StringBuilder sb = new StringBuilder(); + Iterator iterator = records.iterator(); + while (iterator.hasNext()) { + GenericRowData row = (GenericRowData) iterator.next(); + sb.append("ts: " + row.getTimestamp(0, 0) + + ", current: " + row.getFloat(1) + + ", voltage: " + row.getInt(2) + + ", phase: " + row.getFloat(3) + + ", location: " + new String(row.getBinary(4))); + sb.append("\n"); + totalVoltage.addAndGet(row.getInt(2)); + } + return sb.toString(); + }); + resultStream.print(); + env.execute("flink tdengine source"); + + } + //ANCHOR_END: source_batch_test + + //ANCHOR: cdc_source + void testTDengineCdc() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(3); + env.enableCheckpointing(100, AT_LEAST_ONCE); + env.getConfig().setRestartStrategy(RestartStrategies.noRestart()); + Properties config = new Properties(); + config.setProperty(TDengineCdcParams.CONNECT_TYPE, "ws"); + config.setProperty(TDengineCdcParams.BOOTSTRAP_SERVERS, "localhost:6041"); + config.setProperty(TDengineCdcParams.AUTO_OFFSET_RESET, "earliest"); + config.setProperty(TDengineCdcParams.MSG_WITH_TABLE_NAME, "true"); + config.setProperty(TDengineCdcParams.AUTO_COMMIT_INTERVAL_MS, "1000"); + config.setProperty(TDengineCdcParams.GROUP_ID, "group_1"); + config.setProperty(TDengineCdcParams.ENABLE_AUTO_COMMIT, "true"); + config.setProperty(TDengineCdcParams.CONNECT_USER, "root"); + config.setProperty(TDengineCdcParams.CONNECT_PASS, "taosdata"); + config.setProperty(TDengineCdcParams.VALUE_DESERIALIZER, "RowData"); + config.setProperty(TDengineCdcParams.VALUE_DESERIALIZER_ENCODING, "UTF-8"); + TDengineCdcSource tdengineSource = new TDengineCdcSource<>("topic_meters", config, RowData.class); + DataStreamSource input = env.fromSource(tdengineSource, WatermarkStrategy.noWatermarks(), "kafka-source"); + DataStream resultStream = input.map((MapFunction) rowData -> { + StringBuilder sb = new StringBuilder(); + sb.append("tsxx: " + rowData.getTimestamp(0, 0) + + ", current: " + rowData.getFloat(1) + + ", voltage: " + rowData.getInt(2) + + ", phase: " + rowData.getFloat(3) + + ", location: " + new String(rowData.getBinary(4))); + sb.append("\n"); + totalVoltage.addAndGet(rowData.getInt(2)); + return sb.toString(); + }); + resultStream.print(); + JobClient jobClient = env.executeAsync("Flink test cdc Example"); + Thread.sleep(5000L); + // The task submitted by Flink UI cannot be cancle and needs to be stopped on the UI page. + jobClient.cancel().get(); + } + //ANCHOR_END: cdc_source + + //ANCHOR: cdc_batch_source + void testTDengineCdcBatch() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(3); + Properties config = new Properties(); + config.setProperty(TDengineCdcParams.CONNECT_TYPE, "ws"); + config.setProperty(TDengineCdcParams.BOOTSTRAP_SERVERS, "localhost:6041"); + config.setProperty(TDengineCdcParams.AUTO_OFFSET_RESET, "earliest"); + config.setProperty(TDengineCdcParams.MSG_WITH_TABLE_NAME, "true"); + config.setProperty(TDengineCdcParams.AUTO_COMMIT_INTERVAL_MS, "1000"); + config.setProperty(TDengineCdcParams.GROUP_ID, "group_1"); + config.setProperty(TDengineCdcParams.CONNECT_USER, "root"); + config.setProperty(TDengineCdcParams.CONNECT_PASS, "taosdata"); + config.setProperty(TDengineCdcParams.VALUE_DESERIALIZER, "RowData"); + config.setProperty(TDengineCdcParams.VALUE_DESERIALIZER_ENCODING, "UTF-8"); + config.setProperty(TDengineCdcParams.TMQ_BATCH_MODE, "true"); + + Class> typeClass = (Class>) (Class) ConsumerRecords.class; + TDengineCdcSource> tdengineSource = new TDengineCdcSource<>("topic_meters", config, typeClass); + DataStreamSource> input = env.fromSource(tdengineSource, WatermarkStrategy.noWatermarks(), "kafka-source"); + DataStream resultStream = input.map((MapFunction, String>) records -> { + Iterator> iterator = records.iterator(); + StringBuilder sb = new StringBuilder(); + while (iterator.hasNext()) { + GenericRowData row = (GenericRowData) iterator.next().value(); + sb.append("tsxx: " + row.getTimestamp(0, 0) + + ", current: " + row.getFloat(1) + + ", voltage: " + row.getInt(2) + + ", phase: " + row.getFloat(3) + + ", location: " + new String(row.getBinary(4))); + sb.append("\n"); + totalVoltage.addAndGet(row.getInt(2)); + } + return sb.toString(); + + }); + + resultStream.print(); + JobClient jobClient = env.executeAsync("Flink test cdc Example"); + Thread.sleep(5000L); + jobClient.cancel().get(); + } + //ANCHOR_END: cdc_batch_source + + //ANCHOR: cdc_custom_type_test + static void testCustomTypeCdc() throws Exception { + System.out.println("testCustomTypeTDengineCdc start！"); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(3); + env.enableCheckpointing(100, AT_LEAST_ONCE); + env.getConfig().setRestartStrategy(RestartStrategies.noRestart()); + env.getCheckpointConfig().setTolerableCheckpointFailureNumber(4); + Properties config = new Properties(); + config.setProperty(TDengineCdcParams.CONNECT_TYPE, "ws"); + config.setProperty(TDengineCdcParams.BOOTSTRAP_SERVERS, "localhost:6041"); + config.setProperty(TDengineCdcParams.AUTO_OFFSET_RESET, "earliest"); + config.setProperty(TDengineCdcParams.MSG_WITH_TABLE_NAME, "true"); + config.setProperty(TDengineCdcParams.AUTO_COMMIT_INTERVAL_MS, "1000"); + config.setProperty(TDengineCdcParams.GROUP_ID, "group_1"); + config.setProperty(TDengineCdcParams.CONNECT_USER, "root"); + config.setProperty(TDengineCdcParams.CONNECT_PASS, "taosdata"); + config.setProperty(TDengineCdcParams.VALUE_DESERIALIZER, "com.taosdata.flink.entity.ResultDeserializer"); + config.setProperty(TDengineCdcParams.VALUE_DESERIALIZER_ENCODING, "UTF-8"); + TDengineCdcSource tdengineSource = new TDengineCdcSource<>("topic_meters", config, ResultBean.class); + DataStreamSource input = env.fromSource(tdengineSource, WatermarkStrategy.noWatermarks(), "kafka-source"); + DataStream resultStream = input.map((MapFunction) rowData -> { + StringBuilder sb = new StringBuilder(); + sb.append("ts: " + rowData.getTs() + + ", current: " + rowData.getCurrent() + + ", voltage: " + rowData.getVoltage() + + ", phase: " + rowData.getPhase() + + ", groupid: " + rowData.getGroupid() + + ", location" + rowData.getLocation() + + ", tbname: " + rowData.getTbname()); + sb.append("\n"); + totalVoltage.addAndGet(rowData.getVoltage()); + return sb.toString(); + }); + resultStream.print(); + JobClient jobClient = env.executeAsync("Flink test cdc Example"); + Thread.sleep(5000L); + jobClient.cancel().get(); + } + //ANCHOR_END: cdc_custom_type_test + + //ANCHOR: RowDataToSink + static void testRowDataToSink() throws Exception { + Properties connProps = new Properties(); + connProps.setProperty(TDengineConfigParams.VALUE_DESERIALIZER, "RowData"); + connProps.setProperty(TDengineConfigParams.TD_JDBC_URL, "jdbc:TAOS-WS://localhost:6041/power?user=root&password=taosdata"); + SourceSplitSql sql = new SourceSplitSql("select ts, `current`, voltage, phase, tbname from meters"); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + env.enableCheckpointing(1000, CheckpointingMode.AT_LEAST_ONCE); + TDengineSource source = new TDengineSource<>(connProps, sql, RowData.class); + DataStreamSource input = env.fromSource(source, WatermarkStrategy.noWatermarks(), "tdengine-source"); + Properties sinkProps = new Properties(); + sinkProps.setProperty(TDengineConfigParams.VALUE_DESERIALIZER, "RowData"); + sinkProps.setProperty(TDengineConfigParams.TD_SOURCE_TYPE, "tdengine_source"); + sinkProps.setProperty(TDengineConfigParams.TD_DATABASE_NAME, "power_sink"); + sinkProps.setProperty(TDengineConfigParams.TD_SUPERTABLE_NAME, "sink_meters"); + sinkProps.setProperty(TDengineConfigParams.TD_JDBC_URL, "jdbc:TAOS-WS://localhost:6041/power_sink?user=root&password=taosdata"); + sinkProps.setProperty(TDengineConfigParams.TD_BATCH_SIZE, "2000"); + + // Arrays.asList The list of target table field names needs to be consistent with the data order + TDengineSink sink = new TDengineSink<>(sinkProps, + Arrays.asList("ts", "current", "voltage", "phase", "groupid", "location", "tbname")); + + input.sinkTo(sink); + env.execute("flink tdengine source"); + } + //ANCHOR_END: RowDataToSink + + //ANCHOR: CdcRowDataToSink + static void testCdcToSink() throws Exception { + System.out.println("testTDengineCdcToTdSink start！"); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(3); + env.enableCheckpointing(500, CheckpointingMode.AT_LEAST_ONCE); + env.getCheckpointConfig().setMinPauseBetweenCheckpoints(500); + env.getCheckpointConfig().setCheckpointTimeout(5000); + Properties config = new Properties(); + config.setProperty(TDengineCdcParams.CONNECT_TYPE, "ws"); + config.setProperty(TDengineCdcParams.BOOTSTRAP_SERVERS, "localhost:6041"); + config.setProperty(TDengineCdcParams.AUTO_OFFSET_RESET, "earliest"); + config.setProperty(TDengineCdcParams.GROUP_ID, "group_1"); + config.setProperty(TDengineCdcParams.CONNECT_USER, "root"); + config.setProperty(TDengineCdcParams.CONNECT_PASS, "taosdata"); + config.setProperty(TDengineCdcParams.VALUE_DESERIALIZER, "RowData"); + config.setProperty(TDengineCdcParams.VALUE_DESERIALIZER_ENCODING, "UTF-8"); + + TDengineCdcSource tdengineSource = new TDengineCdcSource<>("topic_meters", config, RowData.class); + DataStreamSource input = env.fromSource(tdengineSource, WatermarkStrategy.noWatermarks(), "tdengine-source"); + + Properties sinkProps = new Properties(); + sinkProps.setProperty(TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT, "true"); + sinkProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + sinkProps.setProperty(TDengineConfigParams.VALUE_DESERIALIZER, "RowData"); + sinkProps.setProperty(TDengineConfigParams.TD_DATABASE_NAME, "power_sink"); + sinkProps.setProperty(TDengineConfigParams.TD_SUPERTABLE_NAME, "sink_meters"); + sinkProps.setProperty(TDengineConfigParams.TD_JDBC_URL, "jdbc:TAOS-WS://localhost:6041/power?user=root&password=taosdata"); + sinkProps.setProperty(TDengineConfigParams.TD_BATCH_SIZE, "2000"); + + TDengineSink sink = new TDengineSink<>(sinkProps, Arrays.asList("ts", "current", "voltage", "phase", "location", "groupid", "tbname")); + input.sinkTo(sink); + JobClient jobClient = env.executeAsync("Flink test cdc Example"); + Thread.sleep(6000L); + jobClient.cancel().get(); + System.out.println("testTDengineCdcToTdSink finish！"); + } + //ANCHOR_END: CdcRowDataToSink + + //ANCHOR: source_table + static void testTableToSink() throws Exception { + System.out.println("testTableToSink start！"); + EnvironmentSettings fsSettings = EnvironmentSettings.newInstance().inStreamingMode().build(); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(3); + env.enableCheckpointing(1000, CheckpointingMode.AT_LEAST_ONCE); + StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env, fsSettings); + String tdengineSourceTableDDL = "CREATE TABLE `meters` (" + + " ts TIMESTAMP," + + " `current` FLOAT," + + " voltage INT," + + " phase FLOAT," + + " location VARBINARY," + + " groupid INT," + + " tbname VARBINARY" + + ") WITH (" + + " 'connector' = 'tdengine-connector'," + + " 'td.jdbc.url' = 'jdbc:TAOS-WS://localhost:6041/power?user=root&password=taosdata'," + + " 'td.jdbc.mode' = 'source'," + + " 'table-name' = 'meters'," + + " 'scan.query' = 'SELECT ts, `current`, voltage, phase, location, groupid, tbname FROM `meters`'" + + ")"; + + + String tdengineSinkTableDDL = "CREATE TABLE `sink_meters` (" + + " ts TIMESTAMP," + + " `current` FLOAT," + + " voltage INT," + + " phase FLOAT," + + " location VARBINARY," + + " groupid INT," + + " tbname VARBINARY" + + ") WITH (" + + " 'connector' = 'tdengine-connector'," + + " 'td.jdbc.mode' = 'sink'," + + " 'td.jdbc.url' = 'jdbc:TAOS-WS://localhost:6041/power_sink?user=root&password=taosdata'," + + " 'sink.db.name' = 'power_sink'," + + " 'sink.supertable.name' = 'sink_meters'" + + ")"; + + tableEnv.executeSql(tdengineSourceTableDDL); + tableEnv.executeSql(tdengineSinkTableDDL); + tableEnv.executeSql("INSERT INTO sink_meters SELECT ts, `current`, voltage, phase, location, groupid, tbname FROM `meters`"); + } + //ANCHOR_END: source_table + + //ANCHOR: cdc_table + static void testCdcTableToSink() throws Exception { + EnvironmentSettings fsSettings = EnvironmentSettings.newInstance().inStreamingMode().build(); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(5); + env.enableCheckpointing(1000, CheckpointingMode.AT_LEAST_ONCE); + StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env, fsSettings); + String tdengineSourceTableDDL = "CREATE TABLE `meters` (" + + " ts TIMESTAMP," + + " `current` FLOAT," + + " voltage INT," + + " phase FLOAT," + + " location VARBINARY," + + " groupid INT," + + " tbname VARBINARY" + + ") WITH (" + + " 'connector' = 'tdengine-connector'," + + " 'bootstrap.servers' = 'localhost:6041'," + + " 'td.jdbc.mode' = 'cdc'," + + " 'group.id' = 'group_22'," + + " 'auto.offset.reset' = 'earliest'," + + " 'enable.auto.commit' = 'false'," + + " 'topic' = 'topic_meters'" + + ")"; + + + String tdengineSinkTableDDL = "CREATE TABLE `sink_meters` (" + + " ts TIMESTAMP," + + " `current` FLOAT," + + " voltage INT," + + " phase FLOAT," + + " location VARBINARY," + + " groupid INT," + + " tbname VARBINARY" + + ") WITH (" + + " 'connector' = 'tdengine-connector'," + + " 'td.jdbc.mode' = 'cdc'," + + " 'td.jdbc.url' = 'jdbc:TAOS-WS://localhost:6041/power_sink?user=root&password=taosdata'," + + " 'sink.db.name' = 'power_sink'," + + " 'sink.supertable.name' = 'sink_meters'" + + ")"; + + tableEnv.executeSql(tdengineSourceTableDDL); + tableEnv.executeSql(tdengineSinkTableDDL); + + TableResult tableResult = tableEnv.executeSql("INSERT INTO sink_meters SELECT ts, `current`, voltage, phase, location, groupid, tbname FROM `meters`"); + + Thread.sleep(5000L); + tableResult.getJobClient().get().cancel().get(); + } + //ANCHOR_END: cdc_table + + +} diff --git a/docs/examples/go/go.mod b/docs/examples/go/go.mod index ed8fde2d9f..2dd5d4ec2b 100644 --- a/docs/examples/go/go.mod +++ b/docs/examples/go/go.mod @@ -2,7 +2,7 @@ module goexample go 1.17 -require github.com/taosdata/driver-go/v3 v3.5.6 +require github.com/taosdata/driver-go/v3 v3.6.0 require ( github.com/google/uuid v1.3.0 // indirect diff --git a/docs/examples/go/go.sum b/docs/examples/go/go.sum index 61841429ee..ee2ad55588 100644 --- a/docs/examples/go/go.sum +++ b/docs/examples/go/go.sum @@ -18,8 +18,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/taosdata/driver-go/v3 v3.5.6 h1:LDVtMyT3B9p2VREsd5KKM91D4Y7P4kSdh2SQumXi8bk= -github.com/taosdata/driver-go/v3 v3.5.6/go.mod h1:H2vo/At+rOPY1aMzUV9P49SVX7NlXb3LAbKw+MCLrmU= +github.com/taosdata/driver-go/v3 v3.6.0 h1:4dRXMl01DhIS5xBXUvtkkB+MjL8g64zN674xKd+ojTE= +github.com/taosdata/driver-go/v3 v3.6.0/go.mod h1:H2vo/At+rOPY1aMzUV9P49SVX7NlXb3LAbKw+MCLrmU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/docs/examples/go/stmt2/native/main.go b/docs/examples/go/stmt2/native/main.go new file mode 100644 index 0000000000..fc3ab763c2 --- /dev/null +++ b/docs/examples/go/stmt2/native/main.go @@ -0,0 +1,84 @@ +package main + +import ( + "database/sql/driver" + "fmt" + "log" + "math/rand" + "time" + + "github.com/taosdata/driver-go/v3/af" + "github.com/taosdata/driver-go/v3/common" + "github.com/taosdata/driver-go/v3/common/stmt" +) + +func main() { + host := "127.0.0.1" + numOfSubTable := 10 + numOfRow := 10 + db, err := af.Open(host, "root", "taosdata", "", 0) + if err != nil { + log.Fatalln("Failed to connect to " + host + "; ErrMessage: " + err.Error()) + } + defer db.Close() + // prepare database and table + _, err = db.Exec("CREATE DATABASE IF NOT EXISTS power") + if err != nil { + log.Fatalln("Failed to create database power, ErrMessage: " + err.Error()) + } + _, err = db.Exec("USE power") + if err != nil { + log.Fatalln("Failed to use database power, ErrMessage: " + err.Error()) + } + _, err = db.Exec("CREATE STABLE IF NOT EXISTS meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))") + if err != nil { + log.Fatalln("Failed to create stable meters, ErrMessage: " + err.Error()) + } + // prepare statement + sql := "INSERT INTO ? USING meters TAGS(?,?) VALUES (?,?,?,?)" + reqID := common.GetReqID() + stmt2 := db.Stmt2(reqID, false) + err = stmt2.Prepare(sql) + if err != nil { + log.Fatalln("Failed to prepare sql, sql: " + sql + ", ErrMessage: " + err.Error()) + } + for i := 1; i <= numOfSubTable; i++ { + // generate column data + current := time.Now() + columns := make([][]driver.Value, 4) + for j := 0; j < numOfRow; j++ { + columns[0] = append(columns[0], current.Add(time.Millisecond*time.Duration(j))) + columns[1] = append(columns[1], rand.Float32()*30) + columns[2] = append(columns[2], rand.Int31n(300)) + columns[3] = append(columns[3], rand.Float32()) + } + // generate bind data + tableName := fmt.Sprintf("d_bind_%d", i) + tags := []driver.Value{int32(i), []byte(fmt.Sprintf("location_%d", i))} + bindData := []*stmt.TaosStmt2BindData{ + { + TableName: tableName, + Tags: tags, + Cols: columns, + }, + } + // bind params + err = stmt2.Bind(bindData) + if err != nil { + log.Fatalln("Failed to bind params, ErrMessage: " + err.Error()) + } + // execute batch + err = stmt2.Execute() + if err != nil { + log.Fatalln("Failed to exec, ErrMessage: " + err.Error()) + } + // get affected rows + affected := stmt2.GetAffectedRows() + // you can check exeResult here + fmt.Printf("Successfully inserted %d rows to %s.\n", affected, tableName) + } + err = stmt2.Close() + if err != nil { + log.Fatal("failed to close statement, err:", err) + } +} diff --git a/docs/examples/java/pom.xml b/docs/examples/java/pom.xml index e1a9504249..63ce3159e6 100644 --- a/docs/examples/java/pom.xml +++ b/docs/examples/java/pom.xml @@ -22,7 +22,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 diff --git a/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingExtendInterfaceDemo.java b/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingExtendInterfaceDemo.java new file mode 100644 index 0000000000..8a83c27bbc --- /dev/null +++ b/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingExtendInterfaceDemo.java @@ -0,0 +1,87 @@ +package com.taos.example; + +import com.taosdata.jdbc.ws.TSWSPreparedStatement; + +import java.sql.*; +import java.util.ArrayList; +import java.util.Random; + +// ANCHOR: para_bind +public class WSParameterBindingExtendInterfaceDemo { + + // modify host to your own + private static final String host = "127.0.0.1"; + private static final Random random = new Random(System.currentTimeMillis()); + private static final int numOfSubTable = 10, numOfRow = 10; + + public static void main(String[] args) throws SQLException { + + String jdbcUrl = "jdbc:TAOS-WS://" + host + ":6041"; + try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) { + init(conn); + + String sql = "INSERT INTO ? USING power.meters TAGS(?,?) VALUES (?,?,?,?)"; + + try (TSWSPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSWSPreparedStatement.class)) { + + for (int i = 1; i <= numOfSubTable; i++) { + // set table name + pstmt.setTableName("d_bind_" + i); + + // set tags + pstmt.setTagInt(0, i); + pstmt.setTagString(1, "location_" + i); + + // set column ts + ArrayList tsList = new ArrayList<>(); + long current = System.currentTimeMillis(); + for (int j = 0; j < numOfRow; j++) + tsList.add(current + j); + pstmt.setTimestamp(0, tsList); + + // set column current + ArrayList currentList = new ArrayList<>(); + for (int j = 0; j < numOfRow; j++) + currentList.add(random.nextFloat() * 30); + pstmt.setFloat(1, currentList); + + // set column voltage + ArrayList voltageList = new ArrayList<>(); + for (int j = 0; j < numOfRow; j++) + voltageList.add(random.nextInt(300)); + pstmt.setInt(2, voltageList); + + // set column phase + ArrayList phaseList = new ArrayList<>(); + for (int j = 0; j < numOfRow; j++) + phaseList.add(random.nextFloat()); + pstmt.setFloat(3, phaseList); + // add column + pstmt.columnDataAddBatch(); + } + // execute column + pstmt.columnDataExecuteBatch(); + // you can check exeResult here + System.out.println("Successfully inserted " + (numOfSubTable * numOfRow) + " rows to power.meters."); + } + } catch (Exception ex) { + // please refer to the JDBC specifications for detailed exceptions info + System.out.printf("Failed to insert to table meters using stmt, %sErrMessage: %s%n", + ex instanceof SQLException ? "ErrCode: " + ((SQLException) ex).getErrorCode() + ", " : "", + ex.getMessage()); + // Print stack trace for context in examples. Use logging in production. + ex.printStackTrace(); + throw ex; + } + } + + private static void init(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE DATABASE IF NOT EXISTS power"); + stmt.execute("USE power"); + stmt.execute( + "CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))"); + } + } +} +// ANCHOR_END: para_bind diff --git a/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingBasicDemo.java b/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingStdInterfaceDemo.java similarity index 61% rename from docs/examples/java/src/main/java/com/taos/example/WSParameterBindingBasicDemo.java rename to docs/examples/java/src/main/java/com/taos/example/WSParameterBindingStdInterfaceDemo.java index 1353ebbddc..7f0e523b97 100644 --- a/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingBasicDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingStdInterfaceDemo.java @@ -1,12 +1,10 @@ package com.taos.example; -import com.taosdata.jdbc.ws.TSWSPreparedStatement; - import java.sql.*; import java.util.Random; // ANCHOR: para_bind -public class WSParameterBindingBasicDemo { +public class WSParameterBindingStdInterfaceDemo { // modify host to your own private static final String host = "127.0.0.1"; @@ -19,31 +17,29 @@ public class WSParameterBindingBasicDemo { try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) { init(conn); - String sql = "INSERT INTO ? USING power.meters TAGS(?,?) VALUES (?,?,?,?)"; + // If you are certain that the child table exists, you can avoid binding the tag column to improve performance. + String sql = "INSERT INTO power.meters (tbname, groupid, location, ts, current, voltage, phase) VALUES (?,?,?,?,?,?,?)"; - try (TSWSPreparedStatement pstmt = conn.prepareStatement(sql).unwrap(TSWSPreparedStatement.class)) { + try (PreparedStatement pstmt = conn.prepareStatement(sql)) { + long current = System.currentTimeMillis(); for (int i = 1; i <= numOfSubTable; i++) { - // set table name - pstmt.setTableName("d_bind_" + i); - - // set tags - pstmt.setTagInt(0, i); - pstmt.setTagString(1, "location_" + i); - - // set columns - long current = System.currentTimeMillis(); for (int j = 0; j < numOfRow; j++) { - pstmt.setTimestamp(1, new Timestamp(current + j)); - pstmt.setFloat(2, random.nextFloat() * 30); - pstmt.setInt(3, random.nextInt(300)); - pstmt.setFloat(4, random.nextFloat()); + pstmt.setString(1, "d_bind_" + i); + + pstmt.setInt(2, i); + pstmt.setString(3, "location_" + i); + + pstmt.setTimestamp(4, new Timestamp(current + j)); + pstmt.setFloat(5, random.nextFloat() * 30); + pstmt.setInt(6, random.nextInt(300)); + pstmt.setFloat(7, random.nextFloat()); pstmt.addBatch(); } - int[] exeResult = pstmt.executeBatch(); - // you can check exeResult here - System.out.println("Successfully inserted " + exeResult.length + " rows to power.meters."); } + int[] exeResult = pstmt.executeBatch(); + // you can check exeResult here + System.out.println("Successfully inserted " + exeResult.length + " rows to power.meters."); } } catch (Exception ex) { // please refer to the JDBC specifications for detailed exceptions info diff --git a/docs/examples/java/src/test/java/com/taos/test/TestAll.java b/docs/examples/java/src/test/java/com/taos/test/TestAll.java index a92ddd116c..f73b9d42ee 100644 --- a/docs/examples/java/src/test/java/com/taos/test/TestAll.java +++ b/docs/examples/java/src/test/java/com/taos/test/TestAll.java @@ -118,9 +118,14 @@ public class TestAll { } @Test - public void testWsStmtBasic() throws Exception { + public void testWsStmtStd() throws Exception { dropDB("power"); - WSParameterBindingBasicDemo.main(args); + WSParameterBindingStdInterfaceDemo.main(args); + } + @Test + public void testWsStmtExtend() throws Exception { + dropDB("power"); + WSParameterBindingExtendInterfaceDemo.main(args); } @Test diff --git a/docs/examples/python/stmt2_native.py b/docs/examples/python/stmt2_native.py new file mode 100644 index 0000000000..72f01c9038 --- /dev/null +++ b/docs/examples/python/stmt2_native.py @@ -0,0 +1,71 @@ +import taos +from datetime import datetime +import random + +numOfSubTable = 10 +numOfRow = 10 + +conn = None +stmt2 = None +host="localhost" +port=6030 +try: + # 1 connect + conn = taos.connect( + user="root", + password="taosdata", + host=host, + port=port, + ) + + # 2 create db and table + conn.execute("CREATE DATABASE IF NOT EXISTS power") + conn.execute("USE power") + conn.execute( + "CREATE TABLE IF NOT EXISTS `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(16))" + ) + + # 3 prepare + sql = "INSERT INTO ? USING meters (groupid, location) TAGS(?,?) VALUES (?,?,?,?)" + stmt2 = conn.statement2(sql) + + tbnames = [] + tags = [] + datas = [] + + for i in range(numOfSubTable): + # tbnames + tbnames.append(f"d_bind_{i}") + # tags + tags.append([i, f"location_{i}"]) + # datas + current = int(datetime.now().timestamp() * 1000) + timestamps = [] + currents = [] + voltages = [] + phases = [] + for j in range (numOfRow): + timestamps.append(current + i*1000 + j) + currents.append(float(random.random() * 30)) + voltages.append(random.randint(100, 300)) + phases.append(float(random.random())) + data = [timestamps, currents, voltages, phases] + datas.append(data) + + # 4 bind param + stmt2.bind_param(tbnames, tags, datas) + + # 5 execute + stmt2.execute() + + # show + print(f"Successfully inserted with stmt2 to power.meters. child={numOfSubTable} rows={numOfRow} \n") + +except Exception as err: + print(f"Failed to insert to table meters using stmt2, ErrMessage:{err}") + raise err +finally: + if stmt2: + stmt2.close() + if conn: + conn.close() diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index 2381c49d93..fa22f750f5 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -89,7 +89,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 com.taosdata.jdbc taos-jdbcdriver - 3.4.0 + 3.5.1 ``` diff --git a/docs/zh/07-develop/05-stmt.md b/docs/zh/07-develop/05-stmt.md index 74b44ba8e6..1917a86e74 100644 --- a/docs/zh/07-develop/05-stmt.md +++ b/docs/zh/07-develop/05-stmt.md @@ -26,10 +26,16 @@ import TabItem from "@theme/TabItem"; ## WebSocket 连接 + +参数绑定有两种接口使用方式，一种是 JDBC 标准接口，一种是扩展接口，扩展接口性能更好一些。 + ```java -{{#include docs/examples/java/src/main/java/com/taos/example/WSParameterBindingBasicDemo.java:para_bind}} +{{#include docs/examples/java/src/main/java/com/taos/example/WSParameterBindingStdInterfaceDemo.java:para_bind}} ``` +```java +{{#include docs/examples/java/src/main/java/com/taos/example/WSParameterBindingExtendInterfaceDemo.java:para_bind}} +``` 这是一个[更详细的参数绑定示例](https://github.com/taosdata/TDengine/blob/main/docs/examples/java/src/main/java/com/taos/example/WSParameterBindingFullDemo.java) @@ -87,13 +93,24 @@ import TabItem from "@theme/TabItem"; ```python -{{#include docs/examples/python/stmt_native.py}} +{{#include docs/examples/python/stmt2_native.py}} ``` + +stmt2 绑定参数的示例代码如下（go 连接器 v3.6.0 及以上，TDengine v3.3.5.0 及以上）： + +```go +{{#include docs/examples/go/stmt2/native/main.go}} +``` + +stmt 绑定参数的示例代码如下： + ```go {{#include docs/examples/go/stmt/native/main.go}} ``` + + diff --git a/docs/zh/08-operation/12-multi.md b/docs/zh/08-operation/12-multi.md index d18957d7d2..994192e8fc 100644 --- a/docs/zh/08-operation/12-multi.md +++ b/docs/zh/08-operation/12-multi.md @@ -60,11 +60,15 @@ dataDir /mnt/data6 2 0 ## 对象存储 -本节介绍在 TDengine Enterprise 如何使用 S3 对象存储，本功能基于通用 S3 SDK 实现，对各个 S3 平台的访问参数进行了兼容适配，可以访问如 minio，腾讯云 COS，Amazon S3 等对象存储服务。通过适当的参数配置，可以把大部分较冷的时序数据存储到 S3 服务中。 +本节介绍在 TDengine Enterprise 版本中如何使用对象存储功能，如 Amazon S3、Azure Blob Storage、华为 OBS、腾讯云 COS、阿里云 OSS、MinIO 等对象存储服务。 **注意** 在配合多级存储使用时，每一级存储介质上保存的数据都有可能被按规则备份到远程对象存储中并删除本地数据文件。 -### 配置方式 +### S3 对象存储 + +本功能基于通用 S3 SDK 实现，对各个 S3 平台的访问参数进行了兼容适配，通过适当的参数配置，可以把大部分较冷的时序数据存储到 S3 服务中。 + +#### 配置方式 在配置文件 /etc/taos/taos.cfg 中，添加用于 S3 访问的参数： @@ -78,7 +82,7 @@ dataDir /mnt/data6 2 0 | s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 | | s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1。 | -### 检查配置参数可用性 +#### 检查配置参数可用性 在 taos.cfg 中完成对 S3 的配置后，通过 taosd 命令的 checks3 参数可以检查所配置的 S3 服务是否可用： @@ -88,7 +92,7 @@ taosd --checks3 如果配置的 S3 服务无法访问，此命令会在运行过程中输出相应的错误信息。 -### 创建使用 S3 的 DB +#### 创建使用 S3 的 DB 完成配置后，即可启动 TDengine 集群，创建使用 S3 的数据库，比如： @@ -112,11 +116,11 @@ s3migrate database ; | 2 | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 | | 3 | s3_compact | 1 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作 | -### 对象存储读写次数估算 +#### 对象存储读写次数估算 对象存储服务的使用成本与存储的数据量及请求次数相关，下面分别介绍数据的上传及下载过程。 -#### 数据上传 +##### 数据上传 当 TSDB 时序数据超过 `s3_keeplocal` 参数指定的时间，相关的数据文件会被切分成多个文件块，每个文件块的默认大小是 512M 字节 (`s3_chunkpages * tsdb_pagesize`)。除了最后一个文件块保留在本地文件系统外，其余的文件块会被上传到对象存储服务。 @@ -128,7 +132,7 @@ s3migrate database ; 其它类型的文件如 head, stt, sma 等，保留在本地文件系统，以加速预计算相关查询。 -#### 数据下载 +##### 数据下载 在查询操作中，如果需要访问对象存储中的数据，TSDB 不会下载整个数据文件，而是计算所需数据在文件中的位置，只下载相应的数据到 TSDB 页缓存中，然后将数据返回给查询执行引擎。后续查询首先检查页缓存，查看数据是否已被缓存。如果数据已缓存，则直接使用缓存中的数据，而无需重复从对象存储下载，从而有效降低从对象存储下载数据的次数。 @@ -140,15 +144,15 @@ s3migrate database ; 页缓存是内存缓存，节点重启后，再次查询需要重新下载数据。缓存采用 LRU (Least Recently Used) 策略，当缓存空间不足时，最近最少使用的数据将被淘汰。缓存的大小可以通过 `s3PageCacheSize` 参数进行调整，通常来说，缓存越大，下载次数越少。 -## Azure Blob 存储 -本节介绍在 TDengine Enterprise 如何使用微软 Azure Blob 对象存储。本功能是上一小节‘对象存储’功能的扩展，需额外依赖 Flexify 服务提供的 S3 网关。通过适当的参数配置，可以把大部分较冷的时序数据存储到 Azure Blob 服务中。 +### Azure Blob 存储 +本节介绍在 TDengine Enterprise 版本中如何使用微软 Azure Blob 存储。本功能可以通过两个方式使用：利用 Flexify 服务提供的 S3 网关功能和不依赖 Flexify 服务。通过配置参数，可以把大部分较冷的时序数据存储到 Azure Blob 服务中。 -### Flexify 服务 +#### Flexify 服务 Flexify 是 Azure Marketplace 中的一款应用程序，允许兼容 S3 的应用程序通过标准 S3 API 在 Azure Blob Storage 中存储数据。可使用多个 Flexify 服务对同一个 Blob 存储建立多个 S3 网关。 部署方式请参考 [Flexify](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/flexify.azure-s3-api?tab=Overview) 应用页面说明。 -### 配置方式 +##### 配置方式 在配置文件 /etc/taos/taos.cfg 中，添加用于 S3 访问的参数： @@ -164,7 +168,7 @@ s3BucketName td-test - 在某一 S3 服务上操作失败后会切换至其他服务，全部服务都失败后将返回最后产生的错误码 - 最大支持的 S3 服务配置数为 10 -### 不依赖 Flexify 服务 +#### 不依赖 Flexify 服务 用户界面同 S3，不同的地方在于下面三个参数的配置： diff --git a/docs/zh/08-operation/14-user.md b/docs/zh/08-operation/14-user.md index a894570b46..3a080619aa 100644 --- a/docs/zh/08-operation/14-user.md +++ b/docs/zh/08-operation/14-user.md @@ -17,7 +17,7 @@ create user user_name pass'password' [sysinfo {1|0}] [createdb {1|0}] 相关参数说明如下。 - user_name：用户名最长不超过 23 个字节。 -- password：密码长度必须为 8 到 16 位，并且至少包含大写字母、小写字母、数字、特殊字符中的三类。特殊字符包括 `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`。 +- password：密码长度必须为 8 到 16 位，并且至少包含大写字母、小写字母、数字、特殊字符中的三类。特殊字符包括 `! @ # $ % ^ & * ( ) - _ + = [ ] { } : ; > < ? | ~ , .`。（始自 3.3.5.0 版本） - sysinfo ：用户是否可以查看系统信息。1 表示可以查看，0 表示不可以查看。系统信息包括服务端配置信息、服务端各种节点信息，如 dnode、查询节点（qnode）等，以及与存储相关的信息等。默认为可以查看系统信息。 - createdb：用户是否可以创建数据库。1 表示可以创建，0 表示不可以创建。缺省值为 0。// 从 TDengine 企业版 3.3.2.0 开始支持 diff --git a/docs/zh/10-third-party/01-collection/12-flink.md b/docs/zh/10-third-party/01-collection/12-flink.md new file mode 100644 index 0000000000..e589e36c9a --- /dev/null +++ b/docs/zh/10-third-party/01-collection/12-flink.md @@ -0,0 +1,371 @@ +--- +sidebar_label: Flink +title: TDengine Flink Connector +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Apache Flink 是一款由 Apache 软件基金会支持的开源分布式流批一体化处理框架，可用于流处理、批处理、复杂事件处理、实时数据仓库构建及为机器学习提供实时数据支持等诸多大数据处理场景。与此同时，Flink 拥有丰富的连接器与各类工具，可对接众多不同类型的数据源实现数据的读取与写入。在数据处理的过程中，Flink 还提供了一系列可靠的容错机制，有力保障任务即便遭遇意外状况，依然能稳定、持续运行。 + +借助 TDengine 的 Flink 连接器，Apache Flink 得以与 TDengine 数据库无缝对接，一方面能够将经过复杂运算和深度分析后所得到的结果精准存入 TDengine 数据库，实现数据的高效存储与管理；另一方面，也可以从 TDengine 数据库中快速、稳定地读取海量数据，并在此基础上进行全面、深入的分析处理，充分挖掘数据的潜在价值，为企业的决策制定提供有力的数据支持和科学依据，极大地提升数据处理的效率和质量，增强企业在数字化时代的竞争力和创新能力。 + +## 前置条件 + +准备以下环境： +- TDengine 集群已部署并正常运行（企业及社区版均可） +- taosAdapter 能够正常运行。详细参考 [taosAdapter 使用手册](../../../reference/components/taosadapter) +- Apache Flink v1.19.0 或以上版本已安装。安装 Apache Flink 请参考 [官方文档](https://flink.apache.org/) + +# 支持的平台 + +Flink Connector 支持所有能运行 Flink 1.19 及以上版本的平台。 + +## 版本历史 +| Flink Connector 版本 | 主要变化 | TDengine 版本 | +| ------------------| ------------------------------------ | ---------------- | +| 2.0.0 | 1. 支持 SQL 查询 TDengine 数据库中的数据
2. 支持 CDC 订阅 TDengine 数据库中的数据
3. 支持 Table SQL 方式读取和写入 TDengine 数据库| 3.3.5.0 及以上版本 | +| 1.0.0 | 支持 Sink 功能，将来着其他数据源的数据写入到 TDengine| 3.3.2.0 及以上版本| + +## 异常和错误码 + +在任务执行失败后，查看 Flink 任务执行日志确认失败原因 + +具体的错误码请参考： + +| Error Code | Description | Suggested Actions | +| ---------------- |------------------------------------------------------- | -------------------- | +| 0xa000 |connection param error |连接器参数错误。 +| 0xa001 |the groupid parameter of CDC is incorrect |CDC 的 groupid 参数错误。| +| 0xa002 |wrong topic parameter for CDC |CDC 的 topic 参数错误。| +| 0xa010 |database name configuration error |数据库名配置错误。| +| 0xa011 |table name configuration error |表名配置错误。| +| 0xa012 |no data was obtained from the data source |从数据源中获取数据失败。| +| 0xa013 |value.deserializer parameter not set |未设置序列化方式。| +| 0xa014 |list of column names for target table not set |未设置目标表的列名列表。| +| 0x2301 |connection already closed |连接已经关闭，检查连接情况，或重新创建连接去执行相关指令。| +| 0x2302 |this operation is NOT supported currently! |当前使用接口不支持，可以更换其他连接方式。| +| 0x2303 |invalid variables |参数不合法，请检查相应接口规范，调整参数类型及大小。| +| 0x2304 |statement is closed |statement 已经关闭，请检查 statement 是否关闭后再次使用，或是连接是否正常。| +| 0x2305 |resultSet is closed |resultSet 结果集已经释放，请检查 resultSet 是否释放后再次使用。| +| 0x230d |parameter index out of range |参数越界，请检查参数的合理范围。| +| 0x230e |connection already closed |连接已经关闭，请检查 Connection 是否关闭后再次使用，或是连接是否正常。| +| 0x230f |unknown sql type in TDengine |请检查 TDengine 支持的 Data Type 类型。| +| 0x2315 |unknown taos type in TDengine |在 TDengine 数据类型与 JDBC 数据类型转换时，是否指定了正确的 TDengine 数据类型。| +| 0x2319 |user is required |创建连接时缺少用户名信息。| +| 0x231a |password is required |创建连接时缺少密码信息。| +| 0x231d |can't create connection with server within |通过增加参数 httpConnectTimeout 增加连接耗时，或是请检查与 taosAdapter 之间的连接情况。| +| 0x231e |failed to complete the task within the specified time |通过增加参数 messageWaitTimeout 增加执行耗时，或是请检查与 taosAdapter 之间的连接情况。| +| 0x2352 |Unsupported encoding |本地连接下指定了不支持的字符编码集。| +| 0x2353 |internal error of database, please see taoslog for more details |本地连接执行 prepareStatement 时出现错误，请检查 taos log 进行问题定位。| +| 0x2354 |connection is NULL |本地连接执行命令时，Connection 已经关闭。请检查与 TDengine 的连接情况。| +| 0x2355 |result set is NULL |本地连接获取结果集，结果集异常，请检查连接情况，并重试。| +| 0x2356 |invalid num of fields |本地连接获取结果集的 meta 信息不匹配。| +| 0x2357 |empty sql string |填写正确的 SQL 进行执行。| +| 0x2371 |consumer properties must not be null! |创建订阅时参数为空，请填写正确的参数。| +| 0x2375 |topic reference has been destroyed |创建数据订阅过程中，topic 引用被释放。请检查与 TDengine 的连接情况。| +| 0x2376 |failed to set consumer topic, topic name is empty |创建数据订阅过程中，订阅 topic 名称为空。请检查指定的 topic 名称是否填写正确。| +| 0x2377 |consumer reference has been destroyed |订阅数据传输通道已经关闭，请检查与 TDengine 的连接情况。| +| 0x2378 |consumer create error |创建数据订阅失败，请根据错误信息检查 taos log 进行问题定位。| +| 0x237a |vGroup not found in result set VGroup |没有分配给当前 consumer，由于 Rebalance 机制导致 Consumer 与 VGroup 不是绑定的关系。| + +## 数据类型映射 +TDengine 目前支持时间戳、数字、字符、布尔类型，与 Flink RowData Type 对应类型转换如下： + +| TDengine DataType | Flink RowDataType | +| ----------------- | ------------------ | +| TIMESTAMP | TimestampData | +| INT | Integer | +| BIGINT | Long | +| FLOAT | Float | +| DOUBLE | Double | +| SMALLINT | Short | +| TINYINT | Byte | +| BOOL | Boolean | +| BINARY | byte[] | +| NCHAR | StringData | +| JSON | StringData | +| VARBINARY | byte[] | +| GEOMETRY | byte[] | + +## 使用说明 + +### Flink 语义选择说明 + +采用 At-Least-Once（至少一次）语义原因： + - TDengine 目前不支持事务，不能进行频繁的检查点操作和复杂的事务协调。 + - 由于 TDengine 采用时间戳作为主键，重复数据下游算子可以进行过滤操作，避免重复计算。 + - 采用 At-Least-Once（至少一次）确保达到较高的数据处理的性能和较低的数据延时，设置方式如下： + +使用方式: + +```java +StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); +env.enableCheckpointing(5000); +env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.AT_LEAST_ONCE); +``` + +如果使用 Maven 管理项目，只需在 pom.xml 中加入以下依赖。 + +```xml + + com.taosdata.flink + flink-connector-tdengine + 2.0.0 + +``` + +### 连接参数 + +建立连接的参数有 URL 和 Properties。 +URL 规范格式为： +`jdbc:TAOS-WS://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&timezone={timezone}]` + +参数说明： + +- user：登录 TDengine 用户名，默认值 'root'。 +- password：用户登录密码，默认值 'taosdata'。 +- database_name: 数据库名称。 +- timezone: 时区设置。 +- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 60000。 +- messageWaitTimeout: 消息超时时间，单位 ms， 默认值为 60000。 +- useSSL: 连接中是否使用 SSL。 + +### Source + +Source 拉取 TDengine 数据库中的数据，并将获取到的数据转换为 Flink 内部可处理的格式和类型，并以并行的方式进行读取和分发，为后续的数据处理提供高效的输入。 +通过设置数据源的并行度，实现多个线程并行地从数据源中读取数据，提高数据读取的效率和吞吐量，充分利用集群资源进行大规模数据处理能力。 + +Properties 中配置参数如下： + +- TDengineConfigParams.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 'root'。 +- TDengineConfigParams.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 'taosdata'。 +- TDengineConfigParams.VALUE_DESERIALIZER：下游算子接收结果集反序列化方法, 如果接收结果集类型是 `Flink` 的 `RowData`，仅需要设置为 `RowData`即可。也可继承 `TDengineRecordDeserialization` 并实现 `convert` 和 `getProducedType` 方法，根据 `SQL` 的 `ResultSet` 自定义反序列化方式。 +- TDengineConfigParams.TD_BATCH_MODE：此参数用于批量将数据推送给下游算子，如果设置为 True，创建 `TDengineSource` 对象时需要指定数据类型为 `SourceRecords` 类型的泛型形式。 +- TDengineConfigParams.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 60000。 +- TDengineConfigParams.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。true: 启用，false: 不启用。默认为 false。 +- TDengineConfigParams.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。true: 启用，false: 不启用。默认为 false。 +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_INTERVAL_MS: 自动重连重试间隔，单位毫秒，默认值 2000。仅在 `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` 为 true 时生效。 +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_RETRY_COUNT: 自动重连重试次数，默认值 3，仅在 `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` 为 true 时生效。 +- TDengineConfigParams.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。true: 启用，false: 不启用。默认为 false。 + +#### 按时间分片 + +用户可以对查询的 SQL 按照时间拆分为多个子任务，输入：开始时间，结束时间，拆分间隔，时间字段名称，系统会按照设置的间隔（时间左闭右开）进行拆分并行获取数据。 + +```java +{{#include docs/examples/flink/Main.java:time_interval}} +``` + +#### 按超级表 TAG 分片 + +用户可以按照超级表的 TAG 字段将查询的 SQL 拆分为多个查询条件，系统会以一个查询条件对应一个子任务的方式对其进行拆分，进而并行获取数据。 + +```java +{{#include docs/examples/flink/Main.java:tag_split}} +``` + +#### 按表名分片 + +支持输入多个相同表结构的超级表或普通表进行分片，系统会按照一个表一个任务的方式进行拆分，进而并行获取数据。 + +```java +{{#include docs/examples/flink/Main.java:table_split}} +``` + +#### 使用 Source 连接器 + +查询结果为 RowData 数据类型示例： + +
+RowData Source +```java +{{#include docs/examples/flink/Main.java:source_test}} +``` +
+ +批量查询结果示例： + +
+Batch Source +```java +{{#include docs/examples/flink/Main.java:source_batch_test}} +``` +
+ +查询结果为自定义数据类型示例： + +
+Custom Type Source +```java +{{#include docs/examples/flink/Main.java:source_custom_type_test}} +``` +
+ +- ResultBean 自定义的一个内部类，用于定义 Source 查询结果的数据类型。 +- ResultSoureDeserialization 是自定义的一个内部类，通过继承 `TDengineRecordDeserialization` 并实现 `convert` 和 `getProducedType` 方法。 + +### CDC 数据订阅 + +Flink CDC 主要用于提供数据订阅功能，能实时监控 `TDengine` 数据库的数据变化，并将这些变更以数据流形式传输到 `Flink` 中进行处理，同时确保数据的一致性和完整性。 + +Properties 中配置参数如下： + +- TDengineCdcParams.BOOTSTRAP_SERVERS：TDengine 服务端所在的`ip:port`，如果使用 `WebSocket` 连接，则为 taosAdapter 所在的`ip:port`。 +- TDengineCdcParams.CONNECT_USER：登录 TDengine 用户名，默认值 'root'。 +- TDengineCdcParams.CONNECT_PASS：用户登录密码，默认值 'taosdata'。 +- TDengineCdcParams.POLL_INTERVAL_MS：拉取数据间隔, 默认 500ms。 +- TDengineCdcParams.VALUE_DESERIALIZER：结果集反序列化方法，如果接收结果集类型是 `Flink` 的 `RowData`，仅需要设置为 `RowData`即可。可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`，并指定结果集 bean，实现反序列化。 +- TDengineCdcParams.TMQ_BATCH_MODE：此参数用于批量将数据推送给下游算子，如果设置为 True，创建 `TDengineCdcSource` 对象时需要指定数据类型为 `ConsumerRecords` 类型的泛型形式。 +- TDengineCdcParams.GROUP_ID：消费组 ID，同一消费组共享消费进度。最大长度：192。 +- TDengineCdcParams.AUTO_OFFSET_RESET： 消费组订阅的初始位置 （ `earliest` 从头开始订阅, `latest` 仅从最新数据开始订阅, 默认 `latest`）。 +- TDengineCdcParams.ENABLE_AUTO_COMMIT：是否启用消费位点自动提交，true: 自动提交；false：依赖 `checkpoint` 时间来提交， 默认 false。 +> **注意**：自动提交模式reader获取完成数据后自动提交，不管下游算子是否正确的处理了数据，存在数据丢失的风险，主要用于为了追求高效的无状态算子场景或是数据一致性要求不高的场景。 + +- TDengineCdcParams.AUTO_COMMIT_INTERVAL_MS：消费记录自动提交消费位点时间间隔，单位为毫秒。默认值为 5000, 此参数在 `ENABLE_AUTO_COMMIT` 为 true 生效。 +- TDengineConfigParams.PROPERTY_KEY_ENABLE_COMPRESSION：传输过程是否启用压缩。true: 启用，false: 不启用。默认为 false。 +- TDengineConfigParams.PROPERTY_KEY_ENABLE_AUTO_RECONNECT：是否启用自动重连。true: 启用，false: 不启用。默认为 false。 +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_INTERVAL_MS：自动重连重试间隔，单位毫秒，默认值 2000。仅在 `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` 为 true 时生效。 +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_RETRY_COUNT：自动重连重试次数，默认值 3，仅在 `PROPERTY_KEY_ENABLE_AUTO_RECONNECT` 为 true 时生效。 +- TDengineCdcParams.TMQ_SESSION_TIMEOUT_MS：`consumer` 心跳丢失后超时时间，超时后会触发 `rebalance` 逻辑，成功后该 `consumer` 会被删除（从3.3.3.0版本开始支持）， 默认值为 12000，取值范围 [6000， 1800000]。 +- TDengineCdcParams.TMQ_MAX_POLL_INTERVAL_MS：`consumer poll` 拉取数据间隔的最长时间，超过该时间，会认为该 `consumer` 离线，触发 `rebalance` 逻辑，成功后该 `consumer` 会被删除。 默认值为 300000，[1000，INT32_MAX]。 + +#### 使用 CDC 连接器 + +CDC 连接器会根据用户设置的并行度进行创建 consumer，因此用户根据资源情况合理设置并行度。 + +订阅结果为 RowData 数据类型示例： + +
+CDC Source +```java +{{#include docs/examples/flink/Main.java:cdc_source}} +``` +
+ +将订阅结果批量下发到算子的示例： + +
+CDC Batch Source +```java +{{#include docs/examples/flink/Main.java:cdc_batch_source}} +``` +
+ +订阅结果为自定义数据类型示例： + +
+CDC Custom Type +```java +{{#include docs/examples/flink/Main.java:cdc_custom_type_test}} +``` +
+ +- ResultBean 是自定义的一个内部类，其字段名和数据类型与列的名称和数据类型一一对应，这样根据 `TDengineCdcParams.VALUE_DESERIALIZER` 属性对应的反序列化类可以反序列化出 ResultBean 类型的对象。 + +### Sink + +Sink 的核心功能在于高效且精准地将经过 `Flink` 处理的、源自不同数据源或算子的数据写入 `TDengine`。在这一过程中，`TDengine` 所具备的高效写入机制发挥了至关重要的作用，有力保障了数据的快速和稳定存储。 + +Properties 中配置参数如下： + +- TDengineConfigParams.PROPERTY_KEY_USER：登录 `TDengine` 用户名，默认值 'root'。 +- TDengineConfigParams.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 'taosdata'。 +- TDengineConfigParams.PROPERTY_KEY_DBNAME：写入的数据库名称。 +- TDengineConfigParams.TD_SUPERTABLE_NAME：写入的超级表名称。接收的数据必须有 tbname 字段，确定写入那张子表。 +- TDengineConfigParams.TD_TABLE_NAME：写入子表或普通表的表名，此参数和TD_SUPERTABLE_NAME 仅需要设置一个即可。 +- TDengineConfigParams.VALUE_DESERIALIZER：接收结果集反序列化方法, 如果接收结果集类型是 `Flink` 的 `RowData`，仅需要设置为 `RowData`即可。也可继承 `TDengineSinkRecordSerializer` 并实现 `serialize` 方法，根据 接收的数据类型自定义反序列化方式。 +- TDengineConfigParams.TD_BATCH_SIZE：设置一次写入 `TDengine` 数据库的批大小 | 当到达批的数量后进行写入，或是一个checkpoint的时间也会触发写入数据库。 +- TDengineConfigParams.TD_BATCH_MODE：接收批量数据当设置为 True 时，如果数据来源是 `TDengine Source`，则使用 `SourceRecords` 泛型类型来创建 `TDengineSink` 对象；若来源是 `TDengine CDC`，则使用 `ConsumerRecords` 泛型来创建 `TDengineSink` 对象。 +- TDengineConfigParams.TD_SOURCE_TYPE：设置数据来源。 当数据来源是 `TDengine Source` 是设置为 'tdengine_source', 当来源是 `TDengine CDC` 设置为 'tdengine_cdc'。当配置 `TD_BATCH_MODE` 为 True 生效。 +- TDengineConfigParams.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 60000。 +- TDengineConfigParams.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。true: 启用，false: 不启用。默认为 false。 +- TDengineConfigParams.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。true: 启用，false: 不启用。默认为 false。 +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_INTERVAL_MS: 自动重连重试间隔，单位毫秒，默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。 +- TDengineConfigParams.PROPERTY_KEY_RECONNECT_RETRY_COUNT: 自动重连重试次数，默认值 3，仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。 +- TDengineConfigParams.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。true: 启用，false: 不启用。默认为 false。 + +使用示例： + +将 power 库的 meters 表的子表数据，写入 power_sink 库的 sink_meters 超级表对应的子表中。 + +
+Sink RowData +```java +{{#include docs/examples/flink/Main.java:RowDataToSink}} +``` +
+ +使用示例： + +订阅 power 库的 meters 超级表的子表数据，写入 power_sink 库的 sink_meters 超级表对应的子表中。 + +
+Cdc Sink +```java +{{#include docs/examples/flink/Main.java:CdcRowDataToSink}} +``` +
+ +### Table SQL + +使用 Table SQL 的方式从多个不同的数据源数据库（如 TDengine、MySQL、Oracle 等）中提取数据后， 再进行自定义的算子操作（如数据清洗、格式转换、关联不同表的数据等），然后将处理后的结果加载到目标数据源（如 TDengine、Mysql 等）中。 + +#### Table Source 连接器 + +参数配置说明： + +| 参数名称 | 类型 | 参数说明 | +| ----------------------- | :-----: | ------------ | +| connector | string | 连接器标识，设置 `tdengine-connector` 。| +| td.jdbc.url| string | 连接的 url 。| +| td.jdbc.mode | strng | 连接器类型, 设置 `source`, `sink`。| +| table.name| string| 原表或目标表名称。| +| scan.query| string| 获取数据的 SQL 语句。| +| sink.db.name|string| 目标数据库名称。| +| sink.supertable.name|string |写入的超级表名称。| +| sink.batch.size | integer | 写入的批大小。| +| sink.table.name|string|写入的普通表或子表名称。| + + +使用示例： + +将 power 库的 meters 表的子表数据，写入 power_sink 库的 sink_meters 超级表对应的子表中。 + +
+Table Source +```java +{{#include docs/examples/flink/Main.java:source_table}} +``` +
+ +#### Table CDC 连接器 + +参数配置说明： + +| 参数名称 | 类型 | 参数说明 | +| ----------------------- | :-----: | ------------ | +| connector | string | 连接器标识，设置 `tdengine-connector`。| +| user| string | 用户名， 默认 root。| +| password | string | 密码， 默认taosdata。| +| bootstrap.servers| string | 服务器地址。| +| topic | string | 订阅主题。|| +| td.jdbc.mode | strng | 连接器类型, cdc, sink。| +| group.id| string| 消费组 ID，同一消费组共享消费进度。 | +| auto.offset.reset| string| 消费组订阅的初始位置。
`earliest`: 从头开始订阅
`latest`: 仅从最新数据开始订阅。
默认 `latest`。| +| poll.interval_ms| integer| 拉取数据间隔, 默认 500ms。| +| sink.db.name|string| 目标数据库名称。| +| sink.supertable.name|string |写入的超级表名称。| +| sink.batch.size | integer | 写入的批大小。| +| sink.table.name|string|写入的普通表或子表名称。| + +使用示例： + +订阅 power 库的 meters 超级表的子表数据，写入 power_sink 库的 sink_meters 超级表对应的子表中。 + +
+Table CDC +```java +{{#include docs/examples/flink/Main.java:cdc_table}} +``` +
diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 0a4160a00b..ffc4d8b3c1 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -185,7 +185,8 @@ charset 的有效值是 UTF-8。 |参数名称|支持版本|动态修改|参数含义| |--------------------------|----------|-------------------------|-| |supportVnodes | |支持动态修改 立即生效 |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5| -|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 0-1024，默认值为 4| +|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 1-1024，默认值为 4| +|numOfCompactThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 1-16，默认值为 2| |numOfMnodeReadThreads | |支持动态修改 重启生效 |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| |numOfVnodeQueryThreads | |支持动态修改 重启生效 |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| |numOfVnodeFetchThreads | |支持动态修改 重启生效 |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| @@ -255,7 +256,7 @@ charset 的有效值是 UTF-8。 |slowLogMaxLen |3.3.3.0 后|支持动态修改 立即生效 |慢查询日志最大长度，取值范围 1-16384，默认值 4096| |slowLogScope |3.3.3.0 后|支持动态修改 立即生效 |慢查询记录类型，取值范围 ALL/QUERY/INSERT/OTHERS/NONE，默认值 QUERY| |slowLogExceptDb |3.3.3.0 后|支持动态修改 立即生效 |指定的数据库不上报慢查询，仅支持配置换一个数据库| -|debugFlag | |支持动态修改 立即生效 |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| +|debugFlag | |支持动态修改 立即生效 |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块），该参数的设置会影响所有模块的开关，后设置的参数起效| |tmrDebugFlag | |支持动态修改 立即生效 |定时器模块的日志开关，取值范围同上| |uDebugFlag | |支持动态修改 立即生效 |共用功能模块的日志开关，取值范围同上| |rpcDebugFlag | |支持动态修改 立即生效 |rpc 模块的日志开关，取值范围同上| diff --git a/docs/zh/14-reference/03-taos-sql/19-limit.md b/docs/zh/14-reference/03-taos-sql/19-limit.md index 4e2fa69a14..e5c03db2fd 100644 --- a/docs/zh/14-reference/03-taos-sql/19-limit.md +++ b/docs/zh/14-reference/03-taos-sql/19-limit.md @@ -37,6 +37,6 @@ description: 合法字符集和命名中的限制规则 - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 - 数据库的副本数只能设置为 1 或 3 - 用户名的最大长度是 23 字节 -- 用户密码的最大长度是 31 字节 +- 用户密码的长度范围是 8-16 字节 - 总数据行数取决于可用资源 - 单个数据库的虚拟结点数上限为 1024 diff --git a/docs/zh/14-reference/05-connector/14-java.mdx b/docs/zh/14-reference/05-connector/14-java.mdx index 27f43676f3..7d5096bb66 100644 --- a/docs/zh/14-reference/05-connector/14-java.mdx +++ b/docs/zh/14-reference/05-connector/14-java.mdx @@ -33,6 +33,8 @@ TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致 | taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 | | ------------------| ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| 3.5.1 | 解决了数据订阅获取时间戳对象类型问题 | - | +| 3.5.0 | 1. 优化了 WebSocket 连接参数绑定性能，支持参数绑定查询使用二进制数据
2. 优化了 WebSocket 连接在小查询上的性能
3. WebSocket 连接上支持设置时区和应用信息 | 3.3.5.0 及更高版本 | | 3.4.0 | 1. 使用 jackson 库替换 fastjson 库
2. WebSocket 采用独立协议标识
3. 优化后台拉取线程使用，避免用户误用导致超时 | - | | 3.3.4 | 解决了 getInt 在数据类型为 float 报错 | - | | 3.3.3 | 解决了 WebSocket statement 关闭导致的内存泄漏 | - | @@ -243,13 +245,13 @@ TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可 对于 WebSocket 连接，url 中的配置参数如下： - user：登录 TDengine 用户名，默认值 'root'。 - password：用户登录密码，默认值 'taosdata'。 -- charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 - httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 60000。 - messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 60000。 - useSSL: 连接中是否使用 SSL。 +- timezone：客户端使用的时区，连接上生效，默认值为系统时区。推荐不设置，使用系统时区性能更好。 -**注意**：部分配置项（比如：locale、timezone）在 WebSocket 连接中不生效。 +**注意**：部分配置项（比如：locale、charset）在 WebSocket 连接中不生效。 **REST 连接** 使用 JDBC REST 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要： @@ -261,14 +263,13 @@ TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可 对于 REST 连接，url 中的配置参数如下： - user：登录 TDengine 用户名，默认值 'root'。 - password：用户登录密码，默认值 'taosdata'。 -- charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 - httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 60000。 - httpSocketTimeout: socket 超时时间，单位 ms，默认值为 60000。 - useSSL: 连接中是否使用 SSL。 - httpPoolSize: REST 并发请求大小，默认 20。 -**注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 +**注意**：部分配置项（比如：locale、charset 和 timezone）在 REST 连接中不生效。 :::note @@ -291,7 +292,9 @@ properties 中的配置参数如下： - TSDBDriver.PROPERTY_KEY_CONFIG_DIR：仅在使用 JDBC 原生连接时生效。客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。 - TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。 - TSDBDriver.PROPERTY_KEY_LOCALE：仅在使用 JDBC 原生连接时生效。 客户端语言环境，默认值系统当前 locale。 -- TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。因为历史的原因，我们只支持POSIX标准的部分规范，如UTC-8(代表中国上上海), GMT-8，Asia/Shanghai 这几种形式。 +- TSDBDriver.PROPERTY_KEY_TIME_ZONE： + - 原生连接：客户端使用的时区，默认值为系统当前时区，全局生效。因为历史的原因，我们只支持POSIX标准的部分规范，如UTC-8(代表中国上上海), GMT-8，Asia/Shanghai 这几种形式。 + - WebSocket 连接：客户端使用的时区，连接上生效，默认值为系统时区。仅支持 IANA 时区，即 Asia/Shanghai 这种形式。推荐不设置，使用系统时区性能更好。 - TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 60000。仅在 REST 连接时生效。 - TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 60000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 60000。 仅 WebSocket 连接下有效。 @@ -299,12 +302,15 @@ properties 中的配置参数如下： - TSDBDriver.HTTP_POOL_SIZE: REST 并发请求大小，默认 20。 - TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: 传输过程是否启用压缩。仅在使用 REST/WebSocket 连接时生效。true: 启用，false: 不启用。默认为 false。 - TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: 是否启用自动重连。仅在使用 WebSocket 连接时生效。true: 启用，false: 不启用。默认为 false。 -> **注意**：启用自动重连仅对简单执行 SQL 语句以及 无模式写入、数据订阅有效。对于参数绑定无效。自动重连仅对连接建立时通过参数指定数据库有效，对后面的 `use db` 语句切换数据库无效。 + > **注意**：启用自动重连仅对简单执行 SQL 语句以及 无模式写入、数据订阅有效。对于参数绑定无效。自动重连仅对连接建立时通过参数指定数据库有效，对后面的 `use db` 语句切换数据库无效。 - TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS: 自动重连重试间隔，单位毫秒，默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。 - TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT: 自动重连重试次数，默认值 3，仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。 - TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: 关闭 SSL 证书验证 。仅在使用 WebSocket 连接时生效。true: 启用，false: 不启用。默认为 false。 +- TSDBDriver.PROPERTY_KEY_APP_NAME: App 名称，可用于 `show connections` 查询结果显示。仅在使用 WebSocket 连接时生效。默认值为 java。 +- TSDBDriver.PROPERTY_KEY_APP_IP: App IP，可用于 `show connections` 查询结果显示。仅在使用 WebSocket 连接时生效。默认值为空。 + 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。 **配置参数的优先级** diff --git a/docs/zh/14-reference/05-connector/20-go.mdx b/docs/zh/14-reference/05-connector/20-go.mdx index 85c65a5fb8..210b0e438d 100644 --- a/docs/zh/14-reference/05-connector/20-go.mdx +++ b/docs/zh/14-reference/05-connector/20-go.mdx @@ -23,24 +23,25 @@ import RequestId from "./_request_id.mdx"; ## 版本历史 -| driver-go 版本 | 主要变化 | TDengine 版本 | -|-------------|-------------------------------------|---------------| -| v3.5.8 | 修复空指针异常 | - | -| v3.5.7 | taosWS 和 taosRestful 支持传入 request id | - | -| v3.5.6 | 提升 websocket 查询和写入性能 | 3.3.2.0 及更高版本 | -| v3.5.5 | restful 支持跳过 ssl 证书检查 | - | -| v3.5.4 | 兼容 TDengine 3.3.0.0 tmq raw data | - | -| v3.5.3 | 重构 taosWS | - | -| v3.5.2 | websocket 压缩和优化消息订阅性能 | 3.2.3.0 及更高版本 | -| v3.5.1 | 原生 stmt 查询和 geometry 类型支持 | 3.2.1.0 及更高版本 | -| v3.5.0 | 获取消费进度及按照指定进度开始消费 | 3.0.5.0 及更高版本 | -| v3.3.1 | 基于 websocket 的 schemaless 协议写入 | 3.0.4.1 及更高版本 | -| v3.1.0 | 提供贴近 kafka 的订阅 api | - | -| v3.0.4 | 新增 request id 相关接口 | 3.0.2.2 及更高版本 | -| v3.0.3 | 基于 websocket 的 statement 写入 | - | -| v3.0.2 | 基于 websocket 的数据查询和写入 | 3.0.1.5 及更高版本 | -| v3.0.1 | 基于 websocket 的消息订阅 | - | -| v3.0.0 | 适配 TDengine 3.0 查询和写入 | 3.0.0.0 及更高版本 | +| driver-go 版本 | 主要变化 | TDengine 版本 | +|--------------|--------------------------------------------|---------------| +| v3.6.0 | stmt2 原生接口，DSN 支持密码包含特殊字符（url.QueryEscape） | 3.3.5.0 及更高版本 | +| v3.5.8 | 修复空指针异常 | - | +| v3.5.7 | taosWS 和 taosRestful 支持传入 request id | - | +| v3.5.6 | 提升 websocket 查询和写入性能 | 3.3.2.0 及更高版本 | +| v3.5.5 | restful 支持跳过 ssl 证书检查 | - | +| v3.5.4 | 兼容 TDengine 3.3.0.0 tmq raw data | - | +| v3.5.3 | 重构 taosWS | - | +| v3.5.2 | websocket 压缩和优化消息订阅性能 | 3.2.3.0 及更高版本 | +| v3.5.1 | 原生 stmt 查询和 geometry 类型支持 | 3.2.1.0 及更高版本 | +| v3.5.0 | 获取消费进度及按照指定进度开始消费 | 3.0.5.0 及更高版本 | +| v3.3.1 | 基于 websocket 的 schemaless 协议写入 | 3.0.4.1 及更高版本 | +| v3.1.0 | 提供贴近 kafka 的订阅 api | - | +| v3.0.4 | 新增 request id 相关接口 | 3.0.2.2 及更高版本 | +| v3.0.3 | 基于 websocket 的 statement 写入 | - | +| v3.0.2 | 基于 websocket 的数据查询和写入 | 3.0.1.5 及更高版本 | +| v3.0.1 | 基于 websocket 的消息订阅 | - | +| v3.0.0 | 适配 TDengine 3.0 查询和写入 | 3.0.0.0 及更高版本 | ## 异常和错误码 @@ -137,6 +138,8 @@ WKB规范请参考[Well-Known Binary (WKB)](https://libgeos.org/specifications/w username:password@protocol(address)/dbname?param=value ``` +当密码中包含特殊字符时，需要使用 `url.QueryEscape` 进行转义。 + ##### 原生连接 导入驱动： @@ -494,6 +497,37 @@ Prepare 允许使用预编译的 SQL 语句，可以提高性能并提供参数 - **接口说明**：关闭语句。 - **返回值**：错误信息。 +从 3.6.0 版本开始，提供 stmt2 绑定参数的接口 + +- `func (conn *Connector) Stmt2(reqID int64, singleTableBindOnce bool) *Stmt2` + - **接口说明**：从连接创建 stmt2。 + - **参数说明**： + - `reqID`：请求 ID。 + - `singleTableBindOnce`：单个子表在单次执行中只有一次数据绑定。 + - **返回值**：stmt2 对象。 +- `func (s *Stmt2) Prepare(sql string) error` + - **接口说明**：绑定 sql 语句。 + - **参数说明**： + - `sql`：要绑定的 sql 语句。 + - **返回值**：错误信息。 +- `func (s *Stmt2) Bind(params []*stmt.TaosStmt2BindData) error` + - **接口说明**：绑定数据。 + - **参数说明**： + - params要绑定的数据。 + - **返回值**：错误信息。 +- `func (s *Stmt2) Execute() error` + - **接口说明**：执行语句。 + - **返回值**：错误信息。 +- `func (s *Stmt2) GetAffectedRows() int` + - **接口说明**：获取受影响行数（只在插入语句有效）。 + - **返回值**：受影响行数。 +- `func (s *Stmt2) UseResult() (driver.Rows, error)` + - **接口说明**：获取结果集（只在查询语句有效）。 + - **返回值**：结果集 Rows 对象，错误信息。 +- `func (s *Stmt2) Close() error` + - **接口说明**：关闭stmt2。 + - **返回值**：错误信息。 + `ws/stmt` 包提供了通过 WebSocket 进行参数绑定的接口 - `func (c *Connector) Init() (*Stmt, error)` diff --git a/docs/zh/14-reference/05-connector/30-python.mdx b/docs/zh/14-reference/05-connector/30-python.mdx index d724fc796c..42ed67e927 100644 --- a/docs/zh/14-reference/05-connector/30-python.mdx +++ b/docs/zh/14-reference/05-connector/30-python.mdx @@ -52,7 +52,9 @@ Python Connector 历史版本（建议使用最新版本的 `taospy`）： | Python Connector 版本 | 主要变化 | TDengine 版本 | | -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | -| 2.7.18 | 支持 Apache Superset BI 产品 | - | +| 2.7.21 | Native 支持 STMT2 写入 | - | +| 2.7.19 | 支持 Apache Superset 连接 TDengine Cloud 数据源 | - | +| 2.7.18 | 支持 Apache Superset 产品连接本地 TDengine 数据源 | - | | 2.7.16 | 新增订阅配置 (session.timeout.ms, max.poll.interval.ms) | - | | 2.7.15 | 新增 VARBINARY 和 GEOMETRY 类型支持 | - | | 2.7.14 | 修复已知问题 | - | @@ -131,7 +133,8 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Python 对 | [json_tag.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/json-tag.py) | 使用 JSON 类型的标签 | | [tmq_consumer.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq_consumer.py) | tmq 订阅 | | [native_all_type_query.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/native_all_type_query.py) | 支持全部类型示例 | -| [native_all_type_stmt.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/native_all_type_stmt.py) | 参数绑定支持全部类型示例 | +| [native_all_type_stmt.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/native_all_type_stmt.py) | 参数绑定 stmt 全部类型示例 | +| [test_stmt2.py](https://github.com/taosdata/taos-connector-python/blob/main/tests/test_stmt2.py) | 参数绑定 stmt2 写入示例 | 示例程序源码请参考： @@ -284,6 +287,7 @@ TaosResult 对象可以通过循环遍历获取查询到的数据。 - `fn close(&self) -> PyResult<()>` - **接口说明**： 关闭 stmt 对象。 + #### 数据订阅 - **创建消费者支持属性列表**： - host：主机地址。 @@ -424,50 +428,40 @@ TaosResult 对象可以通过循环遍历获取查询到的数据。 - **异常**：操作失败抛出 `SchemalessError` 异常。 #### 参数绑定 -- `def statement(self, sql=None)` - - **接口说明**：使用连接对象创建 stmt 对象, 如果 sql 不空会进行调用 prepare。 - - `sql`: 预编译的 SQL 语句。 - - **返回值**：stmt 对象。 - - **异常**：操作失败抛出 `StatementError` 异常。 +- `def statement2(self, sql=None, option=None)` + - **接口说明**：使用连接对象创建 stmt2 对象 + - **参数说明** + - `sql`: 绑定的 SQL 语句，如果不为空会调用`prepare`函数 + - `option` 传入 TaosStmt2Option 类实例选项 + - **返回值**：stmt2 对象。 + - **异常**：操作失败抛出 `ConnectionError` 异常。 - `def prepare(self, sql)` - - **接口说明**：绑定预编译 sql 语句。 + - **接口说明**：绑定预编译 sql 语句 - **参数说明**： - - `sql`: 预编译的 SQL 语句。 + - `sql`: 绑定的 SQL 语句 - **异常**：操作失败抛出 `StatementError` 异常。 -- `def set_tbname(self, name)` - - **接口说明**：设置将要写入数据的表名。 +- `def bind_param(self, tbnames, tags, datas)` + - **接口说明**：以独立数组方式绑定数据 - **参数说明**： - - `name`: 表名，如果需要指定数据库, 例如： `db_name.table_name` 即可。 - - **异常**：操作失败抛出 `StatementError` 异常。 -- `def set_tbname_tags(self, name, tags):` - - **接口说明**：设置表和 Tags 数据, 用于自动建表。 + - `tbnames`: 绑定表名数组，数据类型为 list + - `tags`: 绑定 tag 列值数组，数据类型为 list + - `tags`: 绑定普通列值数组，数据类型为 list + - **异常**：操作失败抛出 `StatementError` 异常 +- `def bind_param_with_tables(self, tables)` + - **接口说明**：以独立表方式绑定数据，独立表是以表为组织单位，每张表中有表名，TAG 值及普通列数值属性 - **参数说明**： - - `name`: 表名，如果需要指定数据库, 例如： `db_name.table_name` 即可。 - - `tags`: Tags 数据。 + - `tables`: `BindTable` 独立表对象数组 - **异常**：操作失败抛出 `StatementError` 异常。 -- `def bind_param(self, params, add_batch=True)` - - **接口说明**：绑定一组数据并提交。 - - **参数说明**： - - `params`: 绑定数据。 - - `add_batch`: 是否提交绑定数据。 - - **异常**：操作失败抛出 `StatementError` 异常。 -- `def bind_param_batch(self, binds, add_batch=True)` - - **接口说明**：绑定多组数据并提交。 - - **参数说明**： - - `binds`: 绑定数据。 - - `add_batch`: 是否提交绑定数据。 - - **异常**：操作失败抛出 `StatementError` 异常。 -- `def add_batch(self)` - - **接口说明**：提交绑定数据。 - - **异常**：操作失败抛出 `StatementError` 异常。 -- `def execute(self)` - - **接口说明**：执行将绑定的数据全部写入。 - - **异常**：操作失败抛出 `StatementError` 异常。 -- `def affected_rows(self)` - - **接口说明**： 获取写入条数。 - - **返回值**：写入条数。 -- `def close(&self)` - - **接口说明**： 关闭 stmt 对象。 +- `def execute(self) -> int:` + - **接口说明**：执行将绑定数据全部写入 + - **返回值**：影响行数 + - **异常**：操作失败抛出 `QueryError` 异常。 +- `def result(self)` + - **接口说明**： 获取参数绑定查询结果集 + - **返回值**：返回 TaosResult 对象 +- `def close(self)` + - **接口说明**： 关闭 stmt2 对象 + #### 数据订阅 - **创建消费者支持属性列表**： diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 8b4cabf520..24254c8807 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -302,3 +302,13 @@ TDinsight插件中展示的数据是通过taosKeeper和taosAdapter服务收集 问题影响：服务器重启后，原有数据库丢失（注：并非真正丢失，只是原有的数据磁盘未挂载，暂时看不到）且集群 ID 发生变化，导致无法访问原有数据库。对于企业版用户，如果已针对集群 ID 进行授权，还会发现集群服务器的机器码未变，但原有的授权已失效。如果未针对该问题进行监控或者未及时发现并进行处理，则用户不会注意到原有数据库已经丢失，从而造成损失，增加运维成本。 问题解决：应在 fstab 文件中配置 dataDir 目录的自动挂载，确保 dataDir 始终指向预期的挂载点和目录，此时，再重启服务器，会找回原有的数据库和集群。在后续的版本中，我们将开发一个功能，使 taosd 在检测到启动前后 dataDir 发生变化时，在启动阶段退出，同时提供相应的错误提示。 + +### 33 Windows 平台运行 TDengine 出现丢失 MVCP1400.DLL 解决方法？ +1. 重新安装 Microsoft Visual C++ Redistributable‌：由于 msvcp140.dll 是 Microsoft Visual C++ Redistributable 的一部分，重新安装这个包通常可以解决大部分问题。可以从 Microsoft 官方网站下载相应的版本进行安装‌ +2. 手动上网下载并替换 msvcp140.dll 文件‌：可以从可靠的源下载 msvcp140.dll 文件，并将其复制到系统的相应目录下。确保下载的文件与您的系统架构（32位或64位）相匹配，并确保来源的安全性‌ + +### 34 超级表带 TAG 过滤查子查数据与直接查子表哪个块？ +直接查子表更快。超级表带 TAG 过滤查询子查数据是为满足查询方便性，同时可对多个子表中数据进行过滤，如果目的是追求性能并已明确查询子表，直接从子表查性能更高 + +### 35 如何查看数据压缩率指标？ +TDengine 目前只提供以表为统计单位的压缩率，数据库及整体还未提供，查看命令是在客户端 taos-CLI 中执行 `SHOW TABLE DISTRIBUTED table_name;` 命令，table_name 为要查看压缩率的表，可以为超级表、普通表及子表，详细可 [查看此处](https://docs.taosdata.com/reference/taos-sql/show/#show-table-distributed) \ No newline at end of file diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 0800b3ea50..6beb7c8860 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -112,9 +112,8 @@ extern int32_t tsNumOfSnodeWriteThreads; extern int64_t tsQueueMemoryAllowed; extern int32_t tsRetentionSpeedLimitMB; -extern const char *tsAlterCompactTaskKeywords; -extern int32_t tsNumOfCompactThreads; -extern int32_t tsNumOfRetentionThreads; +extern int32_t tsNumOfCompactThreads; +extern int32_t tsNumOfRetentionThreads; // sync raft extern int32_t tsElectInterval; diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index a2be706dc0..7a4401827c 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -389,7 +389,6 @@ typedef struct SStateStore { int32_t (*streamStateFillGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); - void (*streamStateSetFillInfo)(SStreamState* pState); void (*streamStateClearExpiredState)(SStreamState* pState); int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, @@ -455,7 +454,6 @@ typedef struct SStateStore { int32_t (*streamStateBegin)(SStreamState* pState); void (*streamStateCommit)(SStreamState* pState); void (*streamStateDestroy)(SStreamState* pState, bool remove); - int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst); } SStateStore; diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 8eb30b8184..2d329ec879 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -112,7 +112,7 @@ typedef struct SDatabaseOptions { int8_t s3Compact; int8_t withArbitrator; // for auto-compact - int8_t compactTimeOffset; // hours + int32_t compactTimeOffset; // hours int32_t compactInterval; // minutes int32_t compactStartTime; // minutes int32_t compactEndTime; // minutes diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 2179547352..b4e0087b1a 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -34,7 +34,6 @@ void streamStateClose(SStreamState* pState, bool remove); int32_t streamStateBegin(SStreamState* pState); void streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); -int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); int32_t streamStateDelTaskDb(SStreamState* pState); int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); @@ -108,7 +107,6 @@ int32_t streamStateFillGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, con int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); // twa -void streamStateSetFillInfo(SStreamState* pState); void streamStateClearExpiredState(SStreamState* pState); void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index f1f5b00e38..f47c308e18 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -67,7 +67,6 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState); void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); -int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState); void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts); diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index a6a3c63a50..709d053414 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -148,7 +148,7 @@ int32_t tfsMkdirRecur(STfs *pTfs, const char *rname); * @return int32_t 0 for success, -1 for failure. */ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId); - +#if 0 /** * @brief check directories exist in tfs. * @@ -158,7 +158,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId); * @return true for exist, false for not exist. */ bool tfsDirExistAt(STfs *pTfs, const char *rname, SDiskID diskId); - +#endif /** * @brief Remove directory at all levels in tfs. * @@ -241,7 +241,7 @@ void tfsBasename(const STfsFile *pFile, char *dest); * @param dest The buffer where dirname will be saved. */ void tfsDirname(const STfsFile *pFile, char *dest); - +#if 0 /** * @brief Get the absolute file name of rname. * @@ -251,7 +251,7 @@ void tfsDirname(const STfsFile *pFile, char *dest); * @param aname absolute file name */ void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname); - +#endif /** * @brief Remove file in tfs. * diff --git a/packaging/setup_env.sh b/packaging/setup_env.sh new file mode 100644 index 0000000000..e8b69c964e --- /dev/null +++ b/packaging/setup_env.sh @@ -0,0 +1,2211 @@ +#!/bin/bash + +# Run cleanup function on exit +trap cleanup EXIT + +# define default timezone +DEFAULT_TIMEZONE="Asia/Shanghai" + +# Define default DNS server +DEFAULT_DNS="192.168.1.252" + +# Define the packages to be installed +SYSTEM_APT_TOOLS="git wget vim gdb screen tmux ntp tree atop iotop sysstat fio tcpdump iperf3 qemu-guest-agent dstat linux-tools-common linux-tools-generic jq zip unzip cloud-guest-utils nfs-kernel-server nfs-common" +SYSTEM_YUM_TOOLS="git wget vim gdb screen tmux ntp tree atop iotop sysstat fio tcpdump iperf3 qemu-guest-agent dstat jq zip unzip cloud-utils-growpart python3-devel nfs-utils rpm-build automake autoconf libevent-devel ncurses-devel" + +# Define the packages to be installed for build TDinternal +BUILD_APT_TOOLS="llvm gcc make cmake perl g++ lzma curl locales psmisc libgeos-dev libgoogle-glog-dev valgrind rsync libjemalloc-dev openssh-server sshpass net-tools dirmngr gnupg apt-transport-https \ + ca-certificates software-properties-common r-base iputils-ping build-essential git libssl-dev libgflags2.2 libgflags-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config" +BUILD_YUM_TOOLS="gcc make cmake3 perl gcc-c++ xz curl psmisc geos glog valgrind rsync jemalloc openssh-server sshpass net-tools gnupg2 libarchive snappy-devel pkgconfig libatomic perl-IPC-Cmd libcurl-devel libxml2-devel\ + ca-certificates libicu-devel R-core iputils bison flex glibc-static libstdc++-static libstdc++-devel openssl-devel gflags jansson jansson-devel snappy xz-devel zlib-devel zlib bzip2-devel zlib-static libs3" + +# Define the packages to be installed via pip +PIP_PKGS="wheel setuptools-rust pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog taospy numpy poetry" + +# Gcc version to be updated +GCC_VERSION="9.5.0" + +# Define the version of the Ubuntu release +# Define jdk version to be installed +if [ -f /etc/debian_version ]; then + DIST_VERSION=$(lsb_release -sr) + JDK_VERSION="openjdk-17-jdk" +elif [ -f /etc/redhat-release ]; then + DIST_VERSION=$(grep -oP '\d+\.\d+' < /etc/redhat-release) + JDK_VERSION="java-1.8.0-openjdk" +else + echo "Unsupported Linux distribution." + exit 1 +fi + +# Define the path where the core dump files should be stored +COREPATH="/corefile" + +# Define the path where the repository should be cloned +REPOPATH="$HOME/repos" + +# Define the path to the script directory +SCRIPT_DIR=$(dirname "$(realpath "$0")") + +# Define the path to the .bashrc file +BASH_RC=$HOME/.bashrc + +# Define the path to the Cargo configuration file +CARGO_CONFIG_FILE=$HOME/.cargo/config.toml + +# Define jmeter version to be installed +JMETER_VERSION="5.6.3" + +# Define the path where the Prometheus binary should exist +PROMETHEUS_BINARY="/usr/local/bin/prometheus" + +# Define the path where the Node Exporter binary should exist +NODE_EXPORTER_BINARY="/usr/local/bin/node_exporter" + +# Define the path where the Process Exporter binary should exist +PROCESS_EXPORTER_BINARY="/usr/local/bin/process-exporter" + +# Define fstab input +FSTAB_LINE="share-server.platform.tdengine.dev:/mnt/share_server /mnt/share_server nfs rw,sync,_netdev 0 0" + +# Results need to be stored when source +SOURCE_RESULTS="" + +# ANSI color codes +GREEN='\033[0;32m' # Green color +RED='\033[0;31m' # Red color +NO_COLOR='\033[0m' # Reset to default color +YELLOW='\033[0;33m' # Yellow color + +# read -r -d '' CLOUD_INIT_CONFIG << 'EOF' +# datasource: +# NoCloud: +# seedfrom: /var/lib/cloud/seed/nocloud/ +# meta-data: {} +# user-data: {} +# vendor-data: {} +# ConfigDrive: {} +# None: {} +# datasource_list: [ NoCloud, ConfigDrive, None ] +# EOF + +read -r -d '' CUSTOM_SETTINGS <<'EOF' +export LC_CTYPE="en_US.UTF-8" +export LANG="en_US.UTF-8" +export HISTTIMEFORMAT="%d/%m/%y %T " +parse_git_branch() { + git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/(\1)/' +} +export PS1="\u@\h \[\e[32m\]\w \[\e[91m\]\$(parse_git_branch)\[\e[00m\]$ " +EOF + +read -r -d '' CARGO_CONFIG <<'EOF' +[source.crates-io] +replace-with = 'rsproxy-sparse' +[source.rsproxy] +registry = "https://rsproxy.cn/crates.io-index" +[source.rsproxy-sparse] +registry = "sparse+https://rsproxy.cn/index/" +[registries.rsproxy] +index = "https://rsproxy.cn/crates.io-index" +[net] +git-fetch-with-cli = true +[source.tuna] +registry = "https://mirrors.tuna.tsinghua.edu.cn/git/crates.io-index.git" +[source.ustc] +registry = "git://mirrors.ustc.edu.cn/crates.io-index" +[source.sjtu] +registry = "https://mirrors.sjtug.sjtu.edu.cn/git/crates.io-index" +[source.rustcc] +registry = "git://crates.rustcc.cn/crates.io-index" +EOF + +# Help function to display usage information +help() { + echo "Usage: $0 [option]" + echo "Options:" + echo " --help - Display this help and exit" + echo " setup_all - Setup all configurations and installations" + echo " TDasset - Prepare TDasset env" + echo " TDinternal - Prepare TDinternal env" + echo " TDgpt - Prepare TDgpt env" + echo " taostest - Prepare taostest env" + echo " system_config - Perform system configuration" + echo " deploy_pure - Deploy Pure environment" + echo " deploy_dev - Deploy development environment" + echo " deploy_cmake - Deploy CMake" + echo " update_redhat_gcc - Update GCC on Red Hat or CentOS" + echo " update_redhat_tmux - Update tmux on Red Hat or CentOS" + echo " deploy_tmux - Deploy tmux" + echo " config_ssh - Configure SSH settings" + echo " disable_firewalld - Disable firewalld" + echo " config_cloud_init - Set cloud initialization parameters" + echo " deploy_git - Deploy git repositories" + echo " replace_sources - Replace package sources" + echo " update - Update the system" + echo " upgrade - Upgrade the system" + echo " config_timezone - Configure the system timezone" + echo " config_dns - Set DNS configurations" + echo " config_custom_settings - Add custom settings to your shell configuration" + echo " config_share_server - Configure share server" + echo " install_packages - Install specified packages" + echo " config_system_limits - Configure system limits and kernel parameters" + echo " config_coredump - Configure core dump settings" + echo " disable_service - Disable specified services" + echo " install_python - Install Python and pip" + echo " install_pyenv - Install Pyenv" + echo " install_python_via_pyenv - Install Python via pyenv" + echo " install_java - Install Java" + echo " install_java_via_sdkman - Install Java via sdkman" + echo " install_maven_via_sdkman - Install Maven via sdkman" + echo " deploy_go - Deploy Go environment" + echo " install_gvm - Install GVM" + echo " install_go_via_gvm - Install Go via GVM" + echo " deploy_rust - Deploy Rust environment" + echo " install_node - Install Node via package manager or binary" + echo " install_node_via_nvm - Install Node via NVM" + echo " install_pnpm - Install PNPM, node version >=v18.12.00 required" + echo " deploy_node_exporter - Deploy Node Exporter for Prometheus" + echo " deploy_process_exporter - Deploy Process Exporter" + echo " deploy_prometheus - Deploy Prometheus" + echo " deploy_grafana - Deploy Grafana" + echo " deploy_jmeter - Deploy JMeter" + echo " install_nginx - Install NGINX" + echo " config_qemu_guest_agent - Configure QEMU guest agent" + echo " deploy_docker - Deploy Docker" + echo " deploy_docker_compose - Deploy Docker Compose" + echo " clone_enterprise - Clone the enterprise repository" + echo " clone_community - Clone the community repository" + echo " clone_taosx - Clone TaosX repository" + echo " clone_taoskeeper - Clone TaosKeeper repository" + echo " clone_taostest - Clone TaosTest repository" + echo " clone_operation - Clone operation tools repository" +} + +replace_apt_sources() { + # Define the codename of the Ubuntu release + local CODENAME + CODENAME=$(lsb_release -sc) + if grep -q "mirrors.aliyun.com" /etc/apt/sources.list; then + echo "The Aliyun mirror is already set." + else + echo "Backing up the original sources.list..." + cp /etc/apt/sources.list /etc/apt/sources.list.bak + + echo "Replacing sources.list with the Aliyun mirror..." + tee /etc/apt/sources.list << EOF +deb http://mirrors.aliyun.com/ubuntu/ $CODENAME main restricted universe multiverse +deb http://mirrors.aliyun.com/ubuntu/ $CODENAME-security main restricted universe multiverse +deb http://mirrors.aliyun.com/ubuntu/ $CODENAME-updates main restricted universe multiverse +deb http://mirrors.aliyun.com/ubuntu/ $CODENAME-proposed main restricted universe multiverse +deb http://mirrors.aliyun.com/ubuntu/ $CODENAME-backports main restricted universe multiverse +deb-src http://mirrors.aliyun.com/ubuntu/ $CODENAME main restricted universe multiverse +deb-src http://mirrors.aliyun.com/ubuntu/ $CODENAME-security main restricted universe multiverse +deb-src http://mirrors.aliyun.com/ubuntu/ $CODENAME-updates main restricted universe multiverse +deb-src http://mirrors.aliyun.com/ubuntu/ $CODENAME-proposed main restricted universe multiverse +deb-src http://mirrors.aliyun.com/ubuntu/ $CODENAME-backports main restricted universe multiverse +EOF + fi + echo "Updating repositories..." + apt-get update -y + echo "The sources have been replaced and updated successfully." +} + +replace_yum_sources() { + if grep -q "mirrors.aliyun.com" /etc/yum.repos.d/CentOS-Base.repo; then + echo "The Aliyun mirror is already set." + else + echo "Backing up the original CentOS-Base.repo..." + cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.bak + + echo "Replacing CentOS-Base.repo with the Aliyun mirror..." + tee /etc/yum.repos.d/CentOS-Base.repo << 'EOF' +[base] +name=CentOS-$releasever - Base - Aliyun +baseurl=http://mirrors.aliyun.com/centos/$releasever/os/$basearch/ +gpgcheck=1 +gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 + +#released updates +[updates] +name=CentOS-$releasever - Updates - Aliyun +baseurl=http://mirrors.aliyun.com/centos/$releasever/updates/$basearch/ +gpgcheck=1 +gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 + +#additional packages that may be useful +[extras] +name=CentOS-$releasever - Extras - Aliyun +baseurl=http://mirrors.aliyun.com/centos/$releasever/extras/$basearch/ +gpgcheck=1 +gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 +EOF + fi + echo "Updating repositories..." + yum makecache fast + yum install epel-release -y + yum update -y + + echo "The sources have been replaced and updated successfully." +} + +replace_sources() { + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + echo "Replacing sources for APT package manager." + replace_apt_sources + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + echo "Replacing sources for YUM package manager." + replace_yum_sources + else + echo "Unsupported Linux distribution." + exit 1 + fi +} + +update() { + echo "Updating ..." + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + echo "Using APT package manager." + apt update -y + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + echo "Using YUM package manager." + yum install epel-release -y + yum update -y + else + echo "Unsupported Linux distribution." + exit 1 + fi +} + +upgrade() { + echo "Upgrading ..." + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + echo "Using APT package manager." + apt upgrade -y + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + echo "Using YUM package manager." + yum upgrade -y + else + echo "Unsupported Linux distribution." + exit 1 + fi +} + +config_frontend() { + # only ubuntu need + if [ -f /etc/debian_version ]; then + echo "Configuring frontend..." + add_config_if_not_exist "export DEBIAN_FRONTEND=noninteractive" "$BASH_RC" + fi + update + upgrade + systemctl restart dbus.service network-dispatcher.service + +} + +# Adds a configuration to a file if it does not already exist +add_config_if_not_exist() { + local config="$1" + local file="$2" + grep -qF -- "$config" "$file" || echo "$config" >> "$file" +} + +# General error handling function +check_status() { + local message_on_failure="$1" + local message_on_success="$2" + local exit_code="$3" + + if [ "${exit_code:-0}" -ne 0 ]; then + echo -e "${RED}${message_on_failure}${NO_COLOR}" + exit 1 + else + echo -e "${GREEN}${message_on_success}${NO_COLOR}" + fi +} + +# Config Share-NFS server +config_share_server() { + echo "Configuring share server..." + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + echo "Using APT package manager." + install_package "nfs-common" + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + echo "Using YUM package manager." + install_package "nfs-utils" + else + echo "Unsupported Linux distribution." + exit 1 + fi + mkdir -p /mnt/share_server + add_config_if_not_exist "$FSTAB_LINE" /etc/fstab + mount -a + check_status "Failed to configure share server" "Share server configured successfully." $? +} + +# Init environment +init_env() { + export DEBIAN_FRONTEND=noninteractive + export LC_CTYPE="en_US.UTF-8" +} + +# Install packages +install_packages() { + echo "Installing $package..." + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + echo "Using APT package manager." + install_package $SYSTEM_APT_TOOLS + install_package $BUILD_APT_TOOLS + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + echo "Using YUM package manager." + yum install epel-release -y + install_package $SYSTEM_YUM_TOOLS + install_package $BUILD_YUM_TOOLS + else + echo "Unsupported Linux distribution." + exit 1 + fi +} + +# Install package +install_package() { + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + echo "Using APT package manager." + install_via_apt "$@" + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + echo "Using YUM package manager." + install_via_yum "$@" + else + echo "Unsupported Linux distribution." + exit 1 + fi +} + +# Install package via apt +install_via_apt() { + echo -e "${YELLOW}Installing packages: $*...${NO_COLOR}" + if DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" "$@"; then + echo -e "${GREEN}Installed packages successfully.${NO_COLOR}" + else + echo -e "${RED}Failed to install packages.${NO_COLOR}" + return 1 + fi +} + +# Install package via yum +install_via_yum() { + echo -e "${YELLOW}Installing packages: $*...${NO_COLOR}" + if yum install -y "$@"; then + echo -e "${GREEN}Installed packages successfully.${NO_COLOR}" + else + echo -e "${RED}Failed to install packages.${NO_COLOR}" + return 1 + fi +} + +# disable and stop firewalld +disable_firewalld() { + if [ -f /etc/debian_version ]; then + # Only Debian or Ubuntu + echo "ufw disable..." + ufw disable + check_status "Failed to disable ufw" "Ufw disabled successfully." $? + elif [ -f /etc/redhat-release ]; then + # Only Red Hat or CentOS + echo "Disabling firewalld..." + systemctl stop firewalld + systemctl disable firewalld + check_status "Failed to disable firewalld" "Firewalld disabled successfully." $? + else + echo "Unsupported Linux distribution." + exit 1 + fi +} + +# Modifies SSH configuration and sets the root password +config_ssh() { + echo "Configuring SSH settings..." + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + + echo "Restarting SSH..." + if [ "$DIST_VERSION" = "24.04" ]; then + systemctl restart ssh + else + systemctl restart sshd + fi + check_status "Failed to restart SSH" "SSH restarted successfully." $? +} + +# Sets the timezone +config_timezone() { + echo "Setting timezone to $DEFAULT_TIMEZONE..." + timedatectl set-timezone "$DEFAULT_TIMEZONE" + check_status "Failed to set timezone" "Timezone set to $DEFAULT_TIMEZONE successfully." $? +} + +# Disables service +disable_service() { + if [ -f /etc/debian_version ]; then + # Only Debian or Ubuntu + echo "Stop and disable and related services..." + systemctl stop apt-daily.service apt-daily-upgrade.service apt-daily-upgrade.timer apt-daily.timer unattended-upgrades + systemctl disable apt-daily.service apt-daily-upgrade.service apt-daily-upgrade.timer apt-daily.timer unattended-upgrades + check_status "Failed to disable related services" "Related services disabled successfully." $? + fi +} + +# Config dns for Red Hat or CentOS +config_redhat_dns() { + local DEFAULT_DNS="192.168.2.99" + echo "Configuring DNS settings to use $INTERNAL_DNS and $DEFAULT_DNS..." + echo "nameserver $INTERNAL_DNS" > /etc/resolv.conf + echo "nameserver $DEFAULT_DNS" >> /etc/resolv.conf + check_status "Failed to configure DNS" "DNS configured to use $INTERNAL_DNS and $DEFAULT_DNS successfully." $? +} + +# Config dns for Debian or Ubuntu +config_debian_dns() { + local DEFAULT_DNS="192.168.2.99" + echo "Configuring DNS settings to use $INTERNAL_DNS and $DEFAULT_DNS..." + systemctl stop systemd-resolved.service + echo "[Resolve]" > /etc/systemd/resolved.conf + echo "DNS=$INTERNAL_DNS" >> /etc/systemd/resolved.conf + echo "DNS=$DEFAULT_DNS" >> /etc/systemd/resolved.conf + systemctl restart systemd-resolved.service + ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf + check_status "Failed to configure DNS" "DNS configured to use $INTERNAL_DNS and $DEFAULT_DNS successfully." $? +} + +# Config DNS settings +config_dns() { + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + echo "Configuring DNS settings for Debian or Ubuntu..." + config_debian_dns + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + echo "Configuring DNS settings for Red Hat or CentOS..." + config_redhat_dns + else + echo "Unsupported Linux distribution." + exit 1 + fi +} + +# Config qemu-guest-agent +config_qemu_guest_agent() { + install_package "qemu-guest-agent" + echo "Configuring qemu-guest-agent..." + systemctl enable qemu-guest-agent + systemctl start qemu-guest-agent + check_status "Failed to configure qemu-guest-agent" "Qemu-guest-agent configured successfully." $? +} + +# Config custom settings +config_custom_settings() { + echo "Configuring custom settings..." + marker="parse_git_branch" + + if grep -qF "$marker" "$BASH_RC"; then + echo "Configuration already exists in ""$BASH_RC""." + else + echo "Adding configuration to ""$BASH_RC""." + echo "$CUSTOM_SETTINGS" >>"$BASH_RC" + echo "Custom settings have been updated in your $BASH_RC file." + fi + check_status "Failed to apply custom settings" "Custom settings configured successfully." $? +} + +# Config core dump settings +config_coredump() { + echo "Configuring core dump directory..." + mkdir -p $COREPATH + add_config_if_not_exist "kernel.core_pattern=$COREPATH/core_%e-%p" /etc/sysctl.conf + add_config_if_not_exist "ulimit -n 600000" "$BASH_RC" + add_config_if_not_exist "ulimit -c unlimited" "$BASH_RC" + sysctl -p + check_status "Failed to apply core dump settings" "Core path:$COREPATH applied successfully." $? +} + +# Modifies system resource limits and TCP/IP core parameters +config_system_limits() { + echo "Configuring system limits and kernel parameters..." + local sysctl_conf="/etc/sysctl.conf" + local limits_conf="/etc/security/limits.conf" + + add_config_if_not_exist "fs.nr_open = 1048576" $sysctl_conf + add_config_if_not_exist "net.core.somaxconn=10240" $sysctl_conf + add_config_if_not_exist "net.core.netdev_max_backlog=20480" $sysctl_conf + add_config_if_not_exist "net.ipv4.tcp_max_syn_backlog=10240" $sysctl_conf + add_config_if_not_exist "net.ipv4.tcp_retries2=5" $sysctl_conf + add_config_if_not_exist "net.ipv4.tcp_syn_retries=2" $sysctl_conf + add_config_if_not_exist "net.ipv4.tcp_synack_retries=2" $sysctl_conf + add_config_if_not_exist "net.ipv4.tcp_tw_reuse=1" $sysctl_conf + add_config_if_not_exist "net.ipv4.tcp_keepalive_time=600" $sysctl_conf + add_config_if_not_exist "net.ipv4.tcp_abort_on_overflow=1" $sysctl_conf + add_config_if_not_exist "net.ipv4.tcp_max_tw_buckets=5000" $sysctl_conf + + sysctl -p + check_status "Failed to apply sysctl settings" "Apply sysctl settings successfully." $? + + for limit in "soft nproc 65536" "soft nofile 65536" "soft stack 65536" "hard nproc 65536" "hard nofile 65536" "hard stack 65536"; do + add_config_if_not_exist "* $limit" $limits_conf + add_config_if_not_exist "root $limit" $limits_conf + done + check_status "Failed to apply limits settings" "Apply limits settings successfully." $? +} + +# Check the operating system version +centos_skip_check() { + # Check if the operating system is CentOS 7 + if [[ -f /etc/redhat-release ]]; then + if grep -q "CentOS Linux release 7" /etc/redhat-release; then + echo "This platform requires you to manually upgrade gcc and glibc." + exit 1 + fi + fi +} + +# Other logic can go here... + +# Deploy cmake +deploy_cmake() { + # Check if cmake is installed + if command -v cmake >/dev/null 2>&1; then + echo "Cmake is already installed. Skipping installation." + cmake --version + return + fi + install_package "cmake3" + ln -sf /usr/bin/cmake3 /usr/bin/cmake + check_status "Failed to install cmake" "Install cmake successfully" $? +} + + +# install pkg via pip +install_pip_pkg() { + if [ "$DIST_VERSION" != "24.04" ]; then + echo "Installing $PIP_PKGS ..." + pip3 install --upgrade pip setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple + install_via_pip "$PIP_PKGS" + fi +} + +install_via_pip() { + echo "pip install $*..." + if pip3 install $* -i https://pypi.tuna.tsinghua.edu.cn/simple; then + echo "pip install packages successfully." + else + echo "Failed to install packages." + return 1 + fi +} + + +# Complie python +download_and_compile_python() { + if [ -f /etc/debian_version ]; then + install_package gcc make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev + elif [ -f /etc/redhat-release ]; then + # install_package gcc patch libffi libffi-devel python-devel zlib zlib-devel bzip2-devel openssl-devel openssl11 openssl11-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel + install_package gcc zlib zlib-devel libffi libffi-devel readline-devel openssl-devel openssl11 openssl11-devel + # CFLAGS=$(pkg-config --cflags openssl11) + # export CFLAGS + # LDFLAGS=$(pkg-config --libs openssl11) + # export LDFLAGS + export CFLAGS=$(pkg-config --cflags openssl11) + export LDFLAGS=$(pkg-config --libs openssl11) + else + echo "Unsupported Linux distribution." + exit 1 + fi + + local VERSION="$1" + local DOWNLOAD_URL="https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tgz" + + + echo "Downloading Python $VERSION from $DOWNLOAD_URL..." + wget "$DOWNLOAD_URL" -O "/tmp/Python-$VERSION.tgz" + check_status "Failed to download Python source." "Python source downloaded successfully." $? + + echo "Extracting Python $VERSION..." + cd /tmp || exit + tar -xzf "Python-$VERSION.tgz" + cd "Python-$VERSION" || exit + + echo "Compiling and installing Python $VERSION..." + ./configure --enable-optimizations --prefix=/usr/local/python"$VERSION" + make -j"$(nproc)" + make altinstall + + cd .. + rm -rf "Python-$VERSION" "Python-$VERSION.tgz" + + MAJOR_MINOR_VERSION=$(echo "$VERSION" | cut -d '.' -f 1-2) + + # files=( + # "/usr/bin/python3" + # "/usr/bin/python" + # "/usr/bin/pip3" + # "/usr/bin/pip" + # ) + + # for file in "${files[@]}"; do + # if [ -e "$file" ]; then + # backup_file="$file.bak" + # echo "Backing up $file to $backup_file" + # cp "$file" "$backup_file" + # else + # echo "$file does not exist, skipping." + # fi + # done + + ln -sf /usr/local/python"$VERSION"/bin/python"$MAJOR_MINOR_VERSION" /usr/local/bin/python3 + ln -sf /usr/local/python"$VERSION"/bin/pip"$MAJOR_MINOR_VERSION" /usr/local/bin/pip3 + python3 --version + check_status "Failed to install Python $VERSION" "Python $VERSION installed successfully." $? +} + +upgrage_pip() { + echo "Upgrading pip..." + python3 -m pip install --upgrade pip + check_status "Failed to upgrade pip" "Pip upgraded successfully." $? +} + +# Install Python via package_manager +install_python_via_package_manager() { + if [ -n "$1" ]; then + PYTHON_PACKAGE="$1" + else + PYTHON_PACKAGE="python3" + fi + install_package "$PYTHON_PACKAGE" + install_package "python3-pip" + upgrage_pip + python3 --version +} + +# Install Python and pip +# shellcheck disable=SC2120 +install_python() { + echo -e "${YELLOW}Installing Python...${NO_COLOR}" + # Specify the major python version to search for; default is set to 3.10 if not specified + if [ -n "$1" ]; then + PYTHON_VERSION="$1" + else + install_python_via_package_manager "$PYTHON_PACKAGE" + exit 0 + fi + MAJOR_MINOR_VERSION=$(echo "$PYTHON_VERSION" | cut -d '.' -f 1-2) + # Check if the JDK package is available in the repository + if [ -f /etc/debian_version ]; then + PYTHON_PACKAGE="python${MAJOR_MINOR_VERSION}" + if apt-cache search "$PYTHON_PACKAGE" | grep -w "$PYTHON_PACKAGE"; then + install_python_via_package_manager "$PYTHON_PACKAGE" + exit 0 + else + echo -e "${RED}Failed to install Python using package manager.${NO_COLOR}" + fi + elif [ -f /etc/redhat-release ]; then + PYTHON_PACKAGE="python${MAJOR_MINOR_VERSION//./}" + if yum list available | grep -w "$PYTHON_PACKAGE"; then + install_python_via_package_manager "$PYTHON_PACKAGE" + exit 0 + else + echo -e "${RED}Failed to install Python using package manager.${NO_COLOR}" + fi + else + echo "Unsupported Linux distribution." + exit 1 + fi + + echo -e "${YELLOW}$PYTHON_PACKAGE not found in source repository. Attempting to download and install manually....${NO_COLOR}" + download_and_compile_python "$PYTHON_VERSION" + upgrage_pip + + # Check installation status + INSTALLED_VERSION=$(python3 --version 2>&1) + if echo "$INSTALLED_VERSION" | grep -q "$MAJOR_MINOR_VERSION"; then + echo -e "${GREEN}Python $MAJOR_MINOR_VERSION installed successfully.${NO_COLOR}" + else + echo -e "${YELLOW}Python version not match.${NO_COLOR}" + exit 1 + fi +} + +update_redhat_gcc() { + echo "Updating the system..." + update + + echo "Installing dependencies..." + yum groupinstall -y "Development Tools" + install_package gmp-devel mpfr-devel libmpc-devel wget + + echo "Downloading GCC $GCC_VERSION..." + cd /usr/local/src || exit + wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz + wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz.sig + + echo "Extracting GCC $GCC_VERSION..." + tar -xzf gcc-$GCC_VERSION.tar.gz + cd gcc-$GCC_VERSION || exit + + echo "Downloading necessary dependencies for GCC..." + ./contrib/download_prerequisites + + mkdir build + cd build || exit + + echo "Configuring GCC..." + ../configure --enable-languages=c,c++ --disable-multilib --prefix=/usr + + echo "Compiling GCC, this may take a while..." + make -j"$(nproc)" + make install + + echo "Cleaning up downloaded files..." + cd /usr/local/src || exit + rm -rf gcc-$GCC_VERSION gcc-$GCC_VERSION.tar.gz gcc-$GCC_VERSION.tar.gz.sig + echo "Cleanup completed." + + echo "GCC installation completed. Verifying installation..." + gcc --version + check_status "Failed to install GCC" "GCC $GCC_VERSION installed successfully." $? + + # Backup + if [ -f "/lib64/libstdc++.so.6.0.28-gdb.py" ]; then + # Copy the file + mv -f /lib64/libstdc++.so.6.0.28-gdb.py /tmp/libstdc++.so.6.0.28-gdb.py + echo "File has been successfully moved to /tmp/libstdc++.so.6.0.28-gdb.py" + else + echo "File /lib64/libstdc++.so.6.0.28-gdb.py does not exist, cannot perform copy operation." + fi +} + +update_redhat_tmux() { + echo "Downloading the latest version of tmux..." + cd /usr/local/src || exit + latest_tmux_version=$(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s https://api.github.com/repos/tmux/tmux/releases/latest | grep -Po '"tag_name": "\K.*?(?=")') + wget https://github.com/tmux/tmux/releases/download/"${latest_tmux_version}"/tmux-"${latest_tmux_version}".tar.gz + + echo "Extracting tmux ${latest_tmux_version}..." + tar -xzf tmux-"${latest_tmux_version}".tar.gz + cd tmux-"${latest_tmux_version}" || exit + + echo "Configuring tmux..." + ./configure --prefix=/usr + + echo "Compiling tmux, this may take a while..." + make -j"$(nproc)" + make install + + echo "Cleaning up downloaded files..." + cd /usr/local/src || exit + rm -rf tmux-"${latest_tmux_version}" tmux-"${latest_tmux_version}".tar.gz + + echo "Cleanup completed." + + echo "tmux installation completed. Verifying installation..." + tmux -V + check_status "Failed to install tmux" "tmux ${latest_tmux_version} installed successfully." $? +} + +deploy_tmux() { + if [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + update_redhat_tmux + fi + echo "Copying configuration file..." + + cp "$SCRIPT_DIR/../conf/tmux.conf" ~/.tmux.conf + + echo "Configuration file copied to ~/.tmux.conf." +} + + +# Install Java +# install_java() { +# if command -v java >/dev/null 2>&1; then +# echo "Java is already installed. Skipping installation." +# java -version +# return +# else +# echo "Installing $JDK_VERSION..." +# install_package "$JDK_VERSION" +# check_status "Failed to install Java" "Install Java successfully" $? +# fi +# } + +# Install Java +# shellcheck disable=SC2120 +install_java() { + echo -e "${YELLOW}Installing Java...${NO_COLOR}" + # Specify the major JDK version to search for; default is set to 17 if not specified + if [ -n "$1" ]; then + DEFAULT_JDK_VERSION="$1" + else + DEFAULT_JDK_VERSION="17" + fi + + # Check if the JDK package is available in the repository + if [ -f /etc/debian_version ]; then + JDK_PACKAGE="openjdk-$DEFAULT_JDK_VERSION-jdk" + if apt-cache search "$JDK_PACKAGE" | grep -q "$JDK_PACKAGE"; then + echo "Installing $JDK_PACKAGE using apt..." + fi + elif [ -f /etc/redhat-release ]; then + JDK_PACKAGE="java-$DEFAULT_JDK_VERSION-openjdk" + if yum list available | grep -q "$JDK_PACKAGE"; then + echo "Installing $JDK_PACKAGE using yum..." + fi + else + echo "Unsupported Linux distribution." + exit 1 + fi + + # Check installation status + if ! install_package "$JDK_PACKAGE"; then + echo -e "${RED}Failed to install Java using package manager.${NO_COLOR}" + else + echo -e "${GREEN}Java installed successfully.${NO_COLOR}" + java -version + return + fi + + echo -e "${YELLOW}$JDK_PACKAGE not found in $PACKAGE_MANAGER repository. Attempting to download and install manually....${NO_COLOR}" + + # URL of the archive page to search + ARCHIVE_URL="https://jdk.java.net/archive/" + + # Get the latest minor version number + LATEST_VERSION=$(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s "$ARCHIVE_URL" | \ + grep -o "jdk${DEFAULT_JDK_VERSION}\.[0-9]\+\.[0-9]\+" | \ + sort -V | \ + tail -n 1) + JDK_VERSION_NUM="${LATEST_VERSION#jdk}" + # Confirm the latest version found + if [[ $LATEST_VERSION =~ jdk([0-9]+)\.([0-9]+)\.([0-9]+) ]]; then + # Print the latest version found + echo -e "${YELLOW}Latest JDK version found: $LATEST_VERSION${NO_COLOR}" + MATCH_URL="https://download.java.net/java/GA/${LATEST_VERSION}/[^ ]*linux-x64_bin.tar.gz" + JDK_DOWNLOAD_URL=$(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s "$ARCHIVE_URL" | \ + grep -o "$MATCH_URL" | \ + head -n 1) + else + echo -e "${RED}Failed to find the JDK version: $LATEST_VERSION.${NO_COLOR}" + exit 1 + fi + # Download JDK + echo "Downloading OpenJDK $LATEST_VERSION from $JDK_DOWNLOAD_URL..." + wget "$JDK_DOWNLOAD_URL" -O /tmp/"${LATEST_VERSION}"_linux-x64_bin.tar.gz + check_status "Failed to download OpenJDK." "OpenJDK downloaded successfully." $? + + # Extract and install + echo "Extracting OpenJDK..." + if [ -d "/usr/local/jdk-${JDK_VERSION_NUM}" ]; then + rm -rf "/usr/local/jdk-${JDK_VERSION_NUM}" + fi + tar -xzf /tmp/"${LATEST_VERSION}"_linux-x64_bin.tar.gz -C /usr/local/ + rm -rf /tmp/"${LATEST_VERSION}"_linux-x64_bin.tar.gz + # Configure environment variables + echo "Configuring environment variables..." + add_config_if_not_exist "export JAVA_HOME=/usr/local/jdk-${JDK_VERSION_NUM}" "$BASH_RC" + add_config_if_not_exist "export PATH=\$PATH:\$JAVA_HOME/bin" "$BASH_RC" + # shellcheck source=/dev/null + export JAVA_HOME=/usr/local/jdk-${JDK_VERSION_NUM} + export PATH=$PATH:$JAVA_HOME/bin + INSTALLED_VERSION=$("$JAVA_HOME"/bin/java --version 2>&1) + if echo "$INSTALLED_VERSION" | grep -q "openjdk $DEFAULT_JDK_VERSION"; then + echo -e "${GREEN}Java installed successfully.${NO_COLOR}" + SOURCE_RESULTS+="source /root/.bashrc # For openjdk\n" + else + echo -e "${YELLOW}Java version not match.${NO_COLOR}" + exit 1 + fi +} + +# Install sdkman +install_sdkman() { + echo -e "${YELLOW}Installing SDKMAN...${NO_COLOR}" + if [ -d "$HOME/.sdkman" ]; then + echo -e "${GREEN}SDKMAN is already installed.${NO_COLOR}" + else + echo -e "${YELLOW}Installing SDKMAN...${NO_COLOR}" + install_package zip unzip + curl --retry 10 --retry-delay 5 --retry-max-time 120 -s "https://get.sdkman.io" | bash + fi +} + +# Install gvm +install_gvm() { + echo -e "${YELLOW}Installing GVM...${NO_COLOR}" + if [ -d "$HOME/.gvm" ]; then + echo -e "${GREEN}GVM is already installed.${NO_COLOR}" + else + install_package bison gcc make + bash < <(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer) + source $HOME/.gvm/scripts/gvm + gvm version + check_status "Failed to install GVM" "GVM installed successfully." $? + add_config_if_not_exist "export GO111MODULE=on" "$BASH_RC" + add_config_if_not_exist "export GOPROXY=https://goproxy.cn,direct" "$BASH_RC" + add_config_if_not_exist "export GO_BINARY_BASE_URL=https://mirrors.aliyun.com/golang/" "$BASH_RC" + add_config_if_not_exist "export GOROOT_BOOTSTRAP=$GOROOT" "$BASH_RC" + fi + SOURCE_RESULTS+="source $HOME/.gvm/scripts/gvm # For gvm\n" +} + +# enable pyenv +enable_pyenv() { + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init --path)" + eval "$(pyenv init -)" +} + +# Install pyenv +install_pyenv() { + echo -e "${YELLOW}Installing Pyenv...${NO_COLOR}" + if [ -d "$HOME/.pyenv" ]; then + echo -e "${GREEN}Pyenv is already installed.${NO_COLOR}" + else + curl -L https://gitee.com/xinghuipeng/pyenv-installer/raw/master/bin/pyenv-installer | bash + enable_pyenv + add_config_if_not_exist "export PATH=\"\$HOME/.pyenv/bin:\$PATH\"" "$BASH_RC" + add_config_if_not_exist "eval \"\$(pyenv init --path)\"" "$BASH_RC" + add_config_if_not_exist "eval \"\$(pyenv init -)\"" "$BASH_RC" + pyenv --version + check_status "Failed to install Pyenv" "Pyenv installed successfully." $? + fi + SOURCE_RESULTS+="source $BASH_RC For: pyenv/python\n" +} + +# Install python via pyenv +install_python_via_pyenv() { + echo -e "${YELLOW}Installing Python via Pyenv...${NO_COLOR}" + if [ -f /etc/debian_version ]; then + install_package gcc make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev + elif [ -f /etc/redhat-release ]; then + install_package gcc zlib zlib-devel libffi libffi-devel readline-devel openssl-devel openssl11 openssl11-devel + else + echo "Unsupported Linux distribution." + exit 1 + fi + + if [ -n "$1" ]; then + DEFAULT_PYTHON_VERSION="$1" + else + DEFAULT_PYTHON_VERSION="3.10.12" + fi + install_pyenv + enable_pyenv + pyenv install "$DEFAULT_PYTHON_VERSION" + pyenv global "$DEFAULT_PYTHON_VERSION" + python --version + check_status "Failed to install Python" "Python installed successfully." $? +} + +# Install Maven +# shellcheck disable=SC2120 +install_maven_via_sdkman() { + echo -e "${YELLOW}Installing maven...${NO_COLOR}" + if [ -n "$1" ]; then + DEFAULT_MVN_VERSION="$1" + install_sdkman + [[ -s "$HOME/.sdkman/bin/sdkman-init.sh" ]] && source "$HOME/.sdkman/bin/sdkman-init.sh" + # 3.2.5 + yes | sdk install maven "$DEFAULT_MVN_VERSION" + else + install_package "maven" + fi + [[ -s "$HOME/.sdkman/bin/sdkman-init.sh" ]] && source "$HOME/.sdkman/bin/sdkman-init.sh" + mvn -version + check_status "Failed to install maven" "Maven installed successfully." $? +} + +install_java_via_sdkman() { + echo -e "${YELLOW}Installing java...${NO_COLOR}" + if [ -n "$1" ]; then + DEFAULT_JDK_VERSION="$1" + else + DEFAULT_JDK_VERSION="17" + fi + install_sdkman + [[ -s "$HOME/.sdkman/bin/sdkman-init.sh" ]] && source "$HOME/.sdkman/bin/sdkman-init.sh" + yes | sdk install java "$DEFAULT_JDK_VERSION-open" + [[ -s "$HOME/.sdkman/bin/sdkman-init.sh" ]] && source "$HOME/.sdkman/bin/sdkman-init.sh" + java -version + check_status "Failed to install java" "Java installed successfully." $? + SOURCE_RESULTS+="source $HOME/.sdkman/bin/sdkman-init.sh # For sdkman/java/maven\n" +} + +# Install Go +deploy_go() { + # Define the installation location for Go + GO_INSTALL_DIR="/usr/local/go" + GOPATH_DIR="/root/go" + + # Check if Go is installed + if command -v go >/dev/null 2>&1; then + echo "Go is already installed. Skipping installation." + return + fi + + # Fetch the latest version number of Go + GO_LATEST_DATA=$(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s https://golang.google.cn/VERSION?m=text) + GO_LATEST_VERSION=$(echo "$GO_LATEST_DATA" | grep -oP 'go[0-9]+\.[0-9]+\.[0-9]+') + # Download and install the latest version of Go + echo "Installing $GO_LATEST_VERSION..." + wget https://golang.google.cn/dl/$GO_LATEST_VERSION.linux-amd64.tar.gz -O go.tar.gz + + # Extract to the specified directory + tar -C /usr/local -xzf go.tar.gz + rm -rf go.tar.gz + + # Configure environment variables using the helper function + add_config_if_not_exist "export GOROOT=$GO_INSTALL_DIR" "$BASH_RC" + add_config_if_not_exist "export GOPATH=$GOPATH_DIR" "$BASH_RC" + add_config_if_not_exist "export PATH=\$PATH:\$GOROOT/bin" "$BASH_RC" + add_config_if_not_exist "export GO111MODULE=on" "$BASH_RC" + add_config_if_not_exist "export GOPROXY=https://goproxy.cn,direct" "$BASH_RC" + + # Apply the environment variables + $GO_INSTALL_DIR/bin/go version + check_status "Failed to install GO" "Install GO successfully" $? + SOURCE_RESULTS+="source $BASH_RC # For golang\n" +} + +# Install Go via gvm +install_go_via_gvm() { + echo -e "${YELLOW}Installing Go...${NO_COLOR}" + if [ -n "$1" ]; then + DEFAULT_GO_VERSION="$1" + else + DEFAULT_GO_VERSION="1.23.0" + fi + install_gvm + source $HOME/.gvm/scripts/gvm + export GO111MODULE=on + export GOPROXY=https://goproxy.cn,direct + export GO_BINARY_BASE_URL=https://mirrors.aliyun.com/golang/ + export GOROOT_BOOTSTRAP=$GOROOT + + gvm install go"$DEFAULT_GO_VERSION" -B + gvm use go"$DEFAULT_GO_VERSION" + gvm use go"$DEFAULT_GO_VERSION" --default + + go version + check_status "Failed to install Go" "Go installed successfully." $? + SOURCE_RESULTS+="source $BASH_RC # For golang\n" +} + +# Function to install Rust and Cargo +deploy_rust() { + # Check if Rust is already installed + if ! command -v rustc &> /dev/null; then + # add_config_if_not_exist "export RUSTUP_DIST_SERVER=http://mirrors.ustc.edu.cn/rust-static" $BASH_RC + # export RUSTUP_DIST_SERVER=http://mirrors.ustc.edu.cn/rust-static + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + echo "Using APT package manager." + install_package build-essential + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + echo "Using YUM package manager." + yum groupinstall -y "Development Tools" + else + echo "Unsupported Linux distribution." + exit 1 + fi + + add_config_if_not_exist "export RUSTUP_DIST_SERVER=\"https://rsproxy.cn\"" "$BASH_RC" + add_config_if_not_exist "export RUSTUP_UPDATE_ROOT=\"https://rsproxy.cn/rustup\"" "$BASH_RC" + export RUSTUP_DIST_SERVER="https://rsproxy.cn" + export RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" + + echo "Rust is not installed. Installing Rust and Cargo..." + # Download and install Rust and Cargo + curl --retry 10 --retry-delay 5 --retry-max-time 120 --proto '=https' --tlsv1.2 -sSf https://rsproxy.cn/rustup-init.sh | sh -s -- -y + + echo "Cargo settings..." + marker="git-fetch-with-cli" + + if grep -qF "$marker" "$CARGO_CONFIG_FILE"; then + echo "Configuration already exists in ""$CARGO_CONFIG_FILE""." + else + echo "Adding configuration to ""$CARGO_CONFIG_FILE""." + echo "$CARGO_CONFIG" >>"$CARGO_CONFIG_FILE" + echo "Cargo config have been updated in your $CARGO_CONFIG_FILE file." + fi + + # Source the Cargo environment script to update the current shell + if [ -f "$HOME/.cargo/env" ]; then + source "$HOME/.cargo/env" + fi + # Check if the installation was successful + rustc --version + # Install cargo-make + cargo install cargo-make + check_status "Failed to install Rust" "Install Rust successfully" $? + SOURCE_RESULTS+="source $BASH_RC && source $HOME/.cargo/env # For cargo/rust\n" + else + echo "Rust is already installed." + fi +} + +# Update GCC for Ubuntu 18.04 +update_ubuntu_gcc_18.04() { + echo -e "${YELLOW}Updating GCC for Ubuntu 18.04...${NO_COLOR}" + install_package software-properties-common + yes | add-apt-repository ppa:ubuntu-toolchain-r/test + update + install_package gcc-9 g++-9 + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 + check_status "Failed to update GCC" "GCC updated successfully." $? +} + +install_node_in_ubuntu18.04() { + # very slow and not test + if [ -n "$1" ]; then + DEFAULT_NODE_VERSION="$1" + else + DEFAULT_NODE_VERSION="14" + fi + NODE_DISTRO="node-v$DEFAULT_NODE_VERSION-linux-x64" + update_ubuntu_gcc_18.04 + echo "Installing Node..." + curl --retry 10 --retry-delay 5 --retry-max-time 120 -O https://nodejs.org/dist/v22.0.0/node-v22.0.0.tar.gz + tar -xzf node-v22.0.0.tar.gz + cd node-v22.0.0 || exit + ./configure + make + make install +} + + +# Install pnpm +install_pnpm() { + echo -e "${YELLOW}Installing pnpm...${NO_COLOR}" + centos_skip_check + NODE_VERSION=$(node -v) + + MAJOR_VERSION=$(echo "$NODE_VERSION" | cut -d '.' -f 1 | tr -d 'v') + MINOR_VERSION=$(echo "$NODE_VERSION" | cut -d '.' -f 2) + PATCH_VERSION=$(echo "$NODE_VERSION" | cut -d '.' -f 3) + + VERSION_NUMBER=$((MAJOR_VERSION * 10000 + MINOR_VERSION * 100 + PATCH_VERSION)) + + REQUIRED_VERSION=181200 # v18.12.00 + + if [ $VERSION_NUMBER -ge $REQUIRED_VERSION ]; then + echo "Node version is $NODE_VERSION, installing pnpm..." + npm install --global pnpm + pnpm --version + check_status "Failed to install pnpm" "pnpm installed successfully." $? + else + echo "Node version is $NODE_VERSION, skipping pnpm installation." + fi + +} + +# Install Node via nvm +# shellcheck disable=SC2120 +install_node_via_nvm () { + echo -e "${YELLOW}Installing Node via NVM...${NO_COLOR}" + if [ -n "$1" ]; then + DEFAULT_NODE_VERSION="$1" + else + DEFAULT_NODE_VERSION="" + fi + + if [[ -f /etc/redhat-release ]]; then + if [[ "$1" != "16.20.2" ]]; then + centos_skip_check + fi + fi + + # Install NVM + if ! command -v nvm &> /dev/null; then + NVM_VERSION=$(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")') + curl --retry 10 --retry-delay 5 --retry-max-time 120 -o- https://raw.githubusercontent.com/nvm-sh/nvm/"$NVM_VERSION"/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" + echo -e "${GREEN}NVM installed successfully.${NO_COLOR}" + SOURCE_RESULTS+="source $NVM_DIR/nvm.sh && source $NVM_DIR/bash_completion # For nvm/node/npm/yarn/pnpm\n" + else + echo -e "${GREEN}NVM is already installed.${NO_COLOR}" + fi + + # Check if version is specified + if [ -n "$1" ]; then + NODE_VERSION="$1" # use specified version + echo -e "${YELLOW}Installing Node version $NODE_VERSION...${NO_COLOR}" + nvm install "$NODE_VERSION" + else + echo -e "${YELLOW}Installing the latest version of Node...${NO_COLOR}" + nvm install node # use latest version + fi + nvm alias default node # set default version + + echo -e "${GREEN}Node installed successfully.${NO_COLOR}" + + npm config set fetch-retry-maxtimeout 120000 + npm config set fetch-retry-factor 5 + npm config set registry=https://registry.npmmirror.com + npm install --global yarn + npm install --global pnpm + # NPM_BIN_DIR=$(npm bin -g) + # if [ -d "$NPM_BIN_DIR" ]; then + # ln -sf "$NPM_BIN_DIR/yarn" /usr/bin/yarn + # ln -sf "$NPM_BIN_DIR/yarnpkg" /usr/bin/yarnpkg + # echo -e "${GREEN}Yarn installed successfully.${NO_COLOR}" + # else + # echo -e "${RED}Failed to find npm global bin directory. Yarn installation may not be complete.${NO_COLOR}" + # fi + node --version + check_status "Failed to install Node" "Node installed successfully." $? + npm --version + check_status "Failed to install npm" "npm installed successfully." $? + yarn --version + check_status "Failed to install Yarn" "Yarn installed successfully." $? +} + +# Install Node npm Yarn +# shellcheck disable=SC2120 +install_node() { + echo -e "${YELLOW}Installing Node...${NO_COLOR}" + if [ -n "$1" ]; then + DEFAULT_NODE_VERSION="$1" + if [[ -f /etc/redhat-release ]]; then + if [[ "$1" != "16.20.2" ]]; then + centos_skip_check + fi + fi + echo -e "${YELLOW}Installing Node version $DEFAULT_NODE_VERSION from source...${NO_COLOR}" + NODE_DISTRO="node-v$DEFAULT_NODE_VERSION-linux-x64" + wget "https://nodejs.org/dist/v$DEFAULT_NODE_VERSION/$NODE_DISTRO.tar.xz" -O "/tmp/$NODE_DISTRO.tar.xz" + tar -xf "/tmp/$NODE_DISTRO.tar.xz" -C /usr/local/lib/ + ln -sf "/usr/local/lib/$NODE_DISTRO/bin/node" /usr/bin/node + ln -sf "/usr/local/lib/$NODE_DISTRO/bin/npm" /usr/bin/npm + ln -sf "/usr/local/lib/$NODE_DISTRO/bin/npx" /usr/bin/npx + rm -rf "/tmp/$NODE_DISTRO.tar.xz" + node --version + echo -e "${GREEN}Node version $DEFAULT_NODE_VERSION installed successfully.${NO_COLOR}" + else + install_package "nodejs" + install_package "npm" + fi + node --version + check_status "Failed to install Node" "Node installed successfully." $? + + npm --version + check_status "Failed to install npm" "npm installed successfully." $? + npm config set fetch-retry-maxtimeout 120000 + npm config set fetch-retry-attempts + npm config set fetch-retry-factor 5 + npm config set registry=https://registry.npmmirror.com + npm install --global yarn + NPM_BIN_DIR=$(npm bin -g) + if [ -d "$NPM_BIN_DIR" ]; then + ln -sf "$NPM_BIN_DIR/yarn" /usr/bin/yarn + ln -sf "$NPM_BIN_DIR/yarnpkg" /usr/bin/yarnpkg + echo -e "${GREEN}Yarn installed successfully.${NO_COLOR}" + else + echo -e "${RED}Failed to find npm global bin directory. Yarn installation may not be complete.${NO_COLOR}" + fi + yarn --version + check_status "Failed to install Yarn" "Yarn installed successfully." $? +} + + + +# Deploy Git +deploy_git() { + install_package "git" + git --version + check_status "Failed to install Git" "Git installed successfully." $? + git config --global user.name "taos-support" + git config --global user.email "it@taosdata.com" + git config --global credential.helper store +} + +deploy_node_exporter() { + if [ ! -f "$NODE_EXPORTER_BINARY" ]; then + echo "Node Exporter is not installed. Installing now..." + + echo "Fetching the latest version of Node Exporter..." + LATEST_URL=$(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | jq -r '.assets[] | select(.name | test("node_exporter-.*linux-amd64.tar.gz")) | .browser_download_url') + + if [ -z "$LATEST_URL" ]; then + echo "Failed to fetch the latest Node Exporter release URL. Exiting." + exit 1 + fi + + echo "Downloading Node Exporter from $LATEST_URL..." + wget "$LATEST_URL" -O node_exporter.tar.gz + + echo "Extracting Node Exporter..." + tar -xzf node_exporter.tar.gz + cd node_exporter-*.linux-amd64 || exit + + echo "Copying binary..." + cp node_exporter /usr/local/bin/ + + echo "Creating systemd service..." + cat < /etc/systemd/system/node_exporter.service +[Unit] +Description=Node Exporter + +[Service] +ExecStart=/usr/local/bin/node_exporter + +[Install] +WantedBy=default.target +EOF + + # Start Node Exporter and enable it to run on startup + systemctl daemon-reload + systemctl start node_exporter + systemctl enable node_exporter + + # Clean up the downloaded tar to save space + cd .. + rm -rf node_exporter*.tar.gz node_exporter-*.linux-amd64 + node_exporter --version + check_status "Failed to install Node Exporter" "Node Exporter installed successfully." $? + else + echo "Node Exporter is already installed." + fi +} + +deploy_process_exporter() { + if [ ! -f "$PROCESS_EXPORTER_BINARY" ]; then + echo "Process Exporter is not installed. Installing now..." + + echo "Fetching the latest version of Process Exporter..." + LATEST_URL=$(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s https://api.github.com/repos/ncabatoff/process-exporter/releases/latest | jq -r '.assets[] | select(.name | test("process-exporter-.*linux-amd64.tar.gz")) | .browser_download_url') + + if [ -z "$LATEST_URL" ]; then + echo "Failed to fetch the latest Process Exporter release URL. Exiting." + exit 1 + fi + + echo "Downloading Process Exporter from $LATEST_URL..." + wget "$LATEST_URL" -O process-exporter.tar.gz + + echo "Extracting Process Exporter..." + tar -xzf process-exporter.tar.gz + cd process-exporter-*.linux-amd64 || exit + + echo "Copying binary..." + cp process-exporter /usr/local/bin/process-exporter + + echo "Creating configuration file..." + cat < /etc/process_exporter.yml +process_names: + - name: "{{.Comm}}" + cmdline: + - taosd +EOF + + echo "Creating systemd service..." + cat < /etc/systemd/system/process_exporter.service +[Unit] +Description=Process Exporter + +[Service] +ExecStart=/usr/local/bin/process-exporter --config.path /etc/process_exporter.yml + +[Install] +WantedBy=default.target +EOF + + # Start Process Exporter and enable it to run on startup + systemctl daemon-reload + systemctl start process_exporter + systemctl enable process_exporter + + # Clean up the downloaded tar to save space + cd .. + rm -rf process-exporter*.tar.gz process-exporter-*.linux-amd64 + process-exporter --version + check_status "Failed to install Process Exporter" "Process Exporter installed successfully." $? + else + echo "Process Exporter is already installed." + fi +} + +deploy_prometheus() { + # Check if Prometheus binary exists + if [ ! -f "$PROMETHEUS_BINARY" ]; then + echo "Prometheus is not installed. Installing now..." + + echo "Fetching the latest version of Prometheus..." + LATEST_URL=$(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s https://api.github.com/repos/prometheus/prometheus/releases/latest | jq -r '.assets[] | select(.name | test("prometheus-.*linux-amd64.tar.gz")) | .browser_download_url') + + if [ -z "$LATEST_URL" ]; then + echo "Failed to fetch the latest Prometheus release URL. Exiting." + exit 1 + fi + + echo "Downloading Prometheus from $LATEST_URL..." + wget "$LATEST_URL" -O prometheus.tar.gz + + echo "Extracting Prometheus..." + tar -xzf prometheus.tar.gz + cd prometheus-*.linux-amd64 || exit + + echo "Creating directories..." + mkdir -p /etc/prometheus /var/lib/prometheus + + echo "Copying binaries and configuration..." + cp prometheus promtool /usr/local/bin/ + + echo "Setting up Prometheus configuration..." + cat < /etc/prometheus/prometheus.yml +global: + scrape_interval: 15s +scrape_configs: + - job_name: 'node_exporter' + static_configs: + - targets: ['localhost:9100'] +EOF + + echo "Creating systemd service..." + cat < /etc/systemd/system/prometheus.service +[Unit] +Description=Prometheus Service + +[Service] +ExecStart=/usr/local/bin/prometheus \\ + --config.file /etc/prometheus/prometheus.yml \\ + --storage.tsdb.path /var/lib/prometheus/ \\ + --web.console.templates=/etc/prometheus/consoles \\ + --web.console.libraries=/etc/prometheus/console_libraries + +[Install] +WantedBy=default.target +EOF + + # Start Prometheus and enable it to run on startup + systemctl daemon-reload + systemctl start prometheus + systemctl enable prometheus + + # Clean up the downloaded tar to save space + cd .. + rm -rf prometheus*.tar.gz prometheus-*.linux-amd64 + prometheus --version + check_status "Failed to install Prometheus" "Prometheus installed successfully." $? + else + echo "Prometheus is already installed." + fi +} + +# Install Grafana using a downloaded .deb package +deploy_grafana() { + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + deploy_debian_grafana + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + deploy_redhat_grafana + else + echo "Unsupported Linux distribution." + exit 1 + fi +} + +# Install Grafana for ubuntu/debian +deploy_debian_grafana() { + # Check if Grafana is already installed + if ! dpkg -s "grafana" &> /dev/null; then + echo "Downloading the latest Grafana .deb package..." + # Download the latest Grafana .deb package + wget https://dl.grafana.com/oss/release/grafana_latest_amd64.deb -O grafana.deb + # install the required fontconfig package + install_package libfontconfig1 + echo "Installing Grafana..." + # Install the .deb package + dpkg -i grafana.deb + + # Clean up the downloaded .deb package to save space + rm -rf grafana.deb + + # Start the Grafana server and enable it to run on startup + systemctl start grafana-server + systemctl enable grafana-server + + # Check if Grafana was installed successfully + if [ $? -eq 0 ]; then + echo "Grafana was installed successfully." + else + echo "Failed to install Grafana." + fi + else + echo "Grafana is already installed." + fi +} + +# Install Grafana for centos/redhat +deploy_redhat_grafana() { + # Check if Grafana is already installed + if ! rpm -q grafana &> /dev/null; then + echo "Downloading the latest Grafana .rpm package..." + # Download the latest Grafana .rpm package + wget https://dl.grafana.com/oss/release/grafana-8.5.2-1.x86_64.rpm -O grafana.rpm + + # Install the required fontconfig package + yum install -y fontconfig + + echo "Installing Grafana..." + # Install the .rpm package + rpm -ivh grafana.rpm + + # Clean up the downloaded .rpm package to save space + rm -rf grafana.rpm + + # Start the Grafana server and enable it to run on startup + systemctl start grafana-server + systemctl enable grafana-server + + # Check if Grafana was installed successfully + if [ $? -eq 0 ]; then + echo "Grafana was installed successfully." + else + echo "Failed to install Grafana." + fi + else + echo "Grafana is already installed." + fi +} + +# Install Nginx +install_nginx() { + install_package "nginx" + nginx -v + check_status "Failed to install Nginx" "Nginx installed successfully." $? +} + +# Deploy JMeter +deploy_jmeter() { + if ! command -v jmeter &> /dev/null; then + echo "Installing JMeter..." + install_java + wget -P /opt https://mirrors.aliyun.com/apache/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz + tar -xvzf /opt/apache-jmeter-$JMETER_VERSION.tgz -C /opt/ + ln -sf /opt/apache-jmeter-$JMETER_VERSION/bin/jmeter /usr/local/bin/jmeter + rm -rf /opt/apache-jmeter-$JMETER_VERSION.tgz + jmeter --version + check_status "Failed to install JMeter" "JMeter installed successfully." $? + else + echo "JMeter is already installed." + fi +} + +# Deploy Docker +deploy_docker() { + if [ -f /etc/debian_version ]; then + # Debian or Ubuntu + deploy_debian_docker + elif [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + deploy_redhat_docker + else + echo "Unsupported Linux distribution." + exit 1 + fi +} + +# Deploy Docker for centos/redhat +deploy_redhat_docker() { + # Check if Docker is already installed + if ! command -v docker &> /dev/null; then + echo "Docker is not installed. Installing now..." + + # Set up the repository for Docker + echo "Setting up the Docker repository..." + install_package yum-utils + + echo "Adding Docker's official repository..." + yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo + + # Install Docker CE + echo "Installing Docker CE..." + install_package docker-ce docker-ce-cli containerd.io + + # Enable and start Docker + echo "Enabling and starting Docker..." + systemctl enable docker + systemctl start docker + + # Adding current user to the Docker group + usermod -aG docker "$USER" + + # Print Docker version + docker --version + + # Check the installation status + if [ $? -eq 0 ]; then + echo "Docker installed successfully." + else + echo "Failed to install Docker." + fi + else + echo "Docker is already installed." + fi +} + +# Deploy docker for ubuntu/debian +deploy_debian_docker() { + # Check if Docker is already installed + if ! command -v docker &> /dev/null; then + echo "Docker is not installed. Installing now..." + + # Set up the repository for Docker + echo "Setting up the Docker repository..." + update + install_package apt-transport-https ca-certificates curl software-properties-common gnupg lsb-release + + echo "Adding Docker's official GPG key from Aliyun..." + curl --retry 10 --retry-delay 5 --retry-max-time 120 -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | { + if [ -f /usr/share/keyrings/docker-archive-keyring.gpg ]; then + rm /usr/share/keyrings/docker-archive-keyring.gpg + fi + gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + } + echo "Setting up stable repository using Aliyun..." + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null + + # Install Docker CE + echo "Installing Docker CE..." + update + install_package docker-ce docker-ce-cli containerd.io + + # Enable and start Docker + echo "Enabling and starting Docker..." + systemctl enable docker + systemctl start docker + + # Adding current user to the Docker group + usermod -aG docker "$USER" + + # Print Docker version + docker --version + check_status "Failed to install Docker" "Docker installed successfully." $? + else + echo "Docker is already installed." + fi +} + +# Deploy Docker Compose +deploy_docker_compose() { + # Check if Docker Compose is installed + if ! command -v docker-compose &> /dev/null; then + echo "Docker Compose is not installed. Installing now..." + + # Install Docker Compose + curl --retry 10 --retry-delay 5 --retry-max-time 120 -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + + # Print Docker Compose version + docker-compose --version + check_status "Failed to install Docker Compose" "Docker Compose installed successfully." $? + else + echo "Docker Compose is already installed." + fi +} + +# Reconfigure cloud-init +reconfig_cloud_init() { + echo "Reconfiguring cloud-init..." + apt remove -y cloud-init && apt purge -y cloud-init + rm -rf /var/lib/cloud /etc/cloud + apt update -y + install_package cloud-init + sed -i '/package[-_]update[-_]upgrade[-_]install/s/^/#/' /etc/cloud/cloud.cfg +} + +# Config cloud-init +config_cloud_init() { + if [ "$DIST_VERSION" = "7.9" ];then + install_package "cloud-init" + sed -i '/ssh_pwauth.*/s/^/#/' /etc/cloud/cloud.cfg + else + reconfig_cloud_init + fi + check_status "Failed to configure cloud-init" "Cloud-init configured successfully and you need reboot manually." $? + # if [ "$DIST_VERSION" = "18.04" ] || [ "$DIST_VERSION" = "20.04" ]; then + # reconfig_cloud_init + # elif [ "$DIST_VERSION" = "7.9" ];then + # install_package "cloud-init" + # sed -i '/ssh_pwauth.*/s/^/#/' /etc/cloud/cloud.cfg + # else + # echo "Configuring cloud-init..." + # add_config_if_not_exist "$CLOUD_INIT_CONFIG" "/etc/cloud/cloud.cfg" + + # marker="NoCloud" + + # if grep -qF "$marker" "/etc/cloud/cloud.cfg"; then + # echo "cloud-init settings already exists in /etc/cloud/cloud.cfg." + # else + # echo "Adding configuration to /etc/cloud/cloud.cfg." + # echo "$CLOUD_INIT_CONFIG" >> "/etc/cloud/cloud.cfg" + # echo "cloud-init settings have been updated in /etc/cloud/cloud.cfg." + # fi + + # mkdir -p /var/lib/cloud/seed/nocloud/ + # cd /var/lib/cloud/seed/nocloud/ || exit + # touch meta-data + # touch user-data + # add_config_if_not_exist "hostname: \${name}" user-data + # add_config_if_not_exist "manage_etc_hosts: true" user-data + # fi + # cloud-init clean --logs +} + +cleanup() { + if [ -n "$SOURCE_RESULTS" ]; then + echo -e "${YELLOW}===========================================\n${NO_COLOR}" + echo -e "${YELLOW}Installation complete! \n${NO_COLOR}" + echo -e "${YELLOW}Some tools require you to manually source${NO_COLOR}" + echo -e "${YELLOW}or restart your terminal to take effect.\n${NO_COLOR}" + echo -e "${YELLOW}===========================================\n${NO_COLOR}" + echo -e "${YELLOW}$SOURCE_RESULTS${NO_COLOR}" + else + echo -e "${YELLOW}Installation complete \n${NO_COLOR}" + fi +} + +# Clone a repository with a specified target directory +clone_repo_with_rename() { + local repo_url="$1" + local target_dir="$2" + local branch_name="$3" + + if [ -z "$target_dir" ]; then + target_dir=$(basename -s .git "$repo_url") + fi + + cd "$REPOPATH" || exit + + if [ -d "$target_dir" ]; then + echo "Directory $target_dir already exists. Skipping clone." + else + echo "Cloning into $target_dir..." + if [ -n "$branch_name" ]; then + git clone -b "$branch_name" "$repo_url" "$target_dir" + else + git clone "$repo_url" "$target_dir" + fi + fi +} + +# Clone enterprise repository +clone_enterprise() { + cd "$REPOPATH" || exit + clone_repo_with_rename https://github.com/taosdata/TDinternal + clone_repo_with_rename git@github.com:taosdata/TDengine.git TDinternal/community +} + +# Clone community repository +clone_community() { + cd "$REPOPATH" || exit + clone_repo_with_rename https://github.com:taosdata/TDengine.git +} + +# Clone TaosX repository +clone_taosx() { + cd "$REPOPATH" || exit + clone_repo_with_rename https://github.com/taosdata/taosx +} + +# Clone TaosKeeper repository +clone_taoskeeper() { + cd "$REPOPATH" || exit + clone_repo_with_rename https://github.com/taosdata/taoskeeper +} + +# Clone TaosTest repository +clone_taostest() { + cd "$REPOPATH" || exit + clone_repo_with_rename https://github.com/taosdata/taos-test-framework "" "master" +} + +# Clone TestNG repository +clone_testng() { + cd "$REPOPATH" || exit + clone_repo_with_rename https://github.com/taosdata/TestNG "" "master" +} + +# Clone operation tools repository +clone_operation() { + cd "$REPOPATH" || exit + clone_repo_with_rename https://github.com/taosdata/operation.git +} + +# init system +system_config() { + disable_service + config_dns + replace_sources + config_cloud_init + config_ssh + config_custom_settings + config_timezone + config_share_server + disable_firewalld + config_frontend + config_system_limits + config_coredump + check_status "Failed to config system" "Config system successfully" $? +} + +# Clone all the repositories +clone_repos() { + clone_enterprise + clone_community + clone_taosx + clone_taoskeeper + clone_taostest + clone_operation +} + +new_funcs() { + echo "Adding test..." + install_python_via_pyenv 3.10.12 + install_java_via_sdkman 21.0.2 + install_node 16.20.2 + install_maven_via_sdkman 3.2.5 + deploy_rust +} + +# deploy TDasset +TDasset() { + install_java_via_sdkman 21.0.2 + install_maven_via_sdkman 3.9.9 + # not supported in centos7/ubuntu18 because of the old version of glibc + install_node_via_nvm 22.0.0 + install_pnpm +} + +# deploy TDinternal/TDengine/taosx +TDinternal() { + install_go_via_gvm 1.23.3 + deploy_rust + install_java_via_sdkman 17 + install_maven_via_sdkman 3.9.9 + install_node_via_nvm 16.20.2 + install_python_via_pyenv 3.10.12 +} + +# deploy TDgpt +TDgpt() { + install_python_via_pyenv 3.10.12 +} + +# deploy taos-test-framework +taostest() { + if [ ! -d "$REPOPATH/taos-test-framework" ]; then + echo "Cloning TaosTest repository..." + clone_taostest + else + echo "TaosTest repository already exists. Skipping clone." + fi + check_status "Failed to clone TaosTest repository" "TaosTest repository cloned successfully." $? + + if [ ! -d "$REPOPATH/TestNG" ]; then + echo "Cloning TestNG repository..." + clone_testng + else + echo "TestNG repository already exists. Skipping clone." + fi + check_status "Failed to clone TestNG repository" "TestNG repository cloned successfully." $? + + # Configure environment variables + echo "Configuring TaosTest environment variables..." + mkdir -p "$HOME"/.taostest + add_config_if_not_exist "TEST_ROOT=$REPOPATH/TestNG" "$HOME"/.taostest/.env + + # Install TaosTest + echo "Installing TaosTest..." + cd "$REPOPATH"/taos-test-framework || exit + install_package "python3-pip" + install_via_pip "poetry" + yes | ./reinstall.sh + check_status "Failed to install TaosTest" "TaosTest installed successfully." $? + + # Configure passwdless login + echo "Configuring passwdless login..." + yes | ssh-keygen -t rsa -b 2048 -N "" -f "$HOME/.ssh/testng" + cat "$HOME"/.ssh/testng.pub >> "$HOME"/.ssh/authorized_keys +} + + +# Deploy pure environment +deploy_pure() { + disable_service + config_dns + if [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + echo "Replacing sources for YUM package manager." + replace_yum_sources + fi + config_cloud_init + config_ssh + config_custom_settings + config_timezone + config_share_server + disable_firewalld + install_package "jq" + install_package "wget" + deploy_node_exporter + check_status "Failed to config pure system" "Config pure system successfully" $? +} + +# Deploy development environment +deploy_dev() { + install_packages + deploy_cmake + if [ -f /etc/redhat-release ]; then + # Red Hat or CentOS + update_redhat_gcc + fi + deploy_tmux + deploy_git + install_python + install_pip_pkg + install_java + install_maven_via_sdkman + deploy_go + deploy_rust + install_node + deploy_node_exporter + deploy_process_exporter + deploy_prometheus + deploy_grafana + deploy_jmeter + install_nginx + deploy_docker + deploy_docker_compose + check_status "Failed to deploy some tools" "Deploy all tools successfully" $? +} + +# Setup all configurations +setup_all() { + system_config + deploy_dev +} + +# More installation functions can be added here following the above examples + +# Main execution function +main() { + # Check if at least one argument is provided + if [ $# -eq 0 ]; then + echo "Error: No arguments provided." + echo "Please try $0 --help" + exit 1 + fi + init_env + for arg in "$@"; do + case $arg in + --help) + help + exit 0 + ;; + setup_all) + setup_all + ;; + config_ssh) + config_ssh + ;; + disable_firewalld) + disable_firewalld + ;; + config_cloud_init) + config_cloud_init + ;; + deploy_git) + deploy_git + ;; + replace_sources) + replace_sources + ;; + update) + update + ;; + upgrade) + upgrade + ;; + config_timezone) + config_timezone + ;; + config_dns) + config_dns + ;; + config_custom_settings) + config_custom_settings + ;; + install_packages) + install_packages + ;; + config_system_limits) + config_system_limits + ;; + config_coredump) + config_coredump + ;; + disable_service) + disable_service + ;; + install_python) + install_python + ;; + install_pyenv) + install_pyenv + ;; + install_python_via_pyenv) + install_python_via_pyenv + ;; + install_pip_pkg) + install_pip_pkg + ;; + install_java) + install_java + ;; + install_java_via_sdkman) + install_java_via_sdkman + ;; + install_maven_via_sdkman) + install_maven_via_sdkman + ;; + deploy_cmake) + deploy_cmake + ;; + update_redhat_gcc) + update_redhat_gcc + ;; + update_redhat_tmux) + update_redhat_tmux + ;; + deploy_tmux) + deploy_tmux + ;; + deploy_go) + deploy_go + ;; + install_gvm) + install_gvm + ;; + install_go_via_gvm) + install_go_via_gvm + ;; + deploy_rust) + deploy_rust + ;; + install_node) + install_node + ;; + install_node_via_nvm) + install_node_via_nvm + ;; + install_pnpm) + install_pnpm + ;; + deploy_node_exporter) + deploy_node_exporter + ;; + deploy_process_exporter) + deploy_process_exporter + ;; + deploy_prometheus) + deploy_prometheus + ;; + deploy_grafana) + deploy_grafana + ;; + deploy_jmeter) + deploy_jmeter + ;; + install_nginx) + install_nginx + ;; + config_qemu_guest_agent) + config_qemu_guest_agent + ;; + config_share_server) + config_share_server + ;; + deploy_docker) + deploy_docker + ;; + deploy_docker_compose) + deploy_docker_compose + ;; + clone_enterprise) + clone_enterprise + ;; + clone_community) + clone_community + ;; + clone_taosx) + clone_taosx + ;; + clone_taoskeeper) + clone_taoskeeper + ;; + clone_taostest) + clone_taostest + ;; + clone_operation) + clone_operation + ;; + system_config) + system_config + ;; + deploy_pure) + deploy_pure + ;; + deploy_dev) + deploy_dev + ;; + TDasset) + TDasset + ;; + TDinternal) + TDinternal + ;; + TDgpt) + TDgpt + ;; + taostest) + taostest + ;; + new_funcs) + new_funcs + ;; + *) + echo "Unknown function: $arg" + ;; + esac + done +} + +# Execute the script with specified function arguments +main "$@" diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 35ab328815..68d75f9897 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -66,9 +66,9 @@ int32_t s3Begin() { void s3End() { S3_deinitialize(); } int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); /*s3Begin();*/ } - +#if 0 static int32_t s3ListBucket(char const *bucketname); - +#endif static void s3DumpCfgByEp(int8_t epIndex) { // clang-format off (void)fprintf(stdout, @@ -291,7 +291,7 @@ static int32_t s3ListBucketByEp(char const *bucketname, int8_t epIndex) { TAOS_RETURN(code); } - +#if 0 static int32_t s3ListBucket(char const *bucketname) { int32_t code = 0; @@ -312,7 +312,7 @@ static int32_t s3ListBucket(char const *bucketname) { TAOS_RETURN(code); } - +#endif typedef struct growbuffer { // The total number of bytes, and the start byte int size; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index cb08046399..8bed8d23eb 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -123,6 +123,9 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "encrypt_algorithm", .bytes = TSDB_ENCRYPT_ALGO_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "compact_interval", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "compact_time_range", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "compact_time_offset", .bytes = 4 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema userFuncSchema[] = { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 355a34d05b..d6d3e3a443 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -102,9 +102,8 @@ int32_t tsMaxStreamBackendCache = 128; // M int32_t tsPQSortMemThreshold = 16; // M int32_t tsRetentionSpeedLimitMB = 0; // unlimited -const char *tsAlterCompactTaskKeywords = "max_compact_tasks"; -int32_t tsNumOfCompactThreads = 2; -int32_t tsNumOfRetentionThreads = 1; +int32_t tsNumOfCompactThreads = 2; +int32_t tsNumOfRetentionThreads = 1; // sync raft int32_t tsElectInterval = 25 * 1000; @@ -744,8 +743,9 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); - - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -794,9 +794,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); - tsNumOfCompactThreads = tsNumOfCommitThreads; - tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4); - tsNumOfSupportVnodes = tsNumOfCores * 2 + 5; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); @@ -841,7 +838,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxCompactConcurrency", tsNumOfCompactThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCompactThreads", tsNumOfCompactThreads, 1, 16, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryUseMemoryPool", tsQueryUseMemoryPool, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "memPoolFullFunc", tsMemPoolFullFunc, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); @@ -1038,10 +1035,8 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } - pItem = cfgGetItem(pCfg, "maxCompactConcurrency"); + pItem = cfgGetItem(pCfg, "numOfCompactThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfCompactThreads = numOfCores / 2; - tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4); pItem->i32 = tsNumOfCompactThreads; pItem->stype = stype; } @@ -1546,7 +1541,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCommitThreads"); tsNumOfCommitThreads = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "maxCompactConcurrency"); + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCompactThreads"); tsNumOfCompactThreads = pItem->i32; TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "retentionSpeedLimitMB"); @@ -2349,6 +2344,8 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, TAOS_RETURN(code); } +extern void tsdbAlterNumCompactThreads(); + static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = -1; @@ -2399,6 +2396,17 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { goto _exit; } + if (strcasecmp(name, "numOfCompactThreads") == 0) { +#ifdef TD_ENTERPRISE + tsNumOfCompactThreads = pItem->i32; + code = TSDB_CODE_SUCCESS; + // tsdbAlterNumCompactThreads(); +#else + code = TSDB_CODE_INVALID_CFG; +#endif + goto _exit; + } + { // 'bool/int32_t/int64_t/float/double' variables with general modification function static OptionNameAndVar debugOptions[] = { {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index bb12612273..31afb7377e 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -40,6 +40,46 @@ add_test( COMMAND dataformatTest ) +# cosCpTest.cpp +add_executable(cosCpTest "") +target_sources( + cosCpTest + PRIVATE + "cosCpTest.cpp" +) +target_link_libraries(cosCpTest gtest gtest_main util common) +target_include_directories( + cosCpTest + PUBLIC "${TD_SOURCE_DIR}/include/common" + PUBLIC "${TD_SOURCE_DIR}/include/util" +) +add_test( + NAME cosCpTest + COMMAND cosCpTest +) + +if(TD_LINUX) + +# cosTest.cpp +add_executable(cosTest "") +target_sources( + cosTest + PRIVATE + "cosTest.cpp" +) +target_link_libraries(cosTest gtest gtest_main util common) +target_include_directories( + cosTest + PUBLIC "${TD_SOURCE_DIR}/include/common" + PUBLIC "${TD_SOURCE_DIR}/include/util" +) +add_test( + NAME cosTest + COMMAND cosTest +) + +endif() + if (${TD_LINUX}) # tmsg test add_executable(tmsgTest "") @@ -60,4 +100,4 @@ if (${TD_LINUX}) add_custom_command(TARGET tmsgTest POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MSG_TBL_FILE} $ ) -endif () \ No newline at end of file +endif () diff --git a/source/common/test/cosCpTest.cpp b/source/common/test/cosCpTest.cpp new file mode 100644 index 0000000000..fc16daa8cc --- /dev/null +++ b/source/common/test/cosCpTest.cpp @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} + +TEST(testCase, cpOpenCloseRemove) { + int32_t code = 0, lino = 0; + + int64_t contentLength = 1024; + const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3; + int totalSeq = (contentLength + chunk_size - 1) / chunk_size; + const int max_part_num = 10000; + if (totalSeq > max_part_num) { + chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; + totalSeq = (contentLength + chunk_size - 1) / chunk_size; + } + SCheckpoint cp; + char const *file = "./afile"; + char file_cp_path[TSDB_FILENAME_LEN]; + + (void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file); + + cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart)); + if (!cp.parts) { + TAOS_CHECK_EXIT(terrno); + } + + EXPECT_EQ(cos_cp_open(file_cp_path, &cp), TSDB_CODE_SUCCESS); + + if (cp.thefile) { + EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS); + } + if (cp.parts) { + taosMemoryFree(cp.parts); + } + + EXPECT_EQ(cos_cp_remove(file_cp_path), TSDB_CODE_SUCCESS); + + return; + +_exit: + std::cout << "code: " << code << std::endl; +} + +TEST(testCase, cpBuild) { + int32_t code = 0, lino = 0; + + int64_t contentLength = 1024; + const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3; + int totalSeq = (contentLength + chunk_size - 1) / chunk_size; + const int max_part_num = 10000; + if (totalSeq > max_part_num) { + chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; + totalSeq = (contentLength + chunk_size - 1) / chunk_size; + } + SCheckpoint cp; + char const *file = "./afile"; + char file_cp_path[TSDB_FILENAME_LEN]; + int64_t lmtime = 20241220141705; + char const *upload_id = "upload-id-xxx"; + + (void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file); + (void)memset(&cp, 0, sizeof(cp)); + + cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart)); + if (!cp.parts) { + TAOS_CHECK_EXIT(terrno); + } + + EXPECT_EQ(cos_cp_open(file_cp_path, &cp), TSDB_CODE_SUCCESS); + + cos_cp_build_upload(&cp, file, contentLength, lmtime, upload_id, chunk_size); + + EXPECT_EQ(cos_cp_dump(&cp), TSDB_CODE_SUCCESS); + + if (cp.thefile) { + EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS); + } + if (cp.parts) { + taosMemoryFree(cp.parts); + } + + return; + +_exit: + std::cout << "code: " << code << std::endl; +} + +TEST(testCase, cpLoad) { + int32_t code = 0, lino = 0; + + int64_t contentLength = 1024; + const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3; + int totalSeq = (contentLength + chunk_size - 1) / chunk_size; + const int max_part_num = 10000; + if (totalSeq > max_part_num) { + chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; + totalSeq = (contentLength + chunk_size - 1) / chunk_size; + } + SCheckpoint cp; + char const *file = "./afile"; + char file_cp_path[TSDB_FILENAME_LEN]; + int64_t lmtime = 20241220141705; + char const *upload_id = "upload-id-xxx"; + + (void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file); + (void)memset(&cp, 0, sizeof(cp)); + + cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart)); + if (!cp.parts) { + TAOS_CHECK_EXIT(terrno); + } + + if (taosCheckExistFile(file_cp_path)) { + EXPECT_EQ(cos_cp_load(file_cp_path, &cp), TSDB_CODE_SUCCESS); + + EXPECT_EQ(cos_cp_is_valid_upload(&cp, contentLength, lmtime), true); + + EXPECT_EQ(cp.cp_type, COS_CP_TYPE_UPLOAD); + EXPECT_EQ(cp.md5, std::string("")); + EXPECT_EQ(cp.thefile, nullptr); + EXPECT_EQ(std::string(cp.file_path), "./afile"); + EXPECT_EQ(cp.file_size, 1024); + EXPECT_EQ(cp.file_last_modified, 20241220141705); + EXPECT_EQ(cp.file_md5, std::string("")); + EXPECT_EQ(cp.object_name, std::string("")); + EXPECT_EQ(cp.object_size, 0); + EXPECT_EQ(cp.object_last_modified, std::string("")); + EXPECT_EQ(cp.object_etag, std::string("")); + EXPECT_EQ(cp.upload_id, std::string("upload-id-xxx")); + + EXPECT_EQ(cp.part_num, 1); + EXPECT_EQ(cp.part_size, 8388608); + EXPECT_EQ(cp.parts[0].index, 0); + EXPECT_EQ(cp.parts[0].offset, 0); + EXPECT_EQ(cp.parts[0].size, 1024); + EXPECT_EQ(cp.parts[0].completed, 0); + EXPECT_EQ(cp.parts[0].etag, std::string("")); + EXPECT_EQ(cp.parts[0].crc64, 0); + } + + if (cp.thefile) { + EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS); + } + if (cp.parts) { + taosMemoryFree(cp.parts); + } + + EXPECT_EQ(cos_cp_remove(file_cp_path), TSDB_CODE_SUCCESS); + + return; + +_exit: + std::cout << "code: " << code << std::endl; +} + +TEST(testCase, cpBuildUpdate) { + int32_t code = 0, lino = 0; + + int64_t contentLength = 1024; + const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3; + int totalSeq = (contentLength + chunk_size - 1) / chunk_size; + const int max_part_num = 10000; + if (totalSeq > max_part_num) { + chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; + totalSeq = (contentLength + chunk_size - 1) / chunk_size; + } + SCheckpoint cp; + char const *file = "./afile"; + char file_cp_path[TSDB_FILENAME_LEN]; + int64_t lmtime = 20241220141705; + char const *upload_id = "upload-id-xxx"; + int seq = 1; + char *etags[1] = {"etags-1-xxx"}; + + (void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file); + (void)memset(&cp, 0, sizeof(cp)); + + cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart)); + if (!cp.parts) { + TAOS_CHECK_EXIT(terrno); + } + + EXPECT_EQ(cos_cp_open(file_cp_path, &cp), TSDB_CODE_SUCCESS); + + cos_cp_build_upload(&cp, file, contentLength, lmtime, upload_id, chunk_size); + + cos_cp_update(&cp, cp.parts[seq - 1].index, etags[seq - 1], 0); + + EXPECT_EQ(cos_cp_dump(&cp), TSDB_CODE_SUCCESS); + + if (cp.thefile) { + EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS); + } + if (cp.parts) { + taosMemoryFree(cp.parts); + } + + return; + +_exit: + std::cout << "code: " << code << std::endl; +} + +TEST(testCase, cpLoadUpdate) { + int32_t code = 0, lino = 0; + + int64_t contentLength = 1024; + const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3; + int totalSeq = (contentLength + chunk_size - 1) / chunk_size; + const int max_part_num = 10000; + if (totalSeq > max_part_num) { + chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; + totalSeq = (contentLength + chunk_size - 1) / chunk_size; + } + SCheckpoint cp; + char const *file = "./afile"; + char file_cp_path[TSDB_FILENAME_LEN]; + int64_t lmtime = 20241220141705; + char const *upload_id = "upload-id-xxx"; + + (void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file); + (void)memset(&cp, 0, sizeof(cp)); + + cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart)); + if (!cp.parts) { + TAOS_CHECK_EXIT(terrno); + } + + if (taosCheckExistFile(file_cp_path)) { + EXPECT_EQ(cos_cp_load(file_cp_path, &cp), TSDB_CODE_SUCCESS); + + EXPECT_EQ(cos_cp_is_valid_upload(&cp, contentLength, lmtime), true); + + EXPECT_EQ(cp.cp_type, COS_CP_TYPE_UPLOAD); + EXPECT_EQ(cp.md5, std::string("")); + EXPECT_EQ(cp.thefile, nullptr); + EXPECT_EQ(std::string(cp.file_path), "./afile"); + EXPECT_EQ(cp.file_size, 1024); + EXPECT_EQ(cp.file_last_modified, 20241220141705); + EXPECT_EQ(cp.file_md5, std::string("")); + EXPECT_EQ(cp.object_name, std::string("")); + EXPECT_EQ(cp.object_size, 0); + EXPECT_EQ(cp.object_last_modified, std::string("")); + EXPECT_EQ(cp.object_etag, std::string("")); + EXPECT_EQ(cp.upload_id, std::string("upload-id-xxx")); + + EXPECT_EQ(cp.part_num, 1); + EXPECT_EQ(cp.part_size, 8388608); + EXPECT_EQ(cp.parts[0].index, 0); + EXPECT_EQ(cp.parts[0].offset, 0); + EXPECT_EQ(cp.parts[0].size, 1024); + EXPECT_EQ(cp.parts[0].completed, 1); + EXPECT_EQ(cp.parts[0].etag, std::string("etags-1-xxx")); + EXPECT_EQ(cp.parts[0].crc64, 0); + } + + if (cp.thefile) { + EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS); + } + if (cp.parts) { + taosMemoryFree(cp.parts); + } + + EXPECT_EQ(cos_cp_remove(file_cp_path), TSDB_CODE_SUCCESS); + + return; + +_exit: + std::cout << "code: " << code << std::endl; +} diff --git a/source/common/test/cosTest.cpp b/source/common/test/cosTest.cpp new file mode 100644 index 0000000000..5a6aee52d9 --- /dev/null +++ b/source/common/test/cosTest.cpp @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} + +int32_t cosInitEnv() { + int32_t code = 0; + bool isBlob = false; + + extern int8_t tsS3Ablob; + extern char tsS3Hostname[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; + extern char tsS3BucketName[TSDB_FQDN_LEN]; + + tsS3Ablob = isBlob; + /* + const char *hostname = "endpoint/.blob.core.windows.net"; + const char *accessKeyId = ""; + const char *accessKeySecret = ""; + const char *bucketName = ""; + */ + + // const char *hostname = "http://192.168.1.52:9000"; + // const char *accessKeyId = "zOgllR6bSnw2Ah3mCNel"; + // const char *accessKeySecret = "cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX"; + // const char *bucketName = "test-bucket"; + const char *hostname = "192.168.1.52:9000"; + const char *accessKeyId = "zOgllR6bSnw2Ah3mCNel"; + const char *accessKeySecret = "cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX"; + const char *bucketName = "ci-bucket19"; + + tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); + tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + + // setup s3 env + extern int8_t tsS3EpNum; + extern int8_t tsS3Https[TSDB_MAX_EP_NUM]; + + tsS3EpNum = 1; + tsS3Https[0] = false; + + tstrncpy(tsTempDir, "/tmp/", PATH_MAX); + + tsS3Enabled = true; + + return code; +} + +TEST(testCase, cosCpPutError) { + int32_t code = 0, lino = 0; + + char const *objectName = "testObject"; + + EXPECT_EQ(cosInitEnv(), TSDB_CODE_SUCCESS); + EXPECT_EQ(s3Begin(), TSDB_CODE_SUCCESS); + +#if defined(USE_S3) + EXPECT_EQ(s3Size(objectName), -1); +#else + EXPECT_EQ(s3Size(objectName), 0); +#endif + + s3EvictCache("", 0); + + s3End(); + + return; + +_exit: + std::cout << "code: " << code << std::endl; +} + +TEST(testCase, cosCpPut) { + int32_t code = 0, lino = 0; + + int8_t with_cp = 0; + char *data = nullptr; + + const long objectSize = 65 * 1024 * 1024; + char const *objectName = "cosut.bin"; + const char object_name[] = "cosut.bin"; + + EXPECT_EQ(std::string(object_name), objectName); + + EXPECT_EQ(cosInitEnv(), TSDB_CODE_SUCCESS); + EXPECT_EQ(s3Begin(), TSDB_CODE_SUCCESS); + + { + data = (char *)taosMemoryCalloc(1, objectSize); + if (!data) { + TAOS_CHECK_EXIT(terrno); + } + + for (int i = 0; i < objectSize / 2; ++i) { + data[i * 2 + 1] = 1; + } + + char path[PATH_MAX] = {0}; + char path_download[PATH_MAX] = {0}; + int ds_len = strlen(TD_DIRSEP); + int tmp_len = strlen(tsTempDir); + + (void)snprintf(path, PATH_MAX, "%s", tsTempDir); + if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP); + (void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name); + } else { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name); + } + + tstrncpy(path_download, path, strlen(path) + 1); + tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1); + + TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH); + GTEST_ASSERT_NE(fp, nullptr); + + int n = taosWriteFile(fp, data, objectSize); + GTEST_ASSERT_EQ(n, objectSize); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + code = s3PutObjectFromFile2(path, objectName, with_cp); + GTEST_ASSERT_EQ(code, 0); + + with_cp = 1; + code = s3PutObjectFromFile2(path, objectName, with_cp); + GTEST_ASSERT_EQ(code, 0); + +#if defined(USE_S3) + EXPECT_EQ(s3Size(objectName), objectSize); +#else + EXPECT_EQ(s3Size(objectName), 0); +#endif + + s3End(); + s3EvictCache("", 0); + + taosMemoryFree(data); + + EXPECT_EQ(taosRemoveFile(path), TSDB_CODE_SUCCESS); + } + + return; + +_exit: + if (data) { + taosMemoryFree(data); + s3End(); + } + + std::cout << "code: " << code << std::endl; +} diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 8139e4aa98..9ed4ee83c4 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -475,27 +475,6 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } -extern void tsdbAlterNumCompactThreads(); -static int32_t dmAlterMaxCompactTask(const char *value) { - int32_t max_compact_tasks; - char *endptr = NULL; - - max_compact_tasks = taosStr2Int32(value, &endptr, 10); - if (endptr == value || endptr[0] != '\0') { - return TSDB_CODE_INVALID_MSG; - } - - if (max_compact_tasks != tsNumOfCompactThreads) { - dInfo("alter max compact tasks from %d to %d", tsNumOfCompactThreads, max_compact_tasks); - tsNumOfCompactThreads = max_compact_tasks; -#ifdef TD_ENTERPRISE - (void)tsdbAlterNumCompactThreads(); -#endif - } - - return TSDB_CODE_SUCCESS; -} - int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t code = 0; SDCfgDnodeReq cfgReq = {0}; @@ -509,10 +488,6 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return taosUpdateTfsItemDisable(pCfg, cfgReq.value, pMgmt->pTfs); } - if (strncmp(cfgReq.config, tsAlterCompactTaskKeywords, strlen(tsAlterCompactTaskKeywords) + 1) == 0) { - return dmAlterMaxCompactTask(cfgReq.value); - } - dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_SERVER_CMD, true); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c28fd343f0..1d1f5744d4 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -499,6 +499,20 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return code; if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return code; + if (pCfg->compactInterval != 0 && + (pCfg->compactInterval < TSDB_MIN_COMPACT_INTERVAL || pCfg->compactInterval > pCfg->daysToKeep2)) + return code; + if (pCfg->compactStartTime != 0 && + (pCfg->compactStartTime < -pCfg->daysToKeep2 || pCfg->compactStartTime > -pCfg->daysPerFile)) + return code; + if (pCfg->compactEndTime != 0 && + (pCfg->compactEndTime < -pCfg->daysToKeep2 || pCfg->compactEndTime > -pCfg->daysPerFile)) + return code; + if (pCfg->compactStartTime != 0 && pCfg->compactEndTime != 0 && pCfg->compactStartTime >= pCfg->compactEndTime) + return code; + if (pCfg->compactTimeOffset < TSDB_MIN_COMPACT_TIME_OFFSET || pCfg->compactTimeOffset > TSDB_MAX_COMPACT_TIME_OFFSET) + return code; + code = 0; TAOS_RETURN(code); } @@ -564,6 +578,22 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN if (pNewCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pNewCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return code; if (pNewCfg->s3Compact < TSDB_MIN_S3_COMPACT || pNewCfg->s3Compact > TSDB_MAX_S3_COMPACT) return code; + if (pNewCfg->compactInterval != 0 && + (pNewCfg->compactInterval < TSDB_MIN_COMPACT_INTERVAL || pNewCfg->compactInterval > pNewCfg->daysToKeep2)) + return code; + if (pNewCfg->compactStartTime != 0 && + (pNewCfg->compactStartTime < -pNewCfg->daysToKeep2 || pNewCfg->compactStartTime > -pNewCfg->daysPerFile)) + return code; + if (pNewCfg->compactEndTime != 0 && + (pNewCfg->compactEndTime < -pNewCfg->daysToKeep2 || pNewCfg->compactEndTime > -pNewCfg->daysPerFile)) + return code; + if (pNewCfg->compactStartTime != 0 && pNewCfg->compactEndTime != 0 && + pNewCfg->compactStartTime >= pNewCfg->compactEndTime) + return code; + if (pNewCfg->compactTimeOffset < TSDB_MIN_COMPACT_TIME_OFFSET || + pNewCfg->compactTimeOffset > TSDB_MAX_COMPACT_TIME_OFFSET) + return code; + code = 0; TAOS_RETURN(code); } @@ -1150,20 +1180,24 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { code = 0; } + bool compactTimeRangeChanged = false; if (pAlter->compactStartTime != pDb->cfg.compactStartTime && (pAlter->compactStartTime == TSDB_DEFAULT_COMPACT_START_TIME || pAlter->compactStartTime <= -pDb->cfg.daysPerFile)) { pDb->cfg.compactStartTime = pAlter->compactStartTime; - pDb->vgVersion++; + compactTimeRangeChanged = true; code = 0; } if (pAlter->compactEndTime != pDb->cfg.compactEndTime && (pAlter->compactEndTime == TSDB_DEFAULT_COMPACT_END_TIME || pAlter->compactEndTime <= -pDb->cfg.daysPerFile)) { pDb->cfg.compactEndTime = pAlter->compactEndTime; - pDb->vgVersion++; + compactTimeRangeChanged = true; code = 0; } + if(compactTimeRangeChanged) { + pDb->vgVersion++; + } if (pAlter->compactTimeOffset >= TSDB_MIN_COMPACT_TIME_OFFSET && pAlter->compactTimeOffset != pDb->cfg.compactTimeOffset) { @@ -1408,14 +1442,6 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->compactInterval = pDb->cfg.compactInterval; cfgRsp->compactStartTime = pDb->cfg.compactStartTime; cfgRsp->compactEndTime = pDb->cfg.compactEndTime; - if (cfgRsp->compactInterval > 0) { - if (cfgRsp->compactStartTime == 0) { - cfgRsp->compactStartTime = -cfgRsp->daysToKeep2; - } - if (cfgRsp->compactEndTime == 0) { - cfgRsp->compactEndTime = -cfgRsp->daysPerFile; - } - } cfgRsp->compactTimeOffset = pDb->cfg.compactTimeOffset; } @@ -2432,6 +2458,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)strictVstr, false), &lino, _OVER); + char durationStr[128] = {0}; char durationVstr[128] = {0}; int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], sizeof(durationVstr) - VARSTR_HEADER_SIZE, pDb->cfg.daysPerFile); @@ -2440,10 +2467,10 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER); - char keepVstr[512] = {0}; - char keep0Str[128] = {0}; - char keep1Str[128] = {0}; - char keep2Str[128] = {0}; + char keepVstr[128] = {0}; + char keep0Str[32] = {0}; + char keep1Str[32] = {0}; + char keep2Str[32] = {0}; int32_t lenKeep0 = formatDurationOrKeep(keep0Str, sizeof(keep0Str), pDb->cfg.daysToKeep0); int32_t lenKeep1 = formatDurationOrKeep(keep1Str, sizeof(keep1Str), pDb->cfg.daysToKeep1); @@ -2556,6 +2583,26 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, STR_WITH_MAXSIZE_TO_VARSTR(encryptAlgorithmVStr, encryptAlgorithmStr, 24); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)encryptAlgorithmVStr, false), &lino, _OVER); + + TAOS_UNUSED(formatDurationOrKeep(durationStr, sizeof(durationStr), pDb->cfg.compactInterval)); + STR_WITH_MAXSIZE_TO_VARSTR(durationVstr, durationStr, sizeof(durationVstr)); + if ((pColInfo = taosArrayGet(pBlock->pDataBlock, cols++))) { + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER); + } + + len = formatDurationOrKeep(durationStr, sizeof(durationStr), pDb->cfg.compactStartTime); + TAOS_UNUSED(formatDurationOrKeep(durationVstr, sizeof(durationVstr), pDb->cfg.compactEndTime)); + TAOS_UNUSED(snprintf(durationStr + len, sizeof(durationStr) - len, ",%s", durationVstr)); + STR_WITH_MAXSIZE_TO_VARSTR(durationVstr, durationStr, sizeof(durationVstr)); + if ((pColInfo = taosArrayGet(pBlock->pDataBlock, cols++))) { + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER); + } + + TAOS_UNUSED(snprintf(durationStr, sizeof(durationStr), "%dh", pDb->cfg.compactTimeOffset)); + STR_WITH_MAXSIZE_TO_VARSTR(durationVstr, durationStr, sizeof(durationVstr)); + if ((pColInfo = taosArrayGet(pBlock->pDataBlock, cols++))) { + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER); + } } _OVER: if (code != 0) mError("failed to retrieve at line:%d, since %s", lino, tstrerror(code)); diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c index 4fe4333534..68dc981338 100644 --- a/source/dnode/snode/src/snodeInitApi.c +++ b/source/dnode/snode/src/snodeInitApi.c @@ -68,7 +68,6 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateFillGetGroupKVByCur = streamStateFillGetGroupKVByCur; pStore->streamStateGetKVByCur = streamStateGetKVByCur; - pStore->streamStateSetFillInfo = streamStateSetFillInfo; pStore->streamStateClearExpiredState = streamStateClearExpiredState; pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; @@ -117,7 +116,6 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateBegin = streamStateBegin; pStore->streamStateCommit = streamStateCommit; pStore->streamStateDestroy = streamStateDestroy; - pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; pStore->streamStateReloadInfo = streamStateReloadInfo; pStore->streamStateCopyBackend = streamStateCopyBackend; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 73052c1e5e..3bfc50fcb2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -207,7 +207,7 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* (void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &(pRsp->reqOffset)); (void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &(pRsp->rspOffset)); - tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s,QID:0x%" PRIx64, + tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) start to send rsp, block num:%d, req:%s, rsp:%s,QID:0x%" PRIx64, vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); return tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index a2b6194375..937ca80bcc 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -197,7 +197,9 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { end: tDecoderClear(&dcoder); - return tbSuid == realTbSuid; + bool tmp = tbSuid == realTbSuid; + tqDebug("%s suid:%"PRId64" realSuid:%"PRId64" return:%d", __FUNCTION__, tbSuid, realTbSuid, tmp); + return tmp; } int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId) { @@ -262,6 +264,8 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t END: *fetchOffset = offset; + tqDebug("vgId:%d, end to fetch wal, code:%d , index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64 ", 0x%" PRIx64, + vgId, code, offset, lastVer, committedVer, appliedVer, id); return code; } @@ -273,6 +277,8 @@ bool tqGetTablePrimaryKey(STqReader* pReader) { } void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) { + tqDebug("%s:%p uid:%"PRId64, __FUNCTION__ , pReader, uid); + if (pReader == NULL) { return; } @@ -286,6 +292,7 @@ void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) { } STqReader* tqReaderOpen(SVnode* pVnode) { + tqDebug("%s:%p", __FUNCTION__ , pVnode); if (pVnode == NULL) { return NULL; } @@ -317,6 +324,7 @@ STqReader* tqReaderOpen(SVnode* pVnode) { } void tqReaderClose(STqReader* pReader) { + tqDebug("%s:%p", __FUNCTION__ , pReader); if (pReader == NULL) return; // close wal reader @@ -485,26 +493,25 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { } int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { -if (pReader == NULL) { + if (pReader == NULL) { return TSDB_CODE_INVALID_PARA; } pReader->msg.msgStr = msgStr; pReader->msg.msgLen = msgLen; pReader->msg.ver = ver; - tqDebug("tq reader set msg %p %d", msgStr, msgLen); + tqDebug("tq reader set msg pointer:%p, msg len:%d", msgStr, msgLen); SDecoder decoder = {0}; tDecoderInit(&decoder, pReader->msg.msgStr, pReader->msg.msgLen); int32_t code = tDecodeSubmitReq(&decoder, &pReader->submit); + tDecoderClear(&decoder); + if (code != 0) { - tDecoderClear(&decoder); tqError("DecodeSSubmitReq2 error, msgLen:%d, ver:%" PRId64, msgLen, ver); - return code; } - tDecoderClear(&decoder); - return 0; + return code; } SWalReader* tqGetWalReader(STqReader* pReader) { @@ -529,63 +536,63 @@ int64_t tqGetResultBlockTime(STqReader* pReader) { } bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { - if (pReader == NULL || pReader->msg.msgStr == NULL) { - return false; - } - - int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); - while (pReader->nextBlk < numOfBlocks) { - tqDebug("try next data block, len:%d ver:%" PRId64 " index:%d/%d, %s", pReader->msg.msgLen, pReader->msg.ver, - (pReader->nextBlk + 1), numOfBlocks, idstr); - - SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if (pSubmitTbData == NULL) { - return false; - } - if (pReader->tbIdHash == NULL) { - return true; - } - - void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); - if (ret != NULL) { - tqDebug("block found, ver:%" PRId64 ", uid:%" PRId64 ", %s", pReader->msg.ver, pSubmitTbData->uid, idstr); - return true; - } else { - tqDebug("discard submit block, uid:%" PRId64 ", total queried tables:%d continue %s", pSubmitTbData->uid, - taosHashGetSize(pReader->tbIdHash), idstr); - } - - pReader->nextBlk++; - } - - tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->nextBlk = 0; - pReader->msg.msgStr = NULL; - - return false; -} - -bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) { - if (pReader == NULL || pReader->msg.msgStr == NULL) return false; + int32_t code = false; + int32_t lino = 0; + int64_t uid = 0; + TSDB_CHECK_NULL(pReader, code, lino, END, false); + TSDB_CHECK_NULL(pReader->msg.msgStr, code, lino, END, false); + TSDB_CHECK_NULL(pReader->tbIdHash, code, lino, END, true); int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if (pSubmitTbData == NULL) return false; - if (filterOutUids == NULL) return true; + TSDB_CHECK_NULL(pSubmitTbData, code, lino, END, false); + uid = pSubmitTbData->uid; + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + TSDB_CHECK_CONDITION(ret == NULL, code, lino, END, true); - void* ret = taosHashGet(filterOutUids, &pSubmitTbData->uid, sizeof(int64_t)); - if (ret == NULL) { - return true; - } + tqDebug("iterator data block in hash continue, progress:%d/%d, total queried tables:%d, uid:%"PRId64, pReader->nextBlk, blockSz, taosHashGetSize(pReader->tbIdHash), uid); pReader->nextBlk++; } tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); pReader->nextBlk = 0; pReader->msg.msgStr = NULL; + tqDebug("iterator data block end, block progress:%d/%d, uid:%"PRId64, pReader->nextBlk, blockSz, uid); - return false; +END: + tqDebug("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); + return code; +} + +bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) { + int32_t code = false; + int32_t lino = 0; + int64_t uid = 0; + + TSDB_CHECK_NULL(pReader, code, lino, END, false); + TSDB_CHECK_NULL(pReader->msg.msgStr, code, lino, END, false); + TSDB_CHECK_NULL(filterOutUids, code, lino, END, true); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + TSDB_CHECK_NULL(pSubmitTbData, code, lino, END, false); + uid = pSubmitTbData->uid; + void* ret = taosHashGet(filterOutUids, &pSubmitTbData->uid, sizeof(int64_t)); + TSDB_CHECK_NULL(ret, code, lino, END, true); + tqDebug("iterator data block in hash continue, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid); + pReader->nextBlk++; + } + + tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->nextBlk = 0; + pReader->msg.msgStr = NULL; + tqDebug("iterator data block end, block progress:%d/%d, uid:%"PRId64, pReader->nextBlk, blockSz, uid); + +END: + tqDebug("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); + return code; } int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) { @@ -709,7 +716,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* if (pReader == NULL || pRes == NULL) { return TSDB_CODE_INVALID_PARA; } - tqTrace("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); + tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); int32_t code = 0; int32_t line = 0; STSchema* pTSchema = NULL; @@ -846,7 +853,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* END: if (code != 0) { - tqError("tqRetrieveDataBlock failed, line:%d, code:%d", line, code); + tqError("tqRetrieveDataBlock failed, line:%d, msg:%s", line, tstrerror(code)); } taosMemoryFreeClear(pTSchema); return code; @@ -876,10 +883,6 @@ END: static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas, SSchemaWrapper* pSchemaWrapper, char* assigned, int32_t numOfRows, int32_t curRow, int32_t* lastRow) { - if (pReader == NULL || pSubmitTbData == NULL || blocks == NULL || schemas == NULL || pSchemaWrapper == NULL || - assigned == NULL || lastRow == NULL) { - return TSDB_CODE_INVALID_PARA; - } int32_t code = 0; SSchemaWrapper* pSW = NULL; SSDataBlock* block = NULL; @@ -909,15 +912,15 @@ static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, taosMemoryFreeClear(block); END: + if (code != 0) { + tqError("processBuildNew failed, code:%d", code); + } tDeleteSchemaWrapper(pSW); blockDataFreeRes(block); taosMemoryFree(block); return code; } static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) { - if (pReader == NULL || pSubmitTbData == NULL || blocks == NULL || schemas == NULL) { - return TSDB_CODE_INVALID_PARA; - } int32_t code = 0; int32_t curRow = 0; int32_t lastRow = 0; @@ -931,6 +934,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData TQ_NULL_GO_TO_END(pCol); int32_t numOfRows = pCol->nVal; int32_t numOfCols = taosArrayGetSize(pCols); + tqDebug("vgId:%d, tqProcessColData start, col num: %d, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows); for (int32_t i = 0; i < numOfRows; i++) { bool buildNew = false; @@ -970,16 +974,16 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData } SSDataBlock* pLastBlock = taosArrayGetLast(blocks); pLastBlock->info.rows = curRow - lastRow; - + tqDebug("vgId:%d, tqProcessColData end, col num: %d, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows, (int)taosArrayGetSize(blocks)); END: + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d, process col data failed, code:%d", pReader->pWalReader->pWal->cfg.vgId, code); + } taosMemoryFree(assigned); return code; } int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) { - if (pReader == NULL || pSubmitTbData == NULL || blocks == NULL || schemas == NULL) { - return TSDB_CODE_INVALID_PARA; - } int32_t code = 0; STSchema* pTSchema = NULL; @@ -992,6 +996,8 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra SArray* pRows = pSubmitTbData->aRowP; int32_t numOfRows = taosArrayGetSize(pRows); pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); + TQ_NULL_GO_TO_END(pTSchema); + tqDebug("vgId:%d, tqProcessRowData start, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows); for (int32_t i = 0; i < numOfRows; i++) { bool buildNew = false; @@ -1030,17 +1036,18 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra SSDataBlock* pLastBlock = taosArrayGetLast(blocks); pLastBlock->info.rows = curRow - lastRow; + tqDebug("vgId:%d, tqProcessRowData end, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows, (int)taosArrayGetSize(blocks)); END: + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d, process row data failed, code:%d", pReader->pWalReader->pWal->cfg.vgId, code); + } taosMemoryFreeClear(pTSchema); taosMemoryFree(assigned); return code; } int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, int64_t *createTime) { - if (pReader == NULL || blocks == NULL || schemas == NULL) { - return TSDB_CODE_INVALID_PARA; - } - tqTrace("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk); + tqDebug("tq reader retrieve data block msg pointer:%p, index:%d", pReader->msg.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pSubmitTbData == NULL) { return terrno; diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 3419cd0020..b4dc610a6a 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -16,15 +16,16 @@ #include "tq.h" int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { - if (pBlock == NULL || pRsp == NULL) { - return TSDB_CODE_INVALID_PARA; - } + int32_t code = TDB_CODE_SUCCESS; + int32_t lino = 0; + void* buf = NULL; + TSDB_CHECK_NULL(pBlock, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); + size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + dataEncodeBufSize; - void* buf = taosMemoryCalloc(1, dataStrLen); - if (buf == NULL) { - return terrno; - } + buf = taosMemoryCalloc(1, dataStrLen); + TSDB_CHECK_NULL(buf, code, lino, END, terrno); SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)buf; pRetrieve->version = 1; @@ -33,49 +34,49 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); int32_t actualLen = blockEncode(pBlock, pRetrieve->data, dataEncodeBufSize, numOfCols); - if(actualLen < 0){ - taosMemoryFree(buf); - return terrno; - } + TSDB_CHECK_CONDITION(actualLen >= 0, code, lino, END, terrno); actualLen += sizeof(SRetrieveTableRspForTmq); - if (taosArrayPush(pRsp->blockDataLen, &actualLen) == NULL){ - taosMemoryFree(buf); - return terrno; - } - if (taosArrayPush(pRsp->blockData, &buf) == NULL) { - taosMemoryFree(buf); - return terrno; - } + TSDB_CHECK_NULL(taosArrayPush(pRsp->blockDataLen, &actualLen), code, lino, END, terrno); + TSDB_CHECK_NULL(taosArrayPush(pRsp->blockData, &buf), code, lino, END, terrno); - return TSDB_CODE_SUCCESS; + tqDebug("add block data to block array, blockDataLen:%d, blockData:%p", actualLen, buf); +END: + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + tqError("%s failed at %d, failed to add block data to response:%s", __FUNCTION__, lino, tstrerror(code)); + } + return code; } static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t n) { - if (pRsp == NULL || pTq == NULL) { - return TSDB_CODE_INVALID_PARA; - } + int32_t code = TDB_CODE_SUCCESS; + int32_t lino = 0; SMetaReader mr = {0}; + + TSDB_CHECK_NULL(pTq, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); + metaReaderDoInit(&mr, pTq->pVnode->pMeta, META_READER_LOCK); - int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); - if (code < 0) { - metaReaderClear(&mr); - return code; - } + code = metaReaderGetTableEntryByUidCache(&mr, uid); + TSDB_CHECK_CODE(code, lino, END); for (int32_t i = 0; i < n; i++) { char* tbName = taosStrdup(mr.me.name); - if (tbName == NULL) { - metaReaderClear(&mr); - return terrno; - } + TSDB_CHECK_NULL(tbName, code, lino, END, terrno); if(taosArrayPush(pRsp->blockTbName, &tbName) == NULL){ - tqError("failed to push tbName to blockTbName:%s", tbName); + tqError("failed to push tbName to blockTbName:%s, uid:%"PRId64, tbName, uid); continue; } + tqDebug("add tbName to response success tbname:%s, uid:%"PRId64, tbName, uid); + } + +END: + if (code != TSDB_CODE_SUCCESS) { + tqError("%s failed at %d, failed to add tbName to response:%s, uid:%"PRId64, __FUNCTION__, lino, tstrerror(code), uid); } metaReaderClear(&mr); - return 0; + return code; } int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, SSDataBlock** res) { @@ -96,109 +97,130 @@ int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, S return 0; } -int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest) { - if (pTq == NULL || pHandle == NULL || pRsp == NULL || pOffset == NULL || pRequest == NULL){ - return TSDB_CODE_INVALID_PARA; - } - int32_t vgId = TD_VID(pTq->pVnode); +static int32_t tqProcessReplayRsp(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, const SMqPollReq* pRequest, SSDataBlock* pDataBlock, qTaskInfo_t task){ int32_t code = 0; - int32_t line = 0; + int32_t lino = 0; + + if (IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type) && pHandle->block != NULL) { + blockDataDestroy(pHandle->block); + pHandle->block = NULL; + } + if (pHandle->block == NULL) { + if (pDataBlock == NULL) { + goto END; + } + + STqOffsetVal offset = {0}; + code = qStreamExtractOffset(task, &offset); + TSDB_CHECK_CODE(code, lino, END); + + pHandle->block = NULL; + + code = createOneDataBlock(pDataBlock, true, &pHandle->block); + TSDB_CHECK_CODE(code, lino, END); + + pHandle->blockTime = offset.ts; + tOffsetDestroy(&offset); + int32_t vgId = TD_VID(pTq->pVnode); + code = getDataBlock(task, pHandle, vgId, &pDataBlock); + TSDB_CHECK_CODE(code, lino, END); + } + + const STqExecHandle* pExec = &pHandle->execHandle; + code = tqAddBlockDataToRsp(pHandle->block, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + TSDB_CHECK_CODE(code, lino, END); + + pRsp->blockNum++; + if (pDataBlock == NULL) { + blockDataDestroy(pHandle->block); + pHandle->block = NULL; + } else { + code = copyDataBlock(pHandle->block, pDataBlock); + TSDB_CHECK_CODE(code, lino, END); + + STqOffsetVal offset = {0}; + code = qStreamExtractOffset(task, &offset); + TSDB_CHECK_CODE(code, lino, END); + + pRsp->sleepTime = offset.ts - pHandle->blockTime; + pHandle->blockTime = offset.ts; + tOffsetDestroy(&offset); + } + +END: + if (code != TSDB_CODE_SUCCESS) { + tqError("%s failed at %d, failed to process replay response:%s", __FUNCTION__, lino, tstrerror(code)); + } + return code; +} + +int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest) { + int32_t code = 0; + int32_t lino = 0; + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pTq, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pHandle, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pOffset, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pRequest, code, lino, END, TSDB_CODE_INVALID_PARA); + + int32_t vgId = TD_VID(pTq->pVnode); int32_t totalRows = 0; const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; code = qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); - TSDB_CHECK_CODE(code, line, END); + TSDB_CHECK_CODE(code, lino, END); qStreamSetSourceExcluded(task, pRequest->sourceExcluded); uint64_t st = taosGetTimestampMs(); while (1) { SSDataBlock* pDataBlock = NULL; code = getDataBlock(task, pHandle, vgId, &pDataBlock); - TSDB_CHECK_CODE(code, line, END); + TSDB_CHECK_CODE(code, lino, END); if (pRequest->enableReplay) { - if (IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type) && pHandle->block != NULL) { - blockDataDestroy(pHandle->block); - pHandle->block = NULL; - } - if (pHandle->block == NULL) { - if (pDataBlock == NULL) { - break; - } - - STqOffsetVal offset = {0}; - code = qStreamExtractOffset(task, &offset); - TSDB_CHECK_CODE(code, line, END); - - pHandle->block = NULL; - - code = createOneDataBlock(pDataBlock, true, &pHandle->block); - TSDB_CHECK_CODE(code, line, END); - - pHandle->blockTime = offset.ts; - tOffsetDestroy(&offset); - code = getDataBlock(task, pHandle, vgId, &pDataBlock); - TSDB_CHECK_CODE(code, line, END); - } - - code = tqAddBlockDataToRsp(pHandle->block, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); - TSDB_CHECK_CODE(code, line, END); - - pRsp->blockNum++; - if (pDataBlock == NULL) { - blockDataDestroy(pHandle->block); - pHandle->block = NULL; - } else { - code = copyDataBlock(pHandle->block, pDataBlock); - TSDB_CHECK_CODE(code, line, END); - - STqOffsetVal offset = {0}; - code = qStreamExtractOffset(task, &offset); - TSDB_CHECK_CODE(code, line, END); - - pRsp->sleepTime = offset.ts - pHandle->blockTime; - pHandle->blockTime = offset.ts; - tOffsetDestroy(&offset); - } + code = tqProcessReplayRsp(pTq, pHandle, pRsp, pRequest, pDataBlock, task); + TSDB_CHECK_CODE(code, lino, END); break; - } else { - if (pDataBlock == NULL) { - break; - } - code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); - TSDB_CHECK_CODE(code, line, END); + } + if (pDataBlock == NULL) { + break; + } + code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + TSDB_CHECK_CODE(code, lino, END); - pRsp->blockNum++; - totalRows += pDataBlock->info.rows; - if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { - break; - } + pRsp->blockNum++; + totalRows += pDataBlock->info.rows; + if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { + break; } } - tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", - pHandle->consumerId, vgId, pRsp->blockNum, totalRows); + tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", pHandle->consumerId, vgId, pRsp->blockNum, totalRows); code = qStreamExtractOffset(task, &pRsp->rspOffset); + END: if (code != 0) { - tqError("consumer:0x%" PRIx64 " vgId:%d tmq task executed error, line:%d code:%d", pHandle->consumerId, vgId, line, - code); + tqError("%s failed at %d, tmq task executed error msg:%s", __FUNCTION__, lino, tstrerror(code)); } return code; } int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset) { - if (pTq == NULL || pHandle == NULL || pRsp == NULL || pBatchMetaRsp == NULL || pOffset == NULL) { - return TSDB_CODE_INVALID_PARA; - } + int32_t code = 0; + int32_t lino = 0; + char* tbName = NULL; + SSchemaWrapper* pSW = NULL; + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pTq, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pHandle, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pOffset, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pBatchMetaRsp, code, lino, END, TSDB_CODE_INVALID_PARA); const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; - int code = qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); - if (code != 0) { - return code; - } + code = qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); + TSDB_CHECK_CODE(code, lino, END); int32_t rowCnt = 0; while (1) { @@ -206,52 +228,37 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat uint64_t ts = 0; tqDebug("tmqsnap task start to execute"); code = qExecTask(task, &pDataBlock, &ts); - if (code != 0) { - tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, tstrerror(code)); - return code; - } - + TSDB_CHECK_CODE(code, lino, END); tqDebug("tmqsnap task execute end, get %p", pDataBlock); if (pDataBlock != NULL && pDataBlock->info.rows > 0) { if (pRsp->withTbName) { - char* tbName = taosStrdup(qExtractTbnameFromTask(task)); - if (tbName == NULL) { - tqError("vgId:%d, failed to add tbname to rsp msg, null", pTq->pVnode->config.vgId); - return terrno; - } - if (taosArrayPush(pRsp->blockTbName, &tbName) == NULL){ - tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId); - continue; - } + tbName = taosStrdup(qExtractTbnameFromTask(task)); + TSDB_CHECK_NULL(tbName, code, lino, END, terrno); + TSDB_CHECK_NULL(taosArrayPush(pRsp->blockTbName, &tbName), code, lino, END, terrno); + tbName = NULL; } if (pRsp->withSchema) { - SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); - if(taosArrayPush(pRsp->blockSchema, &pSW) == NULL){ - tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); - continue; - } + pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); + TSDB_CHECK_NULL(pSW, code, lino, END, terrno); + TSDB_CHECK_NULL(taosArrayPush(pRsp->blockSchema, &pSW), code, lino, END, terrno); + pSW = NULL; } - if (tqAddBlockDataToRsp(pDataBlock, pRsp, taosArrayGetSize(pDataBlock->pDataBlock), - pTq->pVnode->config.tsdbCfg.precision) != 0) { - tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); - continue; - } + code = tqAddBlockDataToRsp(pDataBlock, pRsp, taosArrayGetSize(pDataBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision); + TSDB_CHECK_CODE(code, lino, END); pRsp->blockNum++; rowCnt += pDataBlock->info.rows; - if (rowCnt <= tmqRowSize) continue; - + if (rowCnt <= tmqRowSize) { + continue; + } } // get meta SMqBatchMetaRsp* tmp = qStreamExtractMetaMsg(task); if (taosArrayGetSize(tmp->batchMetaReq) > 0) { code = qStreamExtractOffset(task, &tmp->rspOffset); - if (code) { - return code; - } - + TSDB_CHECK_CODE(code, lino, END); *pBatchMetaRsp = *tmp; tqDebug("tmqsnap task get meta"); break; @@ -259,16 +266,13 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat if (pDataBlock == NULL) { code = qStreamExtractOffset(task, pOffset); - if (code) { - break; - } + TSDB_CHECK_CODE(code, lino, END); if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { continue; } - tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), - pHandle->snapshotVer + 1); + tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); code = qStreamExtractOffset(task, &pRsp->rspOffset); break; } @@ -280,119 +284,93 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat } } + tqDebug("%s:%d success", __FUNCTION__, lino); +END: + if (code != 0){ + tqDebug("%s:%d failed, code:%s", __FUNCTION__, lino, tstrerror(code) ); + } + taosMemoryFree(pSW); + taosMemoryFree(tbName); return code; } static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){ - if (pRsp == NULL || pCreateTbReq == NULL) { - return TSDB_CODE_INVALID_PARA; - } int32_t code = 0; + int32_t lino = 0; void* createReq = NULL; + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pCreateTbReq, code, lino, END, TSDB_CODE_INVALID_PARA); + if (pRsp->createTableNum == 0) { pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); - if (pRsp->createTableLen == NULL) { - code = terrno; - goto END; - } + TSDB_CHECK_NULL(pRsp->createTableLen, code, lino, END, terrno); pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); - if (pRsp->createTableReq == NULL) { - code = terrno; - goto END; - } + TSDB_CHECK_NULL(pRsp->createTableReq, code, lino, END, terrno); } uint32_t len = 0; tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code); - if (TSDB_CODE_SUCCESS != code) { - goto END; - } + TSDB_CHECK_CODE(code, lino, END); createReq = taosMemoryCalloc(1, len); - if (createReq == NULL){ - code = terrno; - goto END; - } + TSDB_CHECK_NULL(createReq, code, lino, END, terrno); + SEncoder encoder = {0}; tEncoderInit(&encoder, createReq, len); code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq); tEncoderClear(&encoder); - if (code < 0) { - goto END; - } - if (taosArrayPush(pRsp->createTableLen, &len) == NULL){ - code = terrno; - goto END; - } - if (taosArrayPush(pRsp->createTableReq, &createReq) == NULL){ - code = terrno; - goto END; - } + TSDB_CHECK_CODE(code, lino, END); + TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableLen, &len), code, lino, END, terrno); + TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableReq, &createReq), code, lino, END, terrno); pRsp->createTableNum++; + tqDebug("build create table info msg success"); - return 0; END: - taosMemoryFree(createReq); + if (code != 0){ + tqError("%s failed at %d, failed to build create table info msg:%s", __FUNCTION__, lino, tstrerror(code)); + taosMemoryFree(createReq); + } return code; } static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){ - if (pTq == NULL || pHandle == NULL || pRsp == NULL || totalRows == NULL) { - return; - } int32_t code = 0; - STqExecHandle* pExec = &pHandle->execHandle; - STqReader* pReader = pExec->pTqReader; + int32_t lino = 0; SArray* pBlocks = NULL; SArray* pSchemas = NULL; + + STqExecHandle* pExec = &pHandle->execHandle; + STqReader* pReader = pExec->pTqReader; + pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); - if (pBlocks == NULL) { - code = terrno; - goto END; - } + TSDB_CHECK_NULL(pBlocks, code, lino, END, terrno); pSchemas = taosArrayInit(0, sizeof(void*)); - if(pSchemas == NULL){ - code = terrno; - goto END; - } + TSDB_CHECK_NULL(pSchemas, code, lino, END, terrno); SSubmitTbData* pSubmitTbDataRet = NULL; int64_t createTime = INT64_MAX; code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet, &createTime); - if (code != 0) { - tqError("vgId:%d, failed to retrieve block", pTq->pVnode->config.vgId); - goto END; - } - - if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { - goto END; - } + TSDB_CHECK_CODE(code, lino, END); + bool tmp = (pSubmitTbDataRet->flags & sourceExcluded) != 0; + TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS); if (pRsp->withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; code = tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)); - if (code != 0) { - tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId); - goto END; - } + TSDB_CHECK_CODE(code, lino, END); } if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) { if (pSubmitTbDataRet->ctimeMs - createTime <= 1000) { // judge if table is already created to avoid sending crateTbReq code = buildCreateTbInfo(pRsp, pSubmitTbDataRet->pCreateTbReq); - if (code != 0){ - tqError("vgId:%d, failed to build create table info", pTq->pVnode->config.vgId); - goto END; - } + TSDB_CHECK_CODE(code, lino, END); } } - if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL) { - goto END; - } + tmp = (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL); + TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS); for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); if (pBlock == NULL) { continue; } - if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), - pTq->pVnode->config.tsdbCfg.precision) != 0){ + if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0){ tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); continue; } @@ -405,28 +383,29 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int } pRsp->blockNum++; } - - taosArrayDestroy(pBlocks); - taosArrayDestroy(pSchemas); - return; - + tqDebug("vgId:%d, process sub data success, response blocknum:%d, rows:%d", pTq->pVnode->config.vgId, pRsp->blockNum, *totalRows); END: - taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); - taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); + if (code != 0){ + tqError("%s failed at %d, failed to process sub data:%s", __FUNCTION__, lino, tstrerror(code)); + taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); + taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); + } else { + taosArrayDestroy(pBlocks); + taosArrayDestroy(pSchemas); + } } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, - int8_t sourceExcluded) { - if (pTq == NULL || pHandle == NULL || pRsp == NULL || totalRows == NULL) { - return TSDB_CODE_INVALID_PARA; - } +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) { + int32_t code = 0; + int32_t lino = 0; + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pTq, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pHandle, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(totalRows, code, lino, END, TSDB_CODE_INVALID_PARA); STqExecHandle* pExec = &pHandle->execHandle; - int32_t code = 0; STqReader* pReader = pExec->pTqReader; code = tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); - if (code != 0) { - return code; - } + TSDB_CHECK_CODE(code, lino, END); if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { while (tqNextBlockImpl(pReader, NULL)) { @@ -438,5 +417,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqData } } +END: + if (code != 0){ + tqError("%s failed at %d, failed to scan log:%s", __FUNCTION__, lino, tstrerror(code)); + } return code; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 5405ace89b..7ba77cf813 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -1047,8 +1047,6 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat pTableSinkInfo->uid = 0; code = doPutSinkTableInfoIntoCache(pTask->outputInfo.tbSink.pTbInfo, pTableSinkInfo, groupId, id); } else { - metaReaderClear(&mr); - tqError("s-task:%s vgId:%d dst-table:%s not auto-created, and not create in tsdb, discard data", id, vgId, dstTableName); return TSDB_CODE_TDB_TABLE_NOT_EXIST; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index f6a8563c70..6ecb1b1b4d 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -21,21 +21,27 @@ static int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, c const SMqBatchMetaRsp* pRsp, int32_t vgId); int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { - if (pRsp == NULL) { - return TSDB_CODE_INVALID_PARA; - } - pRsp->blockData = taosArrayInit(0, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + int32_t code = TDB_CODE_SUCCESS; + int32_t lino = 0; + tqDebug("%s called", __FUNCTION__ ); + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); - if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) { - return terrno; - } + pRsp->blockData = taosArrayInit(0, sizeof(void*)); + TSDB_CHECK_NULL(pRsp->blockData, code, lino, END, terrno); + + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + TSDB_CHECK_NULL(pRsp->blockDataLen, code, lino, END, terrno); tOffsetCopy(&pRsp->reqOffset, &pOffset); tOffsetCopy(&pRsp->rspOffset, &pOffset); pRsp->withTbName = 0; pRsp->withSchema = false; - return 0; + +END: + if (code != 0){ + tqError("%s failed at:%d, code:%s", __FUNCTION__ , lino, tstrerror(code)); + } + return code; } void tqUpdateNodeStage(STQ* pTq, bool isLeader) { @@ -44,45 +50,37 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { } static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { - if (pRsp == NULL) { - return TSDB_CODE_INVALID_PARA; - } + int32_t code = TDB_CODE_SUCCESS; + int32_t lino = 0; + tqDebug("%s called", __FUNCTION__ ); + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); tOffsetCopy(&pRsp->reqOffset, &pOffset); tOffsetCopy(&pRsp->rspOffset, &pOffset); pRsp->withTbName = 1; pRsp->withSchema = 1; pRsp->blockData = taosArrayInit(0, sizeof(void*)); + TSDB_CHECK_NULL(pRsp->blockData, code, lino, END, terrno);\ + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + TSDB_CHECK_NULL(pRsp->blockDataLen, code, lino, END, terrno); + pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); + TSDB_CHECK_NULL(pRsp->blockTbName, code, lino, END, terrno); + pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); + TSDB_CHECK_NULL(pRsp->blockSchema, code, lino, END, terrno); - if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || - pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { - if (pRsp->blockData != NULL) { - taosArrayDestroy(pRsp->blockData); - pRsp->blockData = NULL; - } - if (pRsp->blockDataLen != NULL) { - taosArrayDestroy(pRsp->blockDataLen); - pRsp->blockDataLen = NULL; - } - - if (pRsp->blockTbName != NULL) { - taosArrayDestroy(pRsp->blockTbName); - pRsp->blockTbName = NULL; - } - - if (pRsp->blockSchema != NULL) { - taosArrayDestroy(pRsp->blockSchema); - pRsp->blockSchema = NULL; - } - - return terrno; +END: + if (code != 0){ + tqError("%s failed at:%d, code:%s", __FUNCTION__ , lino, tstrerror(code)); + taosArrayDestroy(pRsp->blockData); + taosArrayDestroy(pRsp->blockDataLen); + taosArrayDestroy(pRsp->blockTbName); + taosArrayDestroy(pRsp->blockSchema); } - - return 0; + return code; } static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, @@ -151,23 +149,19 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* pOffset) { - if (pTq == NULL || pHandle == NULL || pRequest == NULL || pMsg == NULL || pOffset == NULL) { - return TSDB_CODE_INVALID_PARA; - } + int32_t code = TDB_CODE_SUCCESS; + int32_t lino = 0; + tqDebug("%s called", __FUNCTION__ ); uint64_t consumerId = pRequest->consumerId; int32_t vgId = TD_VID(pTq->pVnode); terrno = 0; SMqDataRsp dataRsp = {0}; - int code = tqInitDataRsp(&dataRsp, *pOffset); - if (code != 0) { - goto end; - } + code = tqInitDataRsp(&dataRsp, *pOffset); + TSDB_CHECK_CODE(code, lino, end); code = qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); - if (code != 0) { - goto end; - } + TSDB_CHECK_CODE(code, lino, end); code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest); if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { @@ -191,15 +185,21 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, tOffsetCopy(&dataRsp.reqOffset, pOffset); code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); -end : { - char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.rspOffset); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s,QID:0x%" PRIx64 - " code:%d", - consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); - tDeleteMqDataRsp(&dataRsp); - return code; -} +end: + { + char buf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.rspOffset); + if (code != 0){ + tqError("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s,QID:0x%" PRIx64 " error msg:%s, line:%d", + consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, tstrerror(code), lino); + } else { + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s,QID:0x%" PRIx64 " success", + consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId); + } + + tDeleteMqDataRsp(&dataRsp); + return code; + } } #define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC, DELETE_FUNC) \ @@ -224,9 +224,6 @@ static void tDeleteCommon(void* parm) {} static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { - if (pTq == NULL || pHandle == NULL || pRequest == NULL || pMsg == NULL || offset == NULL) { - return TSDB_CODE_INVALID_PARA; - } int32_t vgId = TD_VID(pTq->pVnode); SMqDataRsp taosxRsp = {0}; SMqBatchMetaRsp btMetaRsp = {0}; @@ -238,17 +235,13 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (taosArrayGetSize(btMetaRsp.batchMetaReq) > 0) { code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); - tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 - ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, btMetaRsp.rspOffset.type, btMetaRsp.rspOffset.uid, - btMetaRsp.rspOffset.ts); + tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, btMetaRsp.rspOffset.type, btMetaRsp.rspOffset.uid,btMetaRsp.rspOffset.ts); goto END; } - tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 - ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, - taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts); + tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64",ts:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts); if (taosxRsp.blockNum > 0) { code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto END; @@ -277,30 +270,23 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (totalMetaRows > 0) { tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); - if (totalRows != 0) { - tqError("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, totalRows error, vgId:%d offset %" PRId64, - pRequest->consumerId, pRequest->epoch, vgId, fetchVer); - code = code == 0 ? TSDB_CODE_TQ_INTERNAL_ERROR : code; - } goto END; } tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp( - pHandle, pMsg, pRequest, &taosxRsp, + code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto END; } SWalCont* pHead = &pHandle->pWalReader->pHead->head; - tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", - pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %s", + pRequest->consumerId, pRequest->epoch, vgId, fetchVer, TMSG_INFO(pHead->msgType)); // process meta if (pHead->msgType != TDMT_VND_SUBMIT) { if (totalRows > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp( - pHandle, pMsg, pRequest, &taosxRsp, + code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto END; } @@ -331,15 +317,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (!btMetaRsp.batchMetaReq) { btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES); - if (btMetaRsp.batchMetaReq == NULL) { - code = TAOS_GET_TERRNO(terrno); - goto END; - } + TQ_NULL_GO_TO_END(btMetaRsp.batchMetaReq); btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t)); - if (btMetaRsp.batchMetaLen == NULL) { - code = TAOS_GET_TERRNO(terrno); - goto END; - } + TQ_NULL_GO_TO_END(btMetaRsp.batchMetaLen); } fetchVer++; @@ -355,10 +335,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } int32_t tLen = sizeof(SMqRspHead) + len; void* tBuf = taosMemoryCalloc(1, tLen); - if (tBuf == NULL) { - code = TAOS_GET_TERRNO(terrno); - goto END; - } + TQ_NULL_GO_TO_END(tBuf); void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead)); SEncoder encoder = {0}; tEncoderInit(&encoder, metaBuff, len); @@ -369,14 +346,8 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); continue; } - if (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf) == NULL) { - code = TAOS_GET_TERRNO(terrno); - goto END; - } - if (taosArrayPush(btMetaRsp.batchMetaLen, &tLen) == NULL) { - code = TAOS_GET_TERRNO(terrno); - goto END; - } + TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf)); + TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp.batchMetaLen, &tLen)); totalMetaRows++; if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > 1000)) { tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); @@ -399,17 +370,11 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, .ver = pHead->version, }; - code = tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest->sourceExcluded); - if (code < 0) { - tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, - pRequest->subKey); - goto END; - } + TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest->sourceExcluded)); if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); - code = tqSendDataRsp( - pHandle, pMsg, pRequest, &taosxRsp, + code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto END; } else { @@ -419,6 +384,10 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } END: + if (code != 0){ + tqError("tmq poll: tqTaosxScanLog error. consumerId:0x%" PRIx64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, + pRequest->subKey); + } tDeleteMqBatchMetaRsp(&btMetaRsp); tDeleteSTaosxRsp(&taosxRsp); return code; @@ -457,6 +426,9 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ } END: + if (code != 0){ + uError("failed to extract data for mq, msg:%s", tstrerror(code)); + } tOffsetDestroy(&reqOffset); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 82dd49b0e2..aa68a8af5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -680,7 +680,7 @@ static int32_t fset_cmpr_fn(const struct STFileSet *pSet1, const struct STFileSe return 0; } -static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { +static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) { int32_t code = 0; int32_t lino = 0; @@ -690,6 +690,8 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { TFileSetArray *fsetArray = fs->fSetArrTmp; STFileSet *fset = NULL; const STFileOp *op; + int32_t fid = INT32_MIN; + TSKEY now = taosGetTimestampMs(); TARRAY2_FOREACH_PTR(opArray, op) { if (!fset || fset->fid != op->fid) { STFileSet tfset = {.fid = op->fid}; @@ -708,6 +710,15 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { code = tsdbTFileSetEdit(fs->tsdb, fset, op); TSDB_CHECK_CODE(code, lino, _exit); + + if (fid != op->fid) { + fid = op->fid; + if (etype == TSDB_FEDIT_COMMIT) { + fset->lastCommit = now; + } else if (etype == TSDB_FEDIT_COMPACT) { + fset->lastCompact = now; + } + } } // remove empty empty stt level and empty file set @@ -864,7 +875,7 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e fs->etype = etype; // edit - code = edit_fs(fs, opArray); + code = edit_fs(fs, opArray, etype); TSDB_CHECK_CODE(code, lino, _exit); // save fs @@ -1288,6 +1299,12 @@ int32_t tsdbFileSetReaderOpen(void *pVnode, struct SFileSetReader **ppReader) { return TSDB_CODE_SUCCESS; } +extern bool tsdbShouldCompact(const STFileSet *pFileSet); + +#ifndef TD_ENTERPRISE +bool tsdbShouldCompact(const STFileSet *pFileSet) { return false; } +#endif + static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) { STsdb *pTsdb = pReader->pTsdb; int32_t code = TSDB_CODE_SUCCESS; @@ -1311,7 +1328,7 @@ static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) { // get file set details pReader->fid = pReader->pFileSet->fid; tsdbFidKeyRange(pReader->fid, pTsdb->keepCfg.days, pTsdb->keepCfg.precision, &pReader->startTime, &pReader->endTime); - pReader->lastCompactTime = 0; // TODO + pReader->lastCompactTime = pReader->pFileSet->lastCompact; pReader->totalSize = 0; for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) { STFileObj *fobj = pReader->pFileSet->farr[i]; @@ -1375,7 +1392,7 @@ int32_t tsdbFileSetGetEntryField(struct SFileSetReader *pReader, const char *fie fieldName = "should_compact"; if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { - *(char *)value = 0; // TODO + *(char *)value = tsdbShouldCompact(pReader->pFileSet); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 68914300e4..6530ee2269 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -273,6 +273,15 @@ int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json) { if (code) return code; } + // about compact and commit + if (cJSON_AddNumberToObject(json, "last compact", fset->lastCompact) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (cJSON_AddNumberToObject(json, "last commit", fset->lastCommit) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return 0; } @@ -324,6 +333,20 @@ int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset) { } else { return TSDB_CODE_FILE_CORRUPTED; } + // about compact and commit + item1 = cJSON_GetObjectItem(json, "last compact"); + if (cJSON_IsNumber(item1)) { + (*fset)->lastCompact = item1->valuedouble; + } else { + (*fset)->lastCompact = 0; + } + + item1 = cJSON_GetObjectItem(json, "last commit"); + if (cJSON_IsNumber(item1)) { + (*fset)->lastCommit = item1->valuedouble; + } else { + (*fset)->lastCommit = 0; + } return 0; } @@ -467,6 +490,9 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f } } + fset2->lastCompact = fset1->lastCompact; + fset2->lastCommit = fset1->lastCommit; + return 0; } @@ -522,6 +548,9 @@ int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **f if (code) return code; } + (*fset)->lastCompact = fset1->lastCompact; + (*fset)->lastCommit = fset1->lastCommit; + return 0; } @@ -617,6 +646,9 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs } } + (*fset)->lastCompact = fset1->lastCompact; + (*fset)->lastCommit = fset1->lastCommit; + return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 83ef32e5e5..ca9c133e9c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -92,6 +92,8 @@ struct STFileSet { int64_t maxVerValid; STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array + TSKEY lastCompact; + TSKEY lastCommit; bool mergeScheduled; SVATaskID mergeTask; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index df6fb17730..41e6c6c2c5 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -191,7 +191,6 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateFillGetGroupKVByCur = streamStateFillGetGroupKVByCur; pStore->streamStateGetKVByCur = streamStateGetKVByCur; - pStore->streamStateSetFillInfo = streamStateSetFillInfo; pStore->streamStateClearExpiredState = streamStateClearExpiredState; pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; @@ -243,7 +242,6 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateBegin = streamStateBegin; pStore->streamStateCommit = streamStateCommit; pStore->streamStateDestroy = streamStateDestroy; - pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; pStore->streamStateReloadInfo = streamStateReloadInfo; pStore->streamStateCopyBackend = streamStateCopyBackend; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index f8b75eb15c..4eac1cd5c9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1415,7 +1415,8 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i SVAlterTbReq vAlterTbReq = {0}; SVAlterTbRsp vAlterTbRsp = {0}; SDecoder dc = {0}; - int32_t rcode = 0; + int32_t code = 0; + int32_t lino = 0; int32_t ret; SEncoder ec = {0}; STableMetaRsp vMetaRsp = {0}; @@ -1431,7 +1432,6 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i if (tDecodeSVAlterTbReq(&dc, &vAlterTbReq) < 0) { vAlterTbRsp.code = TSDB_CODE_INVALID_MSG; tDecoderClear(&dc); - rcode = -1; goto _exit; } @@ -1439,7 +1439,6 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i if (metaAlterTable(pVnode->pMeta, ver, &vAlterTbReq, &vMetaRsp) < 0) { vAlterTbRsp.code = terrno; tDecoderClear(&dc); - rcode = -1; goto _exit; } tDecoderClear(&dc); @@ -1449,6 +1448,31 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i vAlterTbRsp.pMeta = &vMetaRsp; } + if (vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL || vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL) { + int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, vAlterTbReq.tbName); + if (uid == 0) { + vError("vgId:%d, %s failed at %s:%d since table %s not found", TD_VID(pVnode), __func__, __FILE__, __LINE__, + vAlterTbReq.tbName); + goto _exit; + } + + SArray* tbUids = taosArrayInit(4, sizeof(int64_t)); + void* p = taosArrayPush(tbUids, &uid); + TSDB_CHECK_NULL(p, code, lino, _exit, terrno); + + vDebug("vgId:%d, remove tags value altered table:%s from query table list", TD_VID(pVnode), vAlterTbReq.tbName); + if ((code = tqUpdateTbUidList(pVnode->pTq, tbUids, false)) < 0) { + vError("vgId:%d, failed to remove tbUid list since %s", TD_VID(pVnode), tstrerror(code)); + } + + vDebug("vgId:%d, try to add table:%s in query table list", TD_VID(pVnode), vAlterTbReq.tbName); + if ((code = tqUpdateTbUidList(pVnode->pTq, tbUids, true)) < 0) { + vError("vgId:%d, failed to add tbUid list since %s", TD_VID(pVnode), tstrerror(code)); + } + + taosArrayDestroy(tbUids); + } + _exit: taosArrayDestroy(vAlterTbReq.pMultiTag); tEncodeSize(tEncodeSVAlterTbRsp, &vAlterTbRsp, pRsp->contLen, ret); @@ -1457,6 +1481,7 @@ _exit: if (tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp) != 0) { vError("vgId:%d, failed to encode alter table response", TD_VID(pVnode)); } + tEncoderClear(&ec); if (vMetaRsp.pSchemas) { taosMemoryFree(vMetaRsp.pSchemas); diff --git a/source/libs/azure/src/td_block_blob_client.cpp b/source/libs/azure/src/td_block_blob_client.cpp index 33ac774d0c..ba2ac14551 100644 --- a/source/libs/azure/src/td_block_blob_client.cpp +++ b/source/libs/azure/src/td_block_blob_client.cpp @@ -38,6 +38,8 @@ TDBlockBlobClient TDBlockBlobClient::CreateFromConnectionString(const std::strin return newClient; } +TDBlockBlobClient::TDBlockBlobClient(BlobClient blobClient) : BlobClient(std::move(blobClient)) {} +#if 0 TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr credential, const BlobClientOptions& options) : BlobClient(blobUrl, std::move(credential), options) {} @@ -50,8 +52,6 @@ TDBlockBlobClient::TDBlockBlobClient(const std::string& TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, const BlobClientOptions& options) : BlobClient(blobUrl, options) {} -TDBlockBlobClient::TDBlockBlobClient(BlobClient blobClient) : BlobClient(std::move(blobClient)) {} - TDBlockBlobClient TDBlockBlobClient::WithSnapshot(const std::string& snapshot) const { TDBlockBlobClient newClient(*this); if (snapshot.empty()) { @@ -74,47 +74,6 @@ TDBlockBlobClient TDBlockBlobClient::WithVersionId(const std::string& versionId) return newClient; } -Azure::Response TDBlockBlobClient::Upload(Azure::Core::IO::BodyStream& content, - const UploadBlockBlobOptions& options, - const Azure::Core::Context& context) const { - _detail::BlockBlobClient::UploadBlockBlobOptions protocolLayerOptions; - if (options.TransactionalContentHash.HasValue()) { - if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) { - protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value; - } else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) { - protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value; - } - } - protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType; - protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding; - protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage; - protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value; - protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition; - protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl; - protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); - protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags); - protocolLayerOptions.Tier = options.AccessTier; - protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; - protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; - protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; - protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; - protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; - protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; - if (m_customerProvidedKey.HasValue()) { - protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; - protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; - protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); - } - protocolLayerOptions.EncryptionScope = m_encryptionScope; - if (options.ImmutabilityPolicy.HasValue()) { - protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn; - protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode; - } - protocolLayerOptions.LegalHold = options.HasLegalHold; - - return _detail::BlockBlobClient::Upload(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context); -} - Azure::Response TDBlockBlobClient::UploadFrom( const uint8_t* buffer, size_t bufferSize, const UploadBlockBlobFromOptions& options, const Azure::Core::Context& context) const { @@ -270,6 +229,47 @@ Azure::Response TDBlockBlobClient::UploadFrom return Azure::Response(std::move(result), std::move(commitBlockListResponse.RawResponse)); } +#endif +Azure::Response TDBlockBlobClient::Upload(Azure::Core::IO::BodyStream& content, + const UploadBlockBlobOptions& options, + const Azure::Core::Context& context) const { + _detail::BlockBlobClient::UploadBlockBlobOptions protocolLayerOptions; + if (options.TransactionalContentHash.HasValue()) { + if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) { + protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value; + } else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) { + protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value; + } + } + protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType; + protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding; + protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage; + protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value; + protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition; + protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl; + protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); + protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags); + protocolLayerOptions.Tier = options.AccessTier; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + if (m_customerProvidedKey.HasValue()) { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } + protocolLayerOptions.EncryptionScope = m_encryptionScope; + if (options.ImmutabilityPolicy.HasValue()) { + protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn; + protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode; + } + protocolLayerOptions.LegalHold = options.HasLegalHold; + + return _detail::BlockBlobClient::Upload(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context); +} Azure::Response TDBlockBlobClient::UploadFrom( const std::string& fileName, int64_t offset, int64_t size, const UploadBlockBlobFromOptions& options, @@ -349,7 +349,7 @@ Azure::Response TDBlockBlobClient::UploadFrom return Azure::Response(std::move(result), std::move(commitBlockListResponse.RawResponse)); } - +#if 0 Azure::Response TDBlockBlobClient::UploadFromUri( const std::string& sourceUri, const UploadBlockBlobFromUriOptions& options, const Azure::Core::Context& context) const { @@ -396,7 +396,7 @@ Azure::Response TDBlockBlobClient::UploadF return _detail::BlockBlobClient::UploadFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context); } - +#endif Azure::Response TDBlockBlobClient::StageBlock(const std::string& blockId, Azure::Core::IO::BodyStream& content, const StageBlockOptions& options, @@ -419,7 +419,7 @@ Azure::Response TDBlockBlobClient::StageBlock(const st protocolLayerOptions.EncryptionScope = m_encryptionScope; return _detail::BlockBlobClient::StageBlock(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context); } - +#if 0 Azure::Response TDBlockBlobClient::StageBlockFromUri( const std::string& blockId, const std::string& sourceUri, const StageBlockFromUriOptions& options, const Azure::Core::Context& context) const { @@ -457,7 +457,7 @@ Azure::Response TDBlockBlobClient::StageBlockFr return _detail::BlockBlobClient::StageBlockFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context); } - +#endif Azure::Response TDBlockBlobClient::CommitBlockList( const std::vector& blockIds, const CommitBlockListOptions& options, const Azure::Core::Context& context) const { @@ -492,7 +492,7 @@ Azure::Response TDBlockBlobClient::CommitBlockLis return _detail::BlockBlobClient::CommitBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions, context); } - +#if 0 Azure::Response TDBlockBlobClient::GetBlockList(const GetBlockListOptions& options, const Azure::Core::Context& context) const { _detail::BlockBlobClient::GetBlockBlobBlockListOptions protocolLayerOptions; @@ -502,6 +502,7 @@ Azure::Response TDBlockBlobClient::GetBlockList(cons return _detail::BlockBlobClient::GetBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); } +#endif /* Azure::Response TDBlockBlobClient::Query(const std::string& querySqlExpression, const QueryBlobOptions& options, diff --git a/source/libs/azure/test/azExceptionTest.cpp b/source/libs/azure/test/azExceptionTest.cpp new file mode 100644 index 0000000000..a83bb4d8f2 --- /dev/null +++ b/source/libs/azure/test/azExceptionTest.cpp @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +// clang-format off +#include "td_block_blob_client.hpp" +#include "az.h" +// clang-format on + +using namespace Azure::Storage; +using namespace Azure::Storage::Blobs; + +extern int8_t tsS3Enabled; +extern char tsS3BucketName[TSDB_FQDN_LEN]; + +static int32_t azInitEnv() { + int32_t code = 0; + + extern int8_t tsS3EpNum; + + extern char tsS3Hostname[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; + + /* TCS parameter format + tsS3Hostname[0] = "/.blob.core.windows.net"; + tsS3AccessKeyId[0] = ""; + tsS3AccessKeySecret[0] = ""; + tsS3BucketName = ""; + */ + + const char *hostname = "/.blob.core.windows.net"; + const char *accessKeyId = ""; + const char *accessKeySecret = ""; + const char *bucketName = ""; + + if (hostname[0] != '<') { + tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); + tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); + } else { + const char *accountId = getenv("ablob_account_id"); + if (!accountId) { + return -1; + } + + const char *accountSecret = getenv("ablob_account_secret"); + if (!accountSecret) { + return -1; + } + + const char *containerName = getenv("ablob_container"); + if (!containerName) { + return -1; + } + + TAOS_STRCPY(&tsS3Hostname[0][0], accountId); + TAOS_STRCAT(&tsS3Hostname[0][0], ".blob.core.windows.net"); + TAOS_STRCPY(&tsS3AccessKeyId[0][0], accountId); + TAOS_STRCPY(&tsS3AccessKeySecret[0][0], accountSecret); + TAOS_STRCPY(tsS3BucketName, containerName); + } + + tstrncpy(tsTempDir, "/tmp/", PATH_MAX); + + tsS3Enabled = true; + + return code; +} + +// TEST(AzTest, DISABLED_InterfaceTest) { +TEST(AzETest, InterfaceTest) { + int code = 0; + bool check = false; + bool withcp = false; + + code = azInitEnv(); + if (code) { + std::cout << "ablob env init failed with: " << code << std::endl; + return; + } + + GTEST_ASSERT_EQ(code, 0); + GTEST_ASSERT_EQ(tsS3Enabled, 1); + + code = azBegin(); + GTEST_ASSERT_EQ(code, 0); + + code = azCheckCfg(); + GTEST_ASSERT_EQ(code, 0); + const int size = 4096; + char data[size] = {0}; + for (int i = 0; i < size / 2; ++i) { + data[i * 2 + 1] = 1; + } + + const char object_name[] = "azut.bin"; + char path[PATH_MAX] = {0}; + char path_download[PATH_MAX] = {0}; + int ds_len = strlen(TD_DIRSEP); + int tmp_len = strlen(tsTempDir); + + (void)snprintf(path, PATH_MAX, "%s", tsTempDir); + if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP); + (void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name); + } else { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name); + } + + tstrncpy(path_download, path, strlen(path) + 1); + tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1); + + TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH); + GTEST_ASSERT_NE(fp, nullptr); + + int n = taosWriteFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + code = azPutObjectFromFileOffset(path, object_name, 0, size); + GTEST_ASSERT_EQ(code, 0); + + uint8_t *pBlock = NULL; + code = azGetObjectBlock(object_name, 0, size, check, &pBlock); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(pBlock[i * 2], 0); + GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1); + } + + taosMemoryFree(pBlock); + + code = azGetObjectToFile(object_name, path_download); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + azDeleteObjectsByPrefix(object_name); + // list object to check + + code = azPutObjectFromFile2(path, object_name, withcp); + GTEST_ASSERT_EQ(code, 0); + + code = azGetObjectsByPrefix(object_name, tsTempDir); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + TDBlockBlobClient blobClient = + TDBlockBlobClient::CreateFromConnectionString(std::getenv("ablob_cs"), std::string(tsS3BucketName), object_name); + + const char *object_name_arr[] = {object_name}; + code = azDeleteObjects(object_name_arr, 1); + GTEST_ASSERT_EQ(code, 0); + + azEnd(); +} diff --git a/source/libs/azure/test/azTest.cpp b/source/libs/azure/test/azTest.cpp index 0459cb5f6a..b57b7ca884 100644 --- a/source/libs/azure/test/azTest.cpp +++ b/source/libs/azure/test/azTest.cpp @@ -199,3 +199,132 @@ TEST(AzTest, InterfaceTest) { azEnd(); } + +// TEST(AzTest, DISABLED_InterfaceTestBig) { +TEST(AzTest, InterfaceTestBig) { + int code = 0; + bool check = false; + bool withcp = false; + + code = azInitEnv(); + if (code) { + std::cout << "ablob env init failed with: " << code << std::endl; + return; + } + + GTEST_ASSERT_EQ(code, 0); + GTEST_ASSERT_EQ(tsS3Enabled, 1); + + code = azBegin(); + GTEST_ASSERT_EQ(code, 0); + + code = azCheckCfg(); + GTEST_ASSERT_EQ(code, 0); + const int size = 256 * 1024 * 1024 + 1; + char *data = (char *)taosMemoryCalloc(1, size); + if (!data) { + std::cout << "code: " << code << "terrno: " << terrno << std::endl; + + return; + } + + for (int i = 0; i < size / 2; ++i) { + data[i * 2 + 1] = 1; + } + + const char object_name[] = "azut.bin"; + char path[PATH_MAX] = {0}; + char path_download[PATH_MAX] = {0}; + int ds_len = strlen(TD_DIRSEP); + int tmp_len = strlen(tsTempDir); + + (void)snprintf(path, PATH_MAX, "%s", tsTempDir); + if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP); + (void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name); + } else { + (void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name); + } + + tstrncpy(path_download, path, strlen(path) + 1); + tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1); + + TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH); + GTEST_ASSERT_NE(fp, nullptr); + + int n = taosWriteFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + code = azPutObjectFromFileOffset(path, object_name, 0, size); + GTEST_ASSERT_EQ(code, 0); + + uint8_t *pBlock = NULL; + code = azGetObjectBlock(object_name, 0, size, check, &pBlock); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(pBlock[i * 2], 0); + GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1); + } + + taosMemoryFree(pBlock); + + code = azGetObjectToFile(object_name, path_download); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + azDeleteObjectsByPrefix(object_name); + // list object to check + + code = azPutObjectFromFile2(path, object_name, withcp); + GTEST_ASSERT_EQ(code, 0); + + code = azGetObjectsByPrefix(object_name, tsTempDir); + GTEST_ASSERT_EQ(code, 0); + + { + TdFilePtr fp = taosOpenFile(path, TD_FILE_READ); + GTEST_ASSERT_NE(fp, nullptr); + + (void)memset(data, 0, size); + + int64_t n = taosReadFile(fp, data, size); + GTEST_ASSERT_EQ(n, size); + + code = taosCloseFile(&fp); + GTEST_ASSERT_EQ(code, 0); + + for (int i = 0; i < size / 2; ++i) { + GTEST_ASSERT_EQ(data[i * 2], 0); + GTEST_ASSERT_EQ(data[i * 2 + 1], 1); + } + } + + const char *object_name_arr[] = {object_name}; + code = azDeleteObjects(object_name_arr, 1); + GTEST_ASSERT_EQ(code, 0); + + taosMemoryFree(data); + + azEnd(); +} diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index dd1a52022e..eb72edb964 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -61,9 +61,13 @@ static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + size_t keyBufSize = 0; + int32_t num = 0; + SExprInfo* pExprInfo = NULL; + const char* id = GET_TASKID(pTaskInfo); - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; SAnomalyWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAnomalyWindowOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SAnomalyWindowPhysiNode* pAnomalyNode = (SAnomalyWindowPhysiNode*)physiNode; @@ -74,13 +78,13 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p } if (!taosAnalGetOptStr(pAnomalyNode->anomalyOpt, "algo", pInfo->algoName, sizeof(pInfo->algoName))) { - qError("failed to get anomaly_window algorithm name from %s", pAnomalyNode->anomalyOpt); + qError("%s failed to get anomaly_window algorithm name from %s", id, pAnomalyNode->anomalyOpt); code = TSDB_CODE_ANA_ALGO_NOT_FOUND; goto _error; } if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) { - qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName); + qError("%s failed to get anomaly_window algorithm url from %s", id, pInfo->algoName); code = TSDB_CODE_ANA_ALGO_NOT_LOAD; goto _error; } @@ -94,20 +98,18 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p SExprInfo* pScalarExprInfo = NULL; code = createExprInfo(pAnomalyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr); QUERY_CHECK_CODE(code, lino, _error); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); } - size_t keyBufSize = 0; - int32_t num = 0; - SExprInfo* pExprInfo = NULL; code = createExprInfo(pAnomalyNode->window.pFuncs, NULL, &pExprInfo, &num); QUERY_CHECK_CODE(code, lino, _error); initResultSizeInfo(&pOperator->resultInfo, 4096); - code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); + code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, id, pTaskInfo->streamInfo.pState, + &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); SSDataBlock* pResBlock = createDataBlockFromDescNode(pAnomalyNode->window.node.pOutputDataBlockDesc); @@ -124,27 +126,19 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p pInfo->anomalyCol = extractColumnFromColumnNode(pColNode); pInfo->anomalyKey.type = pInfo->anomalyCol.type; pInfo->anomalyKey.bytes = pInfo->anomalyCol.bytes; + pInfo->anomalyKey.pData = taosMemoryCalloc(1, pInfo->anomalyCol.bytes); - if (pInfo->anomalyKey.pData == NULL) { - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalyKey.pData, code, lino, _error, terrno) int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize + pInfo->anomalyKey.bytes; pInfo->anomalySup.pResultRow = taosMemoryCalloc(1, itemSize); - if (pInfo->anomalySup.pResultRow == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.pResultRow, code, lino, _error, terrno) + pInfo->anomalySup.blocks = taosArrayInit(16, sizeof(SSDataBlock*)); - if (pInfo->anomalySup.blocks == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.blocks, code, lino, _error, terrno) + pInfo->anomalySup.windows = taosArrayInit(16, sizeof(STimeWindow)); - if (pInfo->anomalySup.windows == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.windows, code, lino, _error, terrno) code = filterInitFromNode((SNode*)pAnomalyNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); QUERY_CHECK_CODE(code, lino, _error); @@ -162,18 +156,21 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p *pOptrInfo = pOperator; - qDebug("anomaly_window operator is created, algo:%s url:%s opt:%s", pInfo->algoName, pInfo->algoUrl, + qDebug("%s anomaly_window operator is created, algo:%s url:%s opt:%s", id, pInfo->algoName, pInfo->algoUrl, pInfo->anomalyOpt); return TSDB_CODE_SUCCESS; _error: + qError("%s failed to create anomaly_window operator, line:%d algo:%s code:%s", id, lino, pAnomalyNode->anomalyOpt, + tstrerror(code)); + if (pInfo != NULL) { anomalyDestroyOperatorInfo(pInfo); } destroyOperatorAndDownstreams(pOperator, &downstream, 1); pTaskInfo->code = code; - qError("failed to create anomaly_window operator, algo:%s code:0x%x", pInfo->algoName, code); + return code; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 91af218a7e..031d2e8bdc 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -4466,6 +4466,19 @@ _end: return code; } +static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pResults) { + SSessionKey checkKey = {0}; + getSessionHashKey(pKey, &checkKey); + if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) { + return true; + } + + if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) { + return true; + } + return false; +} + static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, SSHashObj* pStDeleted) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -4518,7 +4531,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); QUERY_CHECK_CODE(code, lino, _end); - releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); + if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) { + releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); + } setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); code = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 13ae220116..a556f68c32 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1441,7 +1441,7 @@ static int32_t doSetUserTableMetaInfo(SStoreMetaReader* pMetaReaderFn, SStoreMet SMetaReader mr1 = {0}; pMetaReaderFn->initReader(&mr1, pVnode, META_READER_NOLOCK, pMetaFn); - + int64_t suid = pMReader->me.ctbEntry.suid; code = pMetaReaderFn->getTableEntryByUid(&mr1, suid); if (code != TSDB_CODE_SUCCESS) { @@ -1752,7 +1752,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); - + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { @@ -2269,6 +2269,8 @@ static SSDataBlock* sysTableBuildUserFileSets(SOperatorInfo* pOperator) { if (ret) { if (ret == TSDB_CODE_NOT_FOUND) { // no more scan entry + setOperatorCompleted(pOperator); + pAPI->tsdReader.fileSetReaderClose(&pInfo->pFileSetReader); break; } else { code = ret; @@ -2284,7 +2286,7 @@ static SSDataBlock* sysTableBuildUserFileSets(SOperatorInfo* pOperator) { // db_name pColInfoData = taosArrayGet(p->pDataBlock, index++); QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, db, false); + code = colDataSetVal(pColInfoData, numOfRows, dbname, false); QUERY_CHECK_CODE(code, lino, _end); // vgroup_id diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1c77d12fb2..707018ac65 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6429,7 +6429,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return code; } - len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%d] Stt_Rows=[%d] ", + len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Inmem_Rows=[%u] Stt_Rows=[%u] ", pData->numOfInmemRows, pData->numOfSttRows); varDataSetLen(st, len); code = colDataSetVal(pColInfo, row++, st, false); diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index 2814b5dc16..a6954afab6 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -28,7 +28,7 @@ extern "C" { #endif -typedef enum { MATCH, JUMP, SPLIT, RANGE } InstType; +typedef enum { INS_MATCH, INS_JUMP, INS_SPLIT, INS_RANGE } InstType; typedef struct MatchValue { #ifdef WINDOWS diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index 3b0014f16a..fa7dbb5f1f 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -159,14 +159,14 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r if (false == sparSetGet(set, i, &ip)) continue; Inst *inst = taosArrayGet(builder->dfa->insts, ip); - if (inst->ty == JUMP || inst->ty == SPLIT) { + if (inst->ty == INS_JUMP || inst->ty == INS_SPLIT) { continue; - } else if (inst->ty == RANGE) { + } else if (inst->ty == INS_RANGE) { if (taosArrayPush(tinsts, &ip) == NULL) { code = terrno; goto _exception; } - } else if (inst->ty == MATCH) { + } else if (inst->ty == INS_MATCH) { isMatch = true; if (taosArrayPush(tinsts, &ip) == NULL) { code = terrno; @@ -234,11 +234,11 @@ void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) { } bool succ = sparSetAdd(set, ip, NULL); Inst *inst = taosArrayGet(dfa->insts, ip); - if (inst->ty == MATCH || inst->ty == RANGE) { + if (inst->ty == INS_MATCH || inst->ty == INS_RANGE) { // do nothing - } else if (inst->ty == JUMP) { + } else if (inst->ty == INS_JUMP) { dfaAdd(dfa, set, inst->jv.step); - } else if (inst->ty == SPLIT) { + } else if (inst->ty == INS_SPLIT) { dfaAdd(dfa, set, inst->sv.len1); dfaAdd(dfa, set, inst->sv.len2); } @@ -253,11 +253,11 @@ bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte) { if (false == sparSetGet(from, i, &ip)) continue; Inst *inst = taosArrayGet(dfa->insts, ip); - if (inst->ty == JUMP || inst->ty == SPLIT) { + if (inst->ty == INS_JUMP || inst->ty == INS_SPLIT) { continue; - } else if (inst->ty == MATCH) { + } else if (inst->ty == INS_MATCH) { isMatch = true; - } else if (inst->ty == RANGE) { + } else if (inst->ty == INS_RANGE) { if (inst->rv.start <= byte && byte <= inst->rv.end) { dfaAdd(dfa, to, ip + 1); } diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc index 299b62b6fb..5b22b51172 100644 --- a/source/libs/index/test/utilUT.cc +++ b/source/libs/index/test/utilUT.cc @@ -17,6 +17,7 @@ #include "tglobal.h" #include "tskiplist.h" #include "tutil.h" +#include "indexFstDfa.h" class UtilEnv : public ::testing::Test { protected: @@ -41,6 +42,29 @@ class UtilEnv : public ::testing::Test { SArray *rslt; }; +class UtilComm : public ::testing::Test { + protected: + virtual void SetUp() { + // src = (SArray *)taosArrayInit(2, sizeof(void *)); + // for (int i = 0; i < 3; i++) { + // SArray *m = taosArrayInit(10, sizeof(uint64_t)); + // taosArrayPush(src, &m); + // } + + // rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t)); + } + virtual void TearDown() { + // for (int i = 0; i < taosArrayGetSize(src); i++) { + // SArray *m = (SArray *)taosArrayGetP(src, i); + // taosArrayDestroy(m); + // } + // taosArrayDestroy(src); + } + // SArray *src; + // SArray *rslt; + +}; + static void clearSourceArray(SArray *p) { for (int i = 0; i < taosArrayGetSize(p); i++) { SArray *m = (SArray *)taosArrayGetP(p, i); @@ -369,3 +393,35 @@ TEST_F(UtilEnv, testDictComm) { EXPECT_EQ(COMMON_INPUTS[v], i); } } + +TEST_F(UtilComm, testCompress) { + for (int32_t i = 0; i < 6; i++) { + _cache_range_compare cmpFunc = idxGetCompare((RangeType)(i)); + //char[32]a = 0, b = 1; + char a[32] = {0}; + char b[32] = {1}; + for (int32_t j = 0; j < TSDB_DATA_TYPE_MAX; j++) { + cmpFunc(a, b, j); + } + } +} +TEST_F(UtilComm, testfstDfa) { + { + FstDfaBuilder *builder = dfaBuilderCreate(NULL); + ASSERT_TRUE(builder != NULL); + dfaBuilderDestroy(builder); + } + { + SArray *pInst = taosArrayInit(32, sizeof(uint8_t)); + for (int32_t i = 0; i < 26; i++) { + uint8_t v = 'a' + i; + taosArrayPush(pInst, &v); + } + FstDfaBuilder *builder = dfaBuilderCreate(pInst); + FstDfa *dfa = dfaBuilderBuild(builder); + dfaBuilderDestroy(builder); + } +} + + + diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 63eb09d509..fda49e7ee2 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -290,8 +290,7 @@ db_options(A) ::= db_options(B) ENCRYPT_ALGORITHM NK_STRING(C). db_options(A) ::= db_options(B) DNODES NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DNODES, &C); } db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_INTEGER (C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } db_options(A) ::= db_options(B) COMPACT_INTERVAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_INTERVAL, &C); } -db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_integer_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } -db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } +db_options(A) ::= db_options(B) COMPACT_TIME_RANGE signed_duration_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_RANGE, C); } db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } db_options(A) ::= db_options(B) COMPACT_TIME_OFFSET NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMPACT_TIME_OFFSET, &C); } @@ -331,8 +330,7 @@ alter_db_option(A) ::= KEEP_TIME_OFFSET NK_INTEGER(B). alter_db_option(A) ::= ENCRYPT_ALGORITHM NK_STRING(B). { A.type = DB_OPTION_ENCRYPT_ALGORITHM; A.val = B; } alter_db_option(A) ::= COMPACT_INTERVAL NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } alter_db_option(A) ::= COMPACT_INTERVAL NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_INTERVAL; A.val = B; } -alter_db_option(A) ::= COMPACT_TIME_RANGE signed_integer_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } -alter_db_option(A) ::= COMPACT_TIME_RANGE signed_variable_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } +alter_db_option(A) ::= COMPACT_TIME_RANGE signed_duration_list(B). { A.type = DB_OPTION_COMPACT_TIME_RANGE; A.pList = B; } alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_INTEGER(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_VARIABLE(B). { A.type = DB_OPTION_COMPACT_TIME_OFFSET; A.val = B; } @@ -341,20 +339,17 @@ alter_db_option(A) ::= COMPACT_TIME_OFFSET NK_VARIABLE(B). integer_list(A) ::= NK_INTEGER(B). { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); } integer_list(A) ::= integer_list(B) NK_COMMA NK_INTEGER(C). { A = addNodeToList(pCxt, B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &C)); } -%type signed_integer_list { SNodeList* } -%destructor signed_integer_list { nodesDestroyList($$); } -signed_integer_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, B); } -signed_integer_list(A) ::= signed_integer_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, C); } - %type variable_list { SNodeList* } %destructor variable_list { nodesDestroyList($$); } variable_list(A) ::= NK_VARIABLE(B). { A = createNodeList(pCxt, createDurationValueNode(pCxt, &B)); } variable_list(A) ::= variable_list(B) NK_COMMA NK_VARIABLE(C). { A = addNodeToList(pCxt, B, createDurationValueNode(pCxt, &C)); } -%type signed_variable_list { SNodeList* } -%destructor signed_variable_list { nodesDestroyList($$); } -signed_variable_list(A) ::= signed_variable(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } -signed_variable_list(A) ::= signed_variable_list(B) NK_COMMA signed_variable(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } +%type signed_duration_list { SNodeList* } +%destructor signed_duration_list { nodesDestroyList($$); } +signed_duration_list(A) ::= signed_variable(B). { A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); } +signed_duration_list(A) ::= signed_integer(B). { A = createNodeList(pCxt, B); } +signed_duration_list(A) ::= signed_duration_list(B) NK_COMMA signed_integer(C). { A = addNodeToList(pCxt, B, C); } +signed_duration_list(A) ::= signed_duration_list(B) NK_COMMA signed_variable(C). { A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); } %type retention_list { SNodeList* } %destructor retention_list { nodesDestroyList($$); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 342bd6d66e..8f23daf8dd 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2005,7 +2005,7 @@ static EDealRes translateDurationValue(STranslateContext* pCxt, SValueNode* pVal pVal->datum.i = AUTO_DURATION_VALUE; pVal->unit = getPrecisionUnit(pVal->node.resType.precision); } else if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, - pVal->node.resType.precision, false) != TSDB_CODE_SUCCESS) { + pVal->node.resType.precision, true) != TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(int64_t*)&pVal->typeData = pVal->datum.i; @@ -7782,7 +7782,7 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int64_t val, int64_t minVal, int64_t maxVal, bool skipUndef) { - if (skipUndef ? ((val >= 0) && (val < minVal || val > maxVal)) : (val < minVal || val > maxVal)) { + if (skipUndef ? ((val >= 0 || val < -2) && (val < minVal || val > maxVal)) : (val < minVal || val > maxVal)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %" PRId64 ", valid range: [%" PRId64 ", %" PRId64 "]", pName, val, minVal, maxVal); @@ -8148,7 +8148,6 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa static int32_t checkDbCompactIntervalOption(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) { int32_t code = 0; - int64_t interval = 0; int32_t keep2 = pOptions->keep[2]; if (NULL != pOptions->pCompactIntervalNode) { @@ -8163,23 +8162,26 @@ static int32_t checkDbCompactIntervalOption(STranslateContext* pCxt, const char* pOptions->pCompactIntervalNode->unit, TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY); } - interval = getBigintFromValueNode(pOptions->pCompactIntervalNode); + int64_t interval = getBigintFromValueNode(pOptions->pCompactIntervalNode); if (interval != 0) { if (keep2 == -1) { // alter db TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); keep2 = pOptions->pDbCfg->daysToKeep2; } code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); + TAOS_CHECK_RETURN(code); } + pOptions->compactInterval = (int32_t)interval; } else if (pOptions->compactInterval > 0) { - interval = pOptions->compactInterval * 1440; // convert to minutes - if (keep2 == -1) { // alter db + int64_t interval = (int64_t)pOptions->compactInterval * 1440; // convert to minutes + if (keep2 == -1) { // alter db TAOS_CHECK_RETURN(translateGetDbCfg(pCxt, pDbName, &pOptions->pDbCfg)); keep2 = pOptions->pDbCfg->daysToKeep2; } code = checkDbRangeOption(pCxt, "compact_interval", interval, TSDB_MIN_COMPACT_INTERVAL, keep2); + TAOS_CHECK_RETURN(code); + pOptions->compactInterval = (int32_t)interval; } - if (code == 0) pOptions->compactInterval = interval; return code; } @@ -8222,6 +8224,10 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char pOptions->compactStartTime = getBigintFromValueNode(pStart); pOptions->compactEndTime = getBigintFromValueNode(pEnd); + if (pOptions->compactStartTime == 0 && pOptions->compactEndTime == 0) { + return TSDB_CODE_SUCCESS; + } + if (pOptions->compactStartTime >= pOptions->compactEndTime) { return generateSyntaxErrMsgExt( &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, @@ -8238,7 +8244,7 @@ static int32_t checkDbCompactTimeRangeOption(STranslateContext* pCxt, const char } if (pOptions->compactStartTime < -keep2 || pOptions->compactStartTime > -days) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option compact_time_range: %dm, start_time should be in range: [%dm, %dm]", + "Invalid option compact_time_range: %dm, start time should be in range: [%dm, %dm]", pOptions->compactStartTime, -keep2, -days); } if (pOptions->compactEndTime < -keep2 || pOptions->compactEndTime > -days) { @@ -10086,7 +10092,6 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt const char* validConfigs[] = { "encrypt_key", - tsAlterCompactTaskKeywords, }; if (0 == strncasecmp(cfgReq.config, validConfigs[0], strlen(validConfigs[0]) + 1)) { int32_t klen = strlen(cfgReq.value); @@ -10097,28 +10102,6 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt ENCRYPT_KEY_LEN_MIN, ENCRYPT_KEY_LEN); } code = buildCmdMsg(pCxt, TDMT_MND_CREATE_ENCRYPT_KEY, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); - } else if (0 == strncasecmp(cfgReq.config, validConfigs[1], strlen(validConfigs[1]) + 1)) { - char* endptr = NULL; - int32_t maxCompactTasks = taosStr2Int32(cfgReq.value, &endptr, 10); - int32_t minMaxCompactTasks = MIN_MAX_COMPACT_TASKS; - int32_t maxMaxCompactTasks = MAX_MAX_COMPACT_TASKS; - - // check format - if (endptr == cfgReq.value || endptr[0] != '\0') { - tFreeSMCfgDnodeReq(&cfgReq); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, - "Invalid max compact tasks: %s", cfgReq.value); - } - - // check range - if (maxCompactTasks < minMaxCompactTasks || maxCompactTasks > maxMaxCompactTasks) { - tFreeSMCfgDnodeReq(&cfgReq); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, - "Invalid max compact tasks: %d, valid range [%d,%d]", maxCompactTasks, - minMaxCompactTasks, maxMaxCompactTasks); - } - - code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); } else { code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 4a9eea66e2..5dd43ca064 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -746,9 +746,19 @@ _return: } int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF) { - QW_ERR_RET(qwDropTask(QW_FPARAMS())); + int32_t code = TSDB_CODE_SUCCESS; + SQWTaskCtx *ctx = NULL; - return TSDB_CODE_SUCCESS; + QW_ERR_RET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + +_return: + + QW_UNLOCK(QW_WRITE, &ctx->lock); + + return code; } int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 7af64c041d..f922a5e03e 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -254,6 +254,9 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d int32_t chkptTriggerRecvMonitorHelper(SStreamTask* pTask, void* param, SArray** ppNotSendList); int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName); int32_t uploadCheckpointToS3(const char* id, const char* path); +int32_t deleteCheckpointFile(const char* id, const char* name); +int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock, + int32_t transId); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 37fb081a87..6031710bf9 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1808,7 +1808,7 @@ int stateKeyDecode(void* k, char* buf) { return p - buf; } -int stateKeyToString(void* k, char* buf) { +int32_t stateKeyToString(void* k, char* buf) { SStateKey* key = k; int n = 0; n += sprintf(buf + n, "[groupId:%" PRIu64 ",", key->key.groupId); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 118cb1cfb6..f880526541 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -18,7 +18,7 @@ #include "streamBackendRocksdb.h" #include "streamInt.h" -#define CHECK_NOT_RSP_DURATION 10 * 1000 // 10 sec +#define CHECK_NOT_RSP_DURATION 60 * 1000 // 60 sec static void processDownstreamReadyRsp(SStreamTask* pTask); static void rspMonitorFn(void* param, void* tmrId); @@ -660,7 +660,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { pInfo->timeoutRetryCount += 1; - // timeout more than 100 sec, add into node update list + // timeout more than 600 sec, add into node update list if (pInfo->timeoutRetryCount > 10) { pInfo->timeoutRetryCount = 0; @@ -674,7 +674,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { findCheckRspStatus(pInfo, *pTaskId, &p); if (p != NULL) { code = streamTaskAddIntoNodeUpdateList(pTask, p->vgId); - stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 100sec, add into nodeUpdate list", + stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 600sec, add into nodeUpdate list", id, vgId, p->taskId, p->vgId); } } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index ebde9fe50e..d3eba382c9 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -19,7 +19,6 @@ #include "tcs.h" static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName); -static int32_t deleteCheckpointFile(const char* id, const char* name); static int32_t streamTaskUploadCheckpoint(const char* id, const char* path, int64_t checkpointId); #ifdef BUILD_NO_CALL static int32_t deleteCheckpoint(const char* id); @@ -230,8 +229,8 @@ int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStream return code; } -static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock, - int32_t transId) { +int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock, + int32_t transId) { int32_t code = 0; int32_t vgId = pTask->pMeta->vgId; int32_t taskLevel = pTask->info.taskLevel; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 8e8e300ee6..3955343fdb 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -250,9 +250,10 @@ _EXIT: streamBackendCleanup((void*)pBackend); if (code == 0) { - char* state = taosMemoryCalloc(1, strlen(pMeta->path) + 32); + int32_t len = strlen(pMeta->path) + 32; + char* state = taosMemoryCalloc(1, len); if (state != NULL) { - sprintf(state, "%s%s%s", pMeta->path, TD_DIRSEP, "state"); + (void) snprintf(state, len, "%s%s%s", pMeta->path, TD_DIRSEP, "state"); taosRemoveDir(state); taosMemoryFree(state); } else { @@ -379,7 +380,7 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, char* tpath = taosMemoryCalloc(1, len); TSDB_CHECK_NULL(tpath, code, lino, _err, terrno); - sprintf(tpath, "%s%s%s", path, TD_DIRSEP, "stream"); + (void) snprintf(tpath, len, "%s%s%s", path, TD_DIRSEP, "stream"); pMeta->path = tpath; code = streamMetaOpenTdb(pMeta); diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index ae8a71d988..88495ac760 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -119,23 +119,21 @@ static int64_t kBlockSize = 64 * 1024; int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path, void* pMeta); void streamSnapHandleDestroy(SStreamSnapHandle* handle); -// static void streamBuildFname(char* path, char* file, char* fullname) - -#define STREAM_ROCKSDB_BUILD_FULLNAME(path, file, fullname) \ - do { \ - sprintf(fullname, "%s%s%s", path, TD_DIRSEP, file); \ - } while (0) - int32_t streamGetFileSize(char* path, char* name, int64_t* sz) { int32_t ret = 0; + int32_t len = strlen(path) + 32; - char* fullname = taosMemoryCalloc(1, strlen(path) + 32); + char* fullname = taosMemoryCalloc(1, len); if (fullname == NULL) { stError("failed to get file:%s size, code: out of memory", name); return terrno; } - sprintf(fullname, "%s%s%s", path, TD_DIRSEP, name); + ret = snprintf(fullname, len, "%s%s%s", path, TD_DIRSEP, name); + if (ret < 0 || ret >= len) { + stError("%s failed to set the file path for get the file size, code: out of buffer", name); + return TSDB_CODE_OUT_OF_BUFFER; + } ret = taosStatFile(fullname, sz, NULL, NULL); taosMemoryFree(fullname); @@ -146,7 +144,7 @@ int32_t streamGetFileSize(char* path, char* name, int64_t* sz) { TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { char fullname[256] = {0}; - STREAM_ROCKSDB_BUILD_FULLNAME(path, name, fullname); + (void) snprintf(fullname, tListLen(fullname),"%s%s%s", path, TD_DIRSEP, name); return taosOpenFile(fullname, opt); } @@ -155,35 +153,74 @@ int32_t streamCreateTaskDbSnapInfo(void* arg, char* path, SArray* pSnap) { retur int32_t streamDestroyTaskDbSnapInfo(void* arg, SArray* snap) { return taskDbDestroySnap(arg, snap); } void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { - if (qDebugFlag & DEBUG_DEBUG) { - int16_t cap = 512; + int16_t cap = 512; + if (qDebugFlag & DEBUG_DEBUG) { char* buf = taosMemoryCalloc(1, cap); if (buf == NULL) { stError("%s failed to alloc memory, reason:%s", STREAM_STATE_TRANSFER, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); return; } - int32_t nBytes = snprintf(buf + strlen(buf), cap, "["); - if (nBytes <= 0 || nBytes >= cap) { - taosMemoryFree(buf); - stError("%s failed to write buf, reason:%s", STREAM_STATE_TRANSFER, tstrerror(TSDB_CODE_OUT_OF_RANGE)); - return; - } + int32_t len = 0; + int32_t wlen = 1; - if (pSnapFile->pCurrent) sprintf(buf, "current: %s,", pSnapFile->pCurrent); - if (pSnapFile->pMainfest) sprintf(buf + strlen(buf), "MANIFEST: %s,", pSnapFile->pMainfest); - if (pSnapFile->pOptions) sprintf(buf + strlen(buf), "options: %s,", pSnapFile->pOptions); - if (pSnapFile->pSst) { - for (int32_t i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { - char* name = taosArrayGetP(pSnapFile->pSst, i); - if (strlen(buf) + strlen(name) < cap) sprintf(buf + strlen(buf), "%s,", name); + do { + buf[0] = '['; + if (pSnapFile->pCurrent) { + len = snprintf(buf + wlen, cap - wlen, "current: %s,", pSnapFile->pCurrent); + if (len > 0 && len < (cap - wlen)) { + wlen += len; + } else { + stError("%s failed to build buf for debug, code: out of buffer", STREAM_STATE_TRANSFER); + break; + } } + + if (pSnapFile->pMainfest) { + len = snprintf(buf + wlen, cap - wlen, "MANIFEST: %s,", pSnapFile->pMainfest); + if (len > 0 && len < (cap - wlen)) { + wlen += len; + } else { + stError("%s failed to build buf for debug, code: out of buffer", STREAM_STATE_TRANSFER); + break; + } + } + + if (pSnapFile->pOptions) { + len = snprintf(buf + wlen, cap - wlen, "options: %s,", pSnapFile->pOptions); + if (len > 0 && len < (cap - wlen)) { + wlen += len; + } else { + stError("%s failed to build buf for debug, code: out of buffer", STREAM_STATE_TRANSFER); + break; + } + } + + if (pSnapFile->pSst) { + for (int32_t i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { + char* name = taosArrayGetP(pSnapFile->pSst, i); + if (strlen(buf) + strlen(name) < cap) { + len = snprintf(buf + wlen, cap - wlen, "%s,", name); + if (len > 0 && len < (cap - wlen)) { + wlen += len; + } else { + stError("%s failed to build buf for debug, code: out of buffer", STREAM_STATE_TRANSFER); + break; + } + } + } + } + } while (0); + + if (wlen < cap) { + buf[wlen] = ']'; } - if ((strlen(buf)) < cap) sprintf(buf + strlen(buf) - 1, "]"); + buf[cap - 1] = '\0'; stInfo("%s %" PRId64 "-%" PRId64 " get file list: %s", STREAM_STATE_TRANSFER, pSnapFile->snapInfo.streamId, pSnapFile->snapInfo.taskId, buf); + taosMemoryFree(buf); } } @@ -771,16 +808,23 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa SBackendSnapFile2* pDbSnapFile = taosArrayGet(pHandle->pDbSnapSet, pHandle->currIdx); if (pDbSnapFile->inited == 0) { char idstr[64] = {0}; - sprintf(idstr, "0x%" PRIx64 "-0x%x", snapInfo.streamId, (int32_t)(snapInfo.taskId)); + (void)snprintf(idstr, tListLen(idstr), "0x%" PRIx64 "-0x%x", snapInfo.streamId, (int32_t)(snapInfo.taskId)); - char* path = taosMemoryCalloc(1, strlen(pHandle->metaPath) + 256); + int32_t bufLen = strlen(pHandle->metaPath) + 256; + char* path = taosMemoryCalloc(1, bufLen); if (path == NULL) { stError("s-task:0x%x failed to prepare meta header buffer, code:Out of memory", (int32_t) snapInfo.taskId); return terrno; } - sprintf(path, "%s%s%s%s%s%s%s%" PRId64 "", pHandle->metaPath, TD_DIRSEP, idstr, TD_DIRSEP, "checkpoints", TD_DIRSEP, - "checkpoint", snapInfo.chkpId); + int32_t ret = snprintf(path, bufLen, "%s%s%s%s%s%s%s%" PRId64 "", pHandle->metaPath, TD_DIRSEP, idstr, TD_DIRSEP, + "checkpoints", TD_DIRSEP, "checkpoint", snapInfo.chkpId); + if (ret < 0 || ret >= bufLen) { + stError("s-task:0x%x failed to set the path for take snapshot, code: out of buffer, %s", (int32_t)snapInfo.taskId, + pHandle->metaPath); + return TSDB_CODE_OUT_OF_BUFFER; + } + if (!taosIsDir(path)) { code = taosMulMkDir(path); stInfo("%s mkdir %s", STREAM_STATE_TRANSFER, path); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 89f0ea9e1f..7259c0e49a 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -546,10 +546,6 @@ void streamStateDestroy(SStreamState* pState, bool remove) { taosMemoryFreeClear(pState); } -int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark) { - return deleteExpiredCheckPoint(pState->pFileState, mark); -} - void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); } void streamStateCopyBackend(SStreamState* src, SStreamState* dst) { @@ -617,8 +613,6 @@ int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** void streamStateClearExpiredState(SStreamState* pState) { clearExpiredState(pState->pFileState); } -void streamStateSetFillInfo(SStreamState* pState) { setFillInfo(pState->pFileState); } - int32_t streamStateGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen, int32_t* pWinCode) { return getRowStatePrevRow(pState->pFileState, pKey, pResKey, pVal, pVLen, pWinCode); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index f46228fd47..d27ed520c6 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -132,8 +132,12 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool return code; } - char buf[128] = {0}; - sprintf(buf, "0x%" PRIx64 "-0x%x", pTask->id.streamId, pTask->id.taskId); + char buf[128] = {0}; + int32_t ret = snprintf(buf, tListLen(buf), "0x%" PRIx64 "-0x%x", pTask->id.streamId, pTask->id.taskId); + if (ret < 0 || ret >= tListLen(buf)) { + stError("s-task:0x%x failed to set the taskIdstr, code: out of buffer", pTask->id.taskId); + return TSDB_CODE_OUT_OF_BUFFER; + } pTask->id.idStr = taosStrdup(buf); if (pTask->id.idStr == NULL) { @@ -402,7 +406,7 @@ int32_t streamTaskSetBackendPath(SStreamTask* pTask) { } char id[128] = {0}; - int32_t nBytes = sprintf(id, "0x%" PRIx64 "-0x%x", streamId, taskId); + int32_t nBytes = snprintf(id, tListLen(id), "0x%" PRIx64 "-0x%x", streamId, taskId); if (nBytes < 0 || nBytes >= sizeof(id)) { return TSDB_CODE_OUT_OF_BUFFER; } @@ -413,10 +417,14 @@ int32_t streamTaskSetBackendPath(SStreamTask* pTask) { return terrno; } - (void)sprintf(pTask->backendPath, "%s%s%s", pTask->pMeta->path, TD_DIRSEP, id); - stDebug("s-task:%s set backend path:%s", pTask->id.idStr, pTask->backendPath); - - return 0; + int32_t code = snprintf(pTask->backendPath, len + nBytes + 2, "%s%s%s", pTask->pMeta->path, TD_DIRSEP, id); + if (code < 0 || code >= len + nBytes + 2) { + stError("s-task:%s failed to set backend path:%s, code: out of buffer", pTask->id.idStr, pTask->backendPath); + return TSDB_CODE_OUT_OF_BUFFER; + } else { + stDebug("s-task:%s set backend path:%s", pTask->id.idStr, pTask->backendPath); + return 0; + } } int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver) { @@ -1129,7 +1137,11 @@ SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) { int32_t createStreamTaskIdStr(int64_t streamId, int32_t taskId, const char** pId) { char buf[128] = {0}; - sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId); + int32_t code = snprintf(buf, tListLen(buf),"0x%" PRIx64 "-0x%x", streamId, taskId); + if (code < 0 || code >= tListLen(buf)) { + return TSDB_CODE_OUT_OF_BUFFER; + } + *pId = taosStrdup(buf); if (*pId == NULL) { diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 05edad0f5f..aaff58d1b4 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -667,18 +667,6 @@ void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLe } } -int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { - int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen); - int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey); - if (pFileState->searchBuff != NULL) { - deleteHashSortRowBuff(pFileState, pKey); - } - if (code_buff == TSDB_CODE_SUCCESS || code_file == TSDB_CODE_SUCCESS) { - return TSDB_CODE_SUCCESS; - } - return TSDB_CODE_FAILED; -} - static int32_t recoverSessionRowBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; @@ -868,10 +856,6 @@ int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId return streamDefaultDel_rocksdb(pFileState->pFileStore, keyBuf); } -int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list) { - return streamDefaultIterGet_rocksdb(pFileState->pFileStore, TASK_KEY, NULL, list); -} - int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { int32_t code = TSDB_CODE_SUCCESS; int64_t maxCheckPointId = 0; @@ -1227,10 +1211,6 @@ SSHashObj* getGroupIdCache(SStreamFileState* pFileState) { return pFileState->pGroupIdMap; } -void setFillInfo(SStreamFileState* pFileState) { - pFileState->hasFillCatch = false; -} - void clearExpiredState(SStreamFileState* pFileState) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; @@ -1261,6 +1241,7 @@ _end: } } +#ifdef BUILD_NO_CALL int32_t getStateSearchRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, void** pVal, int32_t* pVLen, int32_t* pWinCode) { int32_t code = TSDB_CODE_SUCCESS; @@ -1328,6 +1309,7 @@ _end: } return code; } +#endif int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SWinKey* pResKey, void** ppVal, int32_t* pVLen, int32_t* pWinCode) { diff --git a/source/libs/stream/test/streamCheckPointTest.cpp b/source/libs/stream/test/streamCheckPointTest.cpp index c8297d56b7..c993743dc3 100644 --- a/source/libs/stream/test/streamCheckPointTest.cpp +++ b/source/libs/stream/test/streamCheckPointTest.cpp @@ -390,9 +390,78 @@ TEST(sstreamTaskGetTriggerRecvStatusTest, streamTaskGetTriggerRecvStatusFnTest) extern int8_t tsS3EpNum; tsS3EpNum = 1; - code = uploadCheckpointToS3("123", "/tmp/backend5/stream"); - EXPECT_EQ(code, TSDB_CODE_SUCCESS); + code = uploadCheckpointToS3("123", "/tmp/backend5/stream/stream"); + EXPECT_NE(code, TSDB_CODE_OUT_OF_RANGE); code = downloadCheckpointByNameS3("123", "/root/download", ""); EXPECT_NE(code, TSDB_CODE_OUT_OF_RANGE); + + code = deleteCheckpointFile("aaa123", "bbb"); + EXPECT_NE(code, TSDB_CODE_OUT_OF_RANGE); } + +TEST(doCheckBeforeHandleChkptTriggerTest, doCheckBeforeHandleChkptTriggerFnTest) { + SStreamTask* pTask = NULL; + int64_t uid = 2222222222222; + SArray* array = taosArrayInit(4, POINTER_BYTES); + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array, + false, 1, &pTask); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + initTaskLock(pTask); + + const char *path = "/tmp/doCheckBeforeHandleChkptTriggerTest/stream"; + code = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL, &pTask->pMeta); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + SStreamState *pState = streamStateOpen((char *)path, pTask, 0, 0); + ASSERT(pState != NULL); + + pTask->pBackend = pState->pTdbState->pOwner->pBackend; + + code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.checkpointId = 123; + code = doCheckBeforeHandleChkptTrigger(pTask, 100, NULL, 0); + ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT); + + pTask->chkInfo.pActiveInfo->failedId = 223; + code = doCheckBeforeHandleChkptTrigger(pTask, 200, NULL, 0); + ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT); + + SStreamDataBlock block; + block.srcTaskId = 456; + SStreamTask upTask; + upTask = *pTask; + upTask.id.taskId = 456; + streamTaskSetUpstreamInfo(pTask, &upTask); + pTask->chkInfo.pActiveInfo->failedId = 23; + code = doCheckBeforeHandleChkptTrigger(pTask, 123, &block, 0); + ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT); + + streamTaskSetUpstreamInfo(pTask, &upTask); + streamTaskSetStatusReady(pTask); + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.pActiveInfo->activeId = 223; + + STaskCheckpointReadyInfo readyInfo; + readyInfo.upstreamTaskId = 4567; + block.srcTaskId = 4567; + void* pBuf = rpcMallocCont(sizeof(SMsgHead) + 1); + + initRpcMsg(&readyInfo.msg, 0, pBuf, sizeof(SMsgHead) + 1); + taosArrayPush(pTask->chkInfo.pActiveInfo->pReadyMsgList, &readyInfo); + code = doCheckBeforeHandleChkptTrigger(pTask, 223, &block, 0); + ASSERT_NE(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.pActiveInfo->allUpstreamTriggerRecv = 1; + code = doCheckBeforeHandleChkptTrigger(pTask, 223, &block, 0); + ASSERT_NE(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.pActiveInfo->activeId = 1111; + code = doCheckBeforeHandleChkptTrigger(pTask, 223, &block, 0); + ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT); +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 19a3f211b1..c806bdfbd8 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -2023,16 +2023,29 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { memcpy(pKey, cd.pKey, (size_t)cd.kLen); if (ppVal) { - // TODO: vLen may be zero - pVal = tdbRealloc(*ppVal, cd.vLen); - if (pVal == NULL) { - tdbFree(pKey); - return terrno; + if (cd.vLen > 0) { + pVal = tdbRealloc(*ppVal, cd.vLen); + if (pVal == NULL) { + tdbFree(pKey); + return terrno; + } + + memcpy(pVal, cd.pVal, (size_t)cd.vLen); + if (TDB_CELLDECODER_FREE_VAL(&cd)) { + tdbTrace("tdb/btree-next decoder: %p pVal free: %p", &cd, cd.pVal); + tdbFree(cd.pVal); + } + } else { + pVal = NULL; } *ppVal = pVal; *vLen = cd.vLen; - memcpy(pVal, cd.pVal, (size_t)cd.vLen); + } else { + if (TDB_CELLDECODER_FREE_VAL(&cd)) { + tdbTrace("tdb/btree-next2 decoder: %p pVal free: %p", &cd, cd.pVal); + tdbFree(cd.pVal); + } } ret = tdbBtcMoveToPrev(pBtc); diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 2249123ef5..14845ed8ee 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -66,3 +66,10 @@ int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) { *size = szBytes / szPage; return 0; } + +void tdbCloseDir(TdDirPtr *ppDir) { + int32_t ret = taosCloseDir(ppDir); + if (ret) { + tdbError("failed to close directory, reason:%s", tstrerror(ret)); + } +} diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 993c3ffab8..cd2d4ce57a 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -71,12 +71,7 @@ typedef TdFilePtr tdb_fd_t; #define tdbGetDirEntryName taosGetDirEntryName #define tdbDirEntryBaseName taosDirEntryBaseName -static FORCE_INLINE void tdbCloseDir(TdDirPtr *ppDir) { - int32_t ret = taosCloseDir(ppDir); - if (ret) { - tdbError("failed to close directory, reason:%s", tstrerror(ret)); - } -} +void tdbCloseDir(TdDirPtr *ppDir); #define tdbOsRemove remove #define tdbOsFileSize(FD, PSIZE) taosFStatFile(FD, PSIZE, NULL) diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index 3c2b67da01..5dd9ce568f 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -16,6 +16,10 @@ #ifndef _TD_TFS_INT_H_ #define _TD_TFS_INT_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "os.h" #include "taosdef.h" @@ -74,6 +78,7 @@ typedef struct STfs { SHashObj *hash; // name to did map } STfs; +int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg); int32_t tfsNewDisk(int32_t level, int32_t id, int8_t disable, const char *dir, STfsDisk **ppDisk); STfsDisk *tfsFreeDisk(STfsDisk *pDisk); int32_t tfsUpdateDiskSize(STfsDisk *pDisk); diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index ecc55517b3..5021a6ae39 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -19,7 +19,6 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg); static int32_t tfsCheck(STfs *pTfs); -static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg); static int32_t tfsFormatDir(char *idir, char *odir); static int32_t tfsGetDiskByName(STfs *pTfs, const char *dir, STfsDisk **ppDisk); static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir); @@ -245,13 +244,13 @@ void tfsDirname(const STfsFile *pFile, char *dest) { tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN); tstrncpy(dest, taosDirName(tname), TSDB_FILENAME_LEN); } - +#if 0 void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname) { STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); (void)snprintf(aname, TSDB_FILENAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); } - +#endif int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); } int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2) { @@ -340,7 +339,7 @@ int32_t tfsMkdir(STfs *pTfs, const char *rname) { TAOS_RETURN(0); } - +#if 0 bool tfsDirExistAt(STfs *pTfs, const char *rname, SDiskID diskId) { STfsDisk *pDisk = TFS_DISK_AT(pTfs, diskId); char aname[TMPNAME_LEN]; @@ -348,7 +347,7 @@ bool tfsDirExistAt(STfs *pTfs, const char *rname, SDiskID diskId) { (void)snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); return taosDirExist(aname); } - +#endif int32_t tfsRmdir(STfs *pTfs, const char *rname) { if (rname[0] == 0) { TAOS_RETURN(0); @@ -515,7 +514,7 @@ _exit: TAOS_RETURN(code); } -static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { +int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { int32_t code = 0; char dirName[TSDB_FILENAME_LEN] = "\0"; @@ -577,32 +576,32 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { } static int32_t tfsFormatDir(char *idir, char *odir) { + int32_t code = 0, lino = 0; wordexp_t wep = {0}; + int32_t dirLen = 0; + char tmp[PATH_MAX] = {0}; - int32_t code = wordexp(idir, &wep, 0); + code = wordexp(idir, &wep, 0); if (code != 0) { - TAOS_RETURN(TAOS_SYSTEM_ERROR(code)); + TAOS_CHECK_EXIT(TAOS_SYSTEM_ERROR(code)); } - char tmp[PATH_MAX] = {0}; - if (taosRealPath(wep.we_wordv[0], tmp, PATH_MAX) != 0) { - code = TAOS_SYSTEM_ERROR(errno); - wordfree(&wep); - TAOS_RETURN(code); - } + TAOS_CHECK_EXIT(taosRealPath(wep.we_wordv[0], tmp, PATH_MAX)); - int32_t dirLen = strlen(tmp); + dirLen = strlen(tmp); if (dirLen < 0 || dirLen >= TSDB_FILENAME_LEN) { - wordfree(&wep); - code = TSDB_CODE_OUT_OF_RANGE; - fError("failed to mount %s to FS since %s, real path:%s, len:%d", idir, tstrerror(code), tmp, dirLen); - TAOS_RETURN(code); + TAOS_CHECK_EXIT(TSDB_CODE_OUT_OF_RANGE); } tstrncpy(odir, tmp, TSDB_FILENAME_LEN); +_exit: wordfree(&wep); - TAOS_RETURN(0); + if (code != 0) { + fError("failed to mount %s to FS at line %d since %s, real path:%s, len:%d", idir, lino, tstrerror(code), tmp, + dirLen); + } + TAOS_RETURN(code); } static int32_t tfsCheck(STfs *pTfs) { diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index 2cfcdc6d0a..acc8168538 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -41,13 +41,13 @@ void tfsDestroyTier(STfsTier *pTier) { int32_t tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg, STfsDisk **ppDisk) { int32_t code = 0; int32_t lino = 0; + int32_t id = 0; STfsDisk *pDisk = NULL; if (pTier->ndisk >= TFS_MAX_DISKS_PER_TIER) { TAOS_CHECK_GOTO(TSDB_CODE_FS_TOO_MANY_MOUNT, &lino, _exit); } - int32_t id = 0; if (pTier->level == 0) { if (pTier->disks[0] != NULL) { id = pTier->ndisk; diff --git a/source/libs/tfs/test/CMakeLists.txt b/source/libs/tfs/test/CMakeLists.txt index 2fd0836a1d..050811f0f5 100644 --- a/source/libs/tfs/test/CMakeLists.txt +++ b/source/libs/tfs/test/CMakeLists.txt @@ -7,8 +7,13 @@ target_link_libraries( PUBLIC tfs PUBLIC gtest_main ) +target_include_directories( + tfs_test + PUBLIC "${TD_SOURCE_DIR}/include/libs/tfs" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) -# add_test( -# NAME tfs_test -# COMMAND tfs_test -# ) +add_test( + NAME tfs_test + COMMAND tfs_test +) diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index bb89fbe69f..1570cf173f 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -13,6 +13,7 @@ #include "os.h" #include "tfs.h" +#include "tfsInt.h" class TfsTest : public ::testing::Test { protected: @@ -280,6 +281,9 @@ TEST_F(TfsTest, 04_File) { const STfsFile *pf2 = tfsReaddir(pDir); EXPECT_EQ(pf2, nullptr); + pDir->pDir = taosOpenDir(fulldir); + EXPECT_NE(pDir->pDir, nullptr); + tfsClosedir(pDir); } @@ -744,3 +748,116 @@ TEST_F(TfsTest, 05_MultiDisk) { tfsClose(pTfs); } + +TEST_F(TfsTest, 06_Misc) { + // tfsDisk.c + STfsDisk *pDisk = NULL; + EXPECT_EQ(tfsNewDisk(0, 0, 0, NULL, &pDisk), TSDB_CODE_INVALID_PARA); + EXPECT_NE(tfsNewDisk(0, 0, 0, "", &pDisk), 0); + + STfsDisk disk = {0}; + EXPECT_EQ(tfsUpdateDiskSize(&disk), TSDB_CODE_INVALID_PARA); + + // tfsTier.c + STfsTier tfsTier = {0}; + EXPECT_EQ(taosThreadSpinInit(&tfsTier.lock, 0), 0); + EXPECT_EQ(tfsAllocDiskOnTier(&tfsTier), TSDB_CODE_FS_NO_VALID_DISK); + + tfsTier.ndisk = 3; + tfsTier.nAvailDisks = 1; + + tfsTier.disks[1] = &disk; + disk.disable = 1; + EXPECT_EQ(tfsAllocDiskOnTier(&tfsTier), TSDB_CODE_FS_NO_VALID_DISK); + disk.disable = 0; + disk.size.avail = 0; + EXPECT_EQ(tfsAllocDiskOnTier(&tfsTier), TSDB_CODE_FS_NO_VALID_DISK); + + tfsTier.ndisk = TFS_MAX_DISKS_PER_TIER; + SDiskCfg diskCfg = {0}; + tstrncpy(diskCfg.dir, "testDataDir", TSDB_FILENAME_LEN); + EXPECT_EQ(tfsMountDiskToTier(&tfsTier, &diskCfg, 0), TSDB_CODE_FS_TOO_MANY_MOUNT); + EXPECT_EQ(taosThreadSpinDestroy(&tfsTier.lock), 0); + + // tfs.c + STfs *pTfs = NULL; + EXPECT_EQ(tfsOpen(0, -1, &pTfs), TSDB_CODE_INVALID_PARA); + EXPECT_EQ(tfsOpen(0, 0, &pTfs), TSDB_CODE_INVALID_PARA); + EXPECT_EQ(tfsOpen(0, TFS_MAX_DISKS + 1, &pTfs), TSDB_CODE_INVALID_PARA); + taosMemoryFreeClear(pTfs); + + STfs tfs = {0}; + STfsTier *pTier = &tfs.tiers[0]; + EXPECT_EQ(tfsDiskSpaceAvailable(&tfs, -1), false); + tfs.nlevel = 2; + pTier->ndisk = 3; + pTier->nAvailDisks = 1; + EXPECT_EQ(tfsDiskSpaceAvailable(&tfs, 0), false); + pTier->disks[0] = &disk; + EXPECT_EQ(tfsDiskSpaceAvailable(&tfs, 0), false); + + EXPECT_EQ(tfsDiskSpaceSufficient(&tfs, -1, 0), false); + EXPECT_EQ(tfsDiskSpaceSufficient(&tfs, tfs.nlevel + 1, 0), false); + EXPECT_EQ(tfsDiskSpaceSufficient(&tfs, 0, -1), false); + EXPECT_EQ(tfsDiskSpaceSufficient(&tfs, 0, pTier->ndisk), false); + + EXPECT_EQ(tfsGetDisksAtLevel(&tfs, -1), 0); + EXPECT_EQ(tfsGetDisksAtLevel(&tfs, tfs.nlevel), 0); + + EXPECT_EQ(tfsGetLevel(&tfs), tfs.nlevel); + + for (int32_t l = 0; l < tfs.nlevel; ++l) { + EXPECT_EQ(taosThreadSpinInit(&tfs.tiers[l].lock, 0), 0); + } + + SDiskID diskID = {0}; + disk.size.avail = TFS_MIN_DISK_FREE_SIZE; + EXPECT_EQ(tfsAllocDisk(&tfs, tfs.nlevel, &diskID), 0); + tfs.nlevel = 0; + diskID.level = 0; + EXPECT_EQ(tfsAllocDisk(&tfs, 0, &diskID), 0); + tfs.nlevel = 2; + + diskID.id = 10; + EXPECT_EQ(tfsMkdirAt(&tfs, NULL, diskID), TSDB_CODE_FS_INVLD_CFG); + + EXPECT_NE(tfsMkdirRecurAt(&tfs, NULL, diskID), 0); + + const char *rname = ""; + EXPECT_EQ(tfsRmdir(&tfs, rname), 0); + + EXPECT_EQ(tfsSearch(&tfs, -1, NULL), -1); + EXPECT_EQ(tfsSearch(&tfs, tfs.nlevel, NULL), -1); + + diskCfg.level = -1; + EXPECT_EQ(tfsCheckAndFormatCfg(&tfs, &diskCfg), TSDB_CODE_FS_INVLD_CFG); + diskCfg.level = TFS_MAX_TIERS; + EXPECT_EQ(tfsCheckAndFormatCfg(&tfs, &diskCfg), TSDB_CODE_FS_INVLD_CFG); + diskCfg.level = 0; + diskCfg.primary = -1; + EXPECT_EQ(tfsCheckAndFormatCfg(&tfs, &diskCfg), TSDB_CODE_FS_INVLD_CFG); + diskCfg.primary = 2; + EXPECT_EQ(tfsCheckAndFormatCfg(&tfs, &diskCfg), TSDB_CODE_FS_INVLD_CFG); + diskCfg.primary = 1; + diskCfg.disable = -1; + EXPECT_EQ(tfsCheckAndFormatCfg(&tfs, &diskCfg), TSDB_CODE_FS_INVLD_CFG); + diskCfg.disable = 2; + EXPECT_EQ(tfsCheckAndFormatCfg(&tfs, &diskCfg), TSDB_CODE_FS_INVLD_CFG); + diskCfg.disable = 0; + diskCfg.level = 1; + EXPECT_EQ(tfsCheckAndFormatCfg(&tfs, &diskCfg), TSDB_CODE_FS_INVLD_CFG); + diskCfg.level = 0; + diskCfg.primary = 0; + tstrncpy(diskCfg.dir, "testDataDir1", TSDB_FILENAME_LEN); + EXPECT_NE(tfsCheckAndFormatCfg(&tfs, &diskCfg), 0); + + TdFilePtr pFile = taosCreateFile("testDataDir1", TD_FILE_CREATE); + EXPECT_NE(pFile, nullptr); + EXPECT_EQ(tfsCheckAndFormatCfg(&tfs, &diskCfg), TSDB_CODE_FS_INVLD_CFG); + EXPECT_EQ(taosCloseFile(&pFile), 0); + EXPECT_EQ(taosRemoveFile("testDataDir1"), 0); + + for (int32_t l = 0; l < tfs.nlevel; ++l) { + EXPECT_EQ(taosThreadSpinDestroy(&tfs.tiers[l].lock), 0); + } +} diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 468d9d9b50..3b84fc4574 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -151,7 +151,6 @@ typedef struct SCliThrd { TdThreadMutex msgMtx; SDelayQueue* delayQueue; SDelayQueue* timeoutQueue; - SDelayQueue* waitConnQueue; uint64_t nextTimeout; // next timeout STrans* pInst; // @@ -159,8 +158,6 @@ typedef struct SCliThrd { SHashObj* fqdn2ipCache; SCvtAddr* pCvtAddr; - SHashObj* failFastCache; - SHashObj* batchCache; SHashObj* connHeapCache; SCliReq* stopMsg; @@ -224,8 +221,6 @@ static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); static void cliConnCb(uv_connect_t* req, int status); static void cliAsyncCb(uv_async_t* handle); -SCliBatch* cliGetHeadFromList(SCliBatchList* pList); - static void destroyCliConnQTable(SCliConn* conn); static void cliHandleException(SCliConn* conn); @@ -1299,8 +1294,8 @@ static void cliHandleException(SCliConn* conn) { if (conn->registered) { int8_t ref = transGetRefCount(conn); if (ref == 0 && !uv_is_closing((uv_handle_t*)conn->stream)) { -// tTrace("%s conn %p fd %d,%d,%d,%p uv_closed", CONN_GET_INST_LABEL(conn), conn, conn->stream->u.fd, -// conn->stream->io_watcher.fd, conn->stream->accepted_fd, conn->stream->queued_fds); + // tTrace("%s conn %p fd %d,%d,%d,%p uv_closed", CONN_GET_INST_LABEL(conn), conn, conn->stream->u.fd, + // conn->stream->io_watcher.fd, conn->stream->accepted_fd, conn->stream->queued_fds); uv_close((uv_handle_t*)conn->stream, cliDestroy); } } @@ -2124,144 +2119,7 @@ static void cliDoReq(queue* wq, SCliThrd* pThrd) { tTrace("cli process batch size:%d", count); } } -SCliBatch* cliGetHeadFromList(SCliBatchList* pList) { - if (QUEUE_IS_EMPTY(&pList->wq) || pList->connCnt > pList->connMax || pList->sending > pList->connMax) { - return NULL; - } - queue* hr = QUEUE_HEAD(&pList->wq); - QUEUE_REMOVE(hr); - pList->sending += 1; - pList->len -= 1; - - SCliBatch* batch = QUEUE_DATA(hr, SCliBatch, listq); - return batch; -} -static int32_t createBatch(SCliBatch** ppBatch, SCliBatchList* pList, SCliReq* pReq); - -static int32_t createBatchList(SCliBatchList** ppBatchList, char* key, char* ip, uint32_t port); - -static void destroyBatchList(SCliBatchList* pList); -static void cliBuildBatch(SCliReq* pReq, queue* h, SCliThrd* pThrd) { - int32_t code = 0; - STrans* pInst = pThrd->pInst; - SReqCtx* pCtx = pReq->ctx; - - char* ip = EPSET_GET_INUSE_IP(pCtx->epSet); - uint32_t port = EPSET_GET_INUSE_PORT(pCtx->epSet); - char key[TSDB_FQDN_LEN + 64] = {0}; - CONN_CONSTRUCT_HASH_KEY(key, ip, port); - size_t klen = strlen(key); - SCliBatchList** ppBatchList = taosHashGet(pThrd->batchCache, key, klen); - if (ppBatchList == NULL || *ppBatchList == NULL) { - SCliBatchList* pBatchList = NULL; - code = createBatchList(&pBatchList, key, ip, port); - if (code != 0) { - destroyReq(pReq); - return; - } - - pBatchList->batchLenLimit = pInst->shareConnLimit; - - SCliBatch* pBatch = NULL; - code = createBatch(&pBatch, pBatchList, pReq); - if (code != 0) { - destroyBatchList(pBatchList); - destroyReq(pReq); - return; - } - - code = taosHashPut(pThrd->batchCache, key, klen, &pBatchList, sizeof(void*)); - if (code != 0) { - destroyBatchList(pBatchList); - } - } else { - if (QUEUE_IS_EMPTY(&(*ppBatchList)->wq)) { - SCliBatch* pBatch = NULL; - code = createBatch(&pBatch, *ppBatchList, pReq); - if (code != 0) { - destroyReq(pReq); - cliDestroyBatch(pBatch); - } - } else { - queue* hdr = QUEUE_TAIL(&((*ppBatchList)->wq)); - SCliBatch* pBatch = QUEUE_DATA(hdr, SCliBatch, listq); - if ((pBatch->shareConnLimit + pReq->msg.contLen) < (*ppBatchList)->batchLenLimit) { - QUEUE_PUSH(&pBatch->wq, h); - pBatch->shareConnLimit += pReq->msg.contLen; - pBatch->wLen += 1; - } else { - SCliBatch* tBatch = NULL; - code = createBatch(&tBatch, *ppBatchList, pReq); - if (code != 0) { - destroyReq(pReq); - } - } - } - } - return; -} -static int32_t createBatchList(SCliBatchList** ppBatchList, char* key, char* ip, uint32_t port) { - SCliBatchList* pBatchList = taosMemoryCalloc(1, sizeof(SCliBatchList)); - if (pBatchList == NULL) { - tError("failed to create batch list since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return terrno; - } - QUEUE_INIT(&pBatchList->wq); - pBatchList->port = port; - pBatchList->connMax = 1; - pBatchList->connCnt = 0; - pBatchList->batchLenLimit = 0; - pBatchList->len += 1; - - pBatchList->ip = taosStrdup(ip); - pBatchList->dst = taosStrdup(key); - if (pBatchList->ip == NULL || pBatchList->dst == NULL) { - taosMemoryFree(pBatchList->ip); - taosMemoryFree(pBatchList->dst); - taosMemoryFree(pBatchList); - tError("failed to create batch list since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return terrno; - } - *ppBatchList = pBatchList; - return 0; -} -static void destroyBatchList(SCliBatchList* pList) { - if (pList == NULL) { - return; - } - while (!QUEUE_IS_EMPTY(&pList->wq)) { - queue* h = QUEUE_HEAD(&pList->wq); - QUEUE_REMOVE(h); - - SCliBatch* pBatch = QUEUE_DATA(h, SCliBatch, listq); - cliDestroyBatch(pBatch); - } - taosMemoryFree(pList->ip); - taosMemoryFree(pList->dst); - taosMemoryFree(pList); -} -static int32_t createBatch(SCliBatch** ppBatch, SCliBatchList* pList, SCliReq* pReq) { - SCliBatch* pBatch = taosMemoryCalloc(1, sizeof(SCliBatch)); - if (pBatch == NULL) { - tError("failed to create batch since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return terrno; - } - - QUEUE_INIT(&pBatch->wq); - QUEUE_INIT(&pBatch->listq); - - QUEUE_PUSH(&pBatch->wq, &pReq->q); - pBatch->wLen += 1; - pBatch->shareConnLimit = pReq->msg.contLen; - pBatch->pList = pList; - - QUEUE_PUSH(&pList->wq, &pBatch->listq); - pList->len += 1; - - *ppBatch = pBatch; - return 0; -} static void cliDoBatchReq(queue* wq, SCliThrd* pThrd) { return cliDoReq(wq, pThrd); } static void cliAsyncCb(uv_async_t* handle) { @@ -2494,10 +2352,6 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(code, NULL, _end); } - if ((code = transDQCreate(pThrd->loop, &pThrd->waitConnQueue)) != 0) { - TAOS_CHECK_GOTO(code, NULL, _end); - } - pThrd->destroyAhandleFp = pInst->destroyFp; pThrd->fqdn2ipCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); @@ -2505,11 +2359,6 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(terrno, NULL, _end); } - pThrd->batchCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (pThrd->batchCache == NULL) { - TAOS_CHECK_GOTO(terrno, NULL, _end); - } - pThrd->connHeapCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (pThrd->connHeapCache == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); @@ -2553,10 +2402,7 @@ _end: transDQDestroy(pThrd->delayQueue, NULL); transDQDestroy(pThrd->timeoutQueue, NULL); - transDQDestroy(pThrd->waitConnQueue, NULL); taosHashCleanup(pThrd->fqdn2ipCache); - taosHashCleanup(pThrd->failFastCache); - taosHashCleanup(pThrd->batchCache); taosHashCleanup(pThrd->pIdConnTable); taosArrayDestroy(pThrd->pQIdBuf); @@ -2580,7 +2426,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { transDQDestroy(pThrd->delayQueue, destroyReqAndAhanlde); transDQDestroy(pThrd->timeoutQueue, NULL); - transDQDestroy(pThrd->waitConnQueue, NULL); tDebug("thread destroy %" PRId64, pThrd->pid); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { @@ -2592,24 +2437,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); - void** pIter = taosHashIterate(pThrd->batchCache, NULL); - while (pIter != NULL) { - SCliBatchList* pBatchList = (SCliBatchList*)(*pIter); - while (!QUEUE_IS_EMPTY(&pBatchList->wq)) { - queue* h = QUEUE_HEAD(&pBatchList->wq); - QUEUE_REMOVE(h); - - SCliBatch* pBatch = QUEUE_DATA(h, SCliBatch, listq); - cliDestroyBatch(pBatch); - } - taosMemoryFree(pBatchList->ip); - taosMemoryFree(pBatchList->dst); - taosMemoryFree(pBatchList); - - pIter = (void**)taosHashIterate(pThrd->batchCache, pIter); - } - taosHashCleanup(pThrd->batchCache); - void* pIter2 = taosHashIterate(pThrd->connHeapCache, NULL); while (pIter2 != NULL) { SHeap* heap = (SHeap*)(pIter2); diff --git a/source/libs/transport/test/transUT2.cpp b/source/libs/transport/test/transUT2.cpp index 6dfb5e503a..fabe9e9c4f 100644 --- a/source/libs/transport/test/transUT2.cpp +++ b/source/libs/transport/test/transUT2.cpp @@ -615,6 +615,21 @@ TEST_F(TransEnv, http) { #endif } +#if 1 + STelemAddrMgmt mgt; + taosTelemetryMgtInit(&mgt, "telemetry.taosdata.com"); + int32_t code = taosSendTelemReport(&mgt,tsTelemUri, tsTelemPort, "test", strlen("test"),HTTP_FLAT); + printf("old addr:%s new addr:%s\n",mgt.defaultAddr, mgt.cachedAddr); + + taosMsleep(2000); + code = taosSendTelemReport(&mgt,tsTelemUri, tsTelemPort, pCont, len,HTTP_FLAT); + for (int32_t i = 0; i < 1; i++) { + code = taosSendTelemReport(&mgt,tsTelemUri, tsTelemPort, pCont, len,HTTP_FLAT); + printf("old addr:%s new addr:%s\n",mgt.defaultAddr, mgt.cachedAddr); + taosMsleep(2000); + } + taosTelemetryDestroy(&mgt); +#endif { STelemAddrMgmt mgt; taosTelemetryMgtInit(&mgt, "error"); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 8a2606c4c2..b1198e1cb2 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -1403,7 +1403,7 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { } (*ptrBuf)[totalBytesRead] = '\0'; - ret = totalBytesRead; + ret = (totalBytesRead > 0 ? totalBytesRead : -1); // -1 means EOF #else size_t len = 0; ret = getline(ptrBuf, &len, pFile->fp); diff --git a/source/util/inc/tlogInt.h b/source/util/inc/tlogInt.h new file mode 100644 index 0000000000..1d7f3a063d --- /dev/null +++ b/source/util/inc/tlogInt.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_LOG_INT_H_ +#define _TD_UTIL_LOG_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlog.h" + +void taosOpenNewSlowLogFile(); +void taosLogObjSetToday(int64_t ts); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_LOG_INT_H_*/ diff --git a/source/util/inc/tmempoolInt.h b/source/util/inc/tmempoolInt.h index 8d4cb57ddc..013838e6df 100755 --- a/source/util/inc/tmempoolInt.h +++ b/source/util/inc/tmempoolInt.h @@ -222,7 +222,7 @@ typedef struct SMPSessionChunk { } SMPSessionChunk; typedef struct SMPSession { - // SMPListNode list; + SMPListNode list; char* sessionId; SMPJob* pJob; diff --git a/source/util/src/tanalytics.c b/source/util/src/tanalytics.c index e68edd4b76..bf2cb4fd07 100644 --- a/source/util/src/tanalytics.c +++ b/source/util/src/tanalytics.c @@ -20,7 +20,7 @@ #ifdef USE_ANALYTICS #include -#define ANAL_ALGO_SPLIT "," +#define ANALYTICS_ALOG_SPLIT_CHAR "," typedef struct { int64_t ver; @@ -136,7 +136,7 @@ bool taosAnalGetOptStr(const char *option, const char *optName, char *optValue, return false; } - pEnd = strstr(pStart, ANAL_ALGO_SPLIT); + pEnd = strstr(pStart, ANALYTICS_ALOG_SPLIT_CHAR); if (optMaxLen > 0) { if (pEnd > pStart) { int32_t len = (int32_t)(pEnd - pStart); @@ -168,7 +168,7 @@ bool taosAnalGetOptInt(const char *option, const char *optName, int64_t *optValu int32_t bufLen = tsnprintf(buf, sizeof(buf), "%s=", optName); char *pos1 = strstr(option, buf); - char *pos2 = strstr(option, ANAL_ALGO_SPLIT); + char *pos2 = strstr(option, ANALYTICS_ALOG_SPLIT_CHAR); if (pos1 != NULL) { *optValue = taosStr2Int64(pos1 + bufLen, NULL, 10); return true; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a9eef1bfc9..4df5b322a2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -483,7 +483,7 @@ static int32_t taosOpenNewLogFile() { return 0; } -static void taosOpenNewSlowLogFile() { +void taosOpenNewSlowLogFile() { (void)taosThreadMutexLock(&tsLogObj.logMutex); int64_t delta = taosGetTimestampSec() - tsLogObj.timestampToday; if (delta >= 0 && delta < 86400) { @@ -539,6 +539,8 @@ void taosResetLog() { } } +void taosLogObjSetToday(int64_t ts) { tsLogObj.timestampToday = ts; } + static bool taosCheckFileIsOpen(char *logFileName) { TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE); if (pFile == NULL) { @@ -619,6 +621,7 @@ static void processLogFileName(const char *logName, int32_t maxFileNum) { } static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { + int32_t code = 0, lino = 0; #ifdef WINDOWS_STASH /* * always set maxFileNum to 1 @@ -653,39 +656,28 @@ static int32_t taosInitNormalLog(const char *logName, int32_t maxFileNum) { // only an estimate for number of lines int64_t filesize = 0; - if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) != 0) { - (void)printf("\nfailed to fstat log file:%s, reason:%s\n", name, strerror(errno)); - taosUnLockLogFile(tsLogObj.logHandle->pFile); - return terrno; - } + TAOS_CHECK_EXIT(taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL)); + tsLogObj.lines = (int32_t)(filesize / 60); if (taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END) < 0) { - TAOS_UNUSED(printf("failed to seek to the end of log file:%s, reason:%s\n", name, tstrerror(terrno))); - taosUnLockLogFile(tsLogObj.logHandle->pFile); - return terrno; + TAOS_CHECK_EXIT(terrno); } - (void)snprintf(name, sizeof(name), "==================================================\n"); + (void)snprintf(name, sizeof(name), + "==================================================\n" + " new log file\n" + "==================================================\n"); if (taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)) <= 0) { - TAOS_UNUSED(printf("failed to write to log file:%s, reason:%s\n", name, tstrerror(terrno))); - taosUnLockLogFile(tsLogObj.logHandle->pFile); - return terrno; - } - (void)snprintf(name, sizeof(name), " new log file \n"); - if (taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)) <= 0) { - TAOS_UNUSED(printf("failed to write to log file:%s, reason:%s\n", name, tstrerror(terrno))); - taosUnLockLogFile(tsLogObj.logHandle->pFile); - return terrno; - } - (void)snprintf(name, sizeof(name), "==================================================\n"); - if (taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)) <= 0) { - TAOS_UNUSED(printf("failed to write to log file:%s, reason:%s\n", name, tstrerror(terrno))); - taosUnLockLogFile(tsLogObj.logHandle->pFile); - return terrno; + TAOS_CHECK_EXIT(terrno); } - return 0; +_exit: + if (code != 0) { + taosUnLockLogFile(tsLogObj.logHandle->pFile); + TAOS_UNUSED(printf("failed to init normal log file:%s at line %d, reason:%s\n", name, lino, tstrerror(code))); + } + return code; } static void taosUpdateLogNums(ELogLevel level) { diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index cde1392216..ec05a4e415 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -137,6 +137,10 @@ add_test( NAME logTest COMMAND logTest ) +target_include_directories( + logTest + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) IF(COMPILER_SUPPORT_AVX2) MESSAGE(STATUS "AVX2 instructions is ACTIVATED") diff --git a/source/util/test/log.cpp b/source/util/test/log.cpp index ba32d2d639..1899aac2c4 100644 --- a/source/util/test/log.cpp +++ b/source/util/test/log.cpp @@ -2,7 +2,9 @@ #include #include #include +#include #include +#include #include using namespace std; @@ -44,3 +46,96 @@ TEST(log, check_log_refactor) { } taosCloseLog(); } + +extern char *tsLogOutput; +TEST(log, misc) { + // taosInitLog + const char *path = TD_TMP_DIR_PATH "td"; + taosRemoveDir(path); + taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); + EXPECT_EQ(taosInitLog("taoslog", 1, true), 0); + + taosOpenNewSlowLogFile(); + taosLogObjSetToday(INT64_MIN); + taosPrintSlowLog("slow log test"); + + // test taosInitLogOutput + const char *pLogName = NULL; + tsLogOutput = (char *)taosMemCalloc(1, TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), TSDB_CODE_INVALID_CFG); + tstrncpy(tsLogOutput, "stdout", TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), 0); + tstrncpy(tsLogOutput, "stderr", TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), 0); + tstrncpy(tsLogOutput, "/dev/null", TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), 0); + tsLogOutput[0] = '#'; + EXPECT_EQ(taosInitLogOutput(&pLogName), TSDB_CODE_INVALID_CFG); + tstrncpy(tsLogOutput, "/", TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), 0); + tstrncpy(tsLogOutput, "\\", TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), 0); + tstrncpy(tsLogOutput, "testLogOutput", TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), 0); + tstrncpy(tsLogOutput, "testLogOutputDir/testLogOutput", TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), 0); + tstrncpy(tsLogOutput, ".", TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), TSDB_CODE_INVALID_CFG); + tstrncpy(tsLogOutput, "/..", TSDB_FILENAME_LEN); + EXPECT_EQ(taosInitLogOutput(&pLogName), TSDB_CODE_INVALID_CFG); + tsLogOutput[0] = 0; + + // test taosAssertDebug + tsAssert = false; + taosAssertDebug(true, __FILE__, __LINE__, 0, "test_assert_true_without_core"); + taosAssertDebug(false, __FILE__, __LINE__, 0, "test_assert_false_with_core"); + tsAssert = true; + + // test taosLogCrashInfo, taosReadCrashInfo and taosReleaseCrashLogFile + char nodeType[16] = "nodeType"; + char *pCrashMsg = (char *)taosMemoryCalloc(1, 16); + EXPECT_NE(pCrashMsg, nullptr); + tstrncpy(pCrashMsg, "crashMsg", 16); + +#if !defined(_TD_DARWIN_64) && !defined(WINDOWS) + pid_t pid = taosGetPId(); + EXPECT_EQ(pid > 0, true); + siginfo_t sigInfo = {0}; + sigInfo.si_pid = pid; + taosLogCrashInfo(nodeType, pCrashMsg, strlen(pCrashMsg), 0, &sigInfo); +#else + taosLogCrashInfo(nodeType, pCrashMsg, strlen(pCrashMsg), 0, nullptr); +#endif + + char crashInfo[PATH_MAX] = {0}; + snprintf(crashInfo, sizeof(crashInfo), "%s%s.%sCrashLog", tsLogDir, TD_DIRSEP, nodeType); + + char *pReadMsg = NULL; + int64_t readMsgLen = 0; + TdFilePtr pFile = NULL; + taosReadCrashInfo(crashInfo, &pReadMsg, &readMsgLen, &pFile); + EXPECT_NE(pReadMsg, nullptr); + EXPECT_NE(pFile, nullptr); + EXPECT_EQ(strncasecmp(pReadMsg, "crashMsg", strlen("crashMsg")), 0); + EXPECT_EQ(taosCloseFile(&pFile), 0); + taosMemoryFreeClear(pReadMsg); + + pFile = taosOpenFile(crashInfo, TD_FILE_WRITE); + EXPECT_NE(pFile, nullptr); + EXPECT_EQ(taosWriteFile(pFile, "00000", 1), 1); + EXPECT_EQ(taosCloseFile(&pFile), 0); + + taosReadCrashInfo(crashInfo, &pReadMsg, &readMsgLen, &pFile); + EXPECT_EQ(pReadMsg, nullptr); + EXPECT_EQ(pFile, nullptr); + + pFile = taosOpenFile(crashInfo, TD_FILE_WRITE); + EXPECT_NE(pFile, nullptr); + taosReleaseCrashLogFile(pFile, true); + + // clean up + taosRemoveDir(path); + + taosCloseLog(); +} diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index e59121f631..16e9b1d62c 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -47,7 +47,7 @@ class TDTestCase: tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE -3650d,-10d COMPACT_TIME_OFFSET 0h") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE 0d,0d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd4') tdSql.checkRows(1) @@ -65,7 +65,7 @@ class TDTestCase: tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE -3650d,-10d COMPACT_TIME_OFFSET 0h") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0h ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1 COMPACT_INTERVAL 1d COMPACT_TIME_RANGE 0d,0d COMPACT_TIME_OFFSET 0h") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/custom_col_tag.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/custom_col_tag.json index fec5775cd6..b844f7b1b8 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/custom_col_tag.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/custom_col_tag.json @@ -36,7 +36,7 @@ "insert_mode": "taosc", "line_protocol": "line", "childtable_limit": -10, - "childtable_offset": 10, + "childtable_offset": 0, "insert_rows": 20, "insert_interval": 0, "interlace_rows": 0, diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_insert_alltypes.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_insert_alltypes.json index 5694b58407..5ba870a3bd 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_insert_alltypes.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_insert_alltypes.json @@ -36,7 +36,7 @@ "insert_mode": "taosc", "line_protocol": "line", "childtable_limit": -10, - "childtable_offset": 10, + "childtable_offset": 0, "insert_rows": 20, "insert_interval": 0, "interlace_rows": 0, diff --git a/tests/docs-examples-test/go.sh b/tests/docs-examples-test/go.sh index 606265435d..fa02b33b10 100644 --- a/tests/docs-examples-test/go.sh +++ b/tests/docs-examples-test/go.sh @@ -61,6 +61,11 @@ check_transactions || exit 1 reset_cache || exit 1 go run ./stmt/ws/main.go +taos -s "drop database if exists power" +check_transactions || exit 1 +reset_cache || exit 1 +go run ./stmt2/native/main.go + taos -s "drop database if exists power" check_transactions || exit 1 reset_cache || exit 1 diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index 3a9812637c..536155437b 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -196,3 +196,4 @@ check_transactions || exit 1 reset_cache || exit 1 python3 tmq_websocket_example.py +python3 stmt2_native.py \ No newline at end of file diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ed73a2742c..5fce3821da 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -6,6 +6,16 @@ ,,n,unit-test,bash test.sh +#docs-examples test +,,n,docs-examples-test,bash c.sh +,,n,docs-examples-test,bash python.sh +,,n,docs-examples-test,bash node.sh +,,n,docs-examples-test,bash csharp.sh +,,n,docs-examples-test,bash jdbc.sh +,,n,docs-examples-test,bash rust.sh +,,n,docs-examples-test,bash go.sh +,,n,docs-examples-test,bash test_R.sh + # # army-test # @@ -411,6 +421,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_auto.py ,,n,system-test,python3 ./test.py -f 0-others/dumpsdb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact.py -N 3 @@ -442,6 +453,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_wide_column.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_column_value.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_from_csv.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_benchmark.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -R @@ -481,6 +493,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ins_filesets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/grant.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R @@ -1382,6 +1395,8 @@ ,,y,script,./test.sh -f tsim/stream/basic2.sim ,,y,script,./test.sh -f tsim/stream/basic3.sim ,,y,script,./test.sh -f tsim/stream/basic4.sim +,,y,script,./test.sh -f tsim/stream/basic5.sim +,,y,script,./test.sh -f tsim/stream/tag.sim ,,y,script,./test.sh -f tsim/stream/snodeCheck.sim ,,y,script,./test.sh -f tsim/stream/concurrentcheckpt.sim ,,y,script,./test.sh -f tsim/stream/checkpointInterval0.sim @@ -1657,13 +1672,3 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R - -#docs-examples test -,,n,docs-examples-test,bash c.sh -,,n,docs-examples-test,bash python.sh -,,n,docs-examples-test,bash node.sh -,,n,docs-examples-test,bash csharp.sh -,,n,docs-examples-test,bash jdbc.sh -,,n,docs-examples-test,bash rust.sh -,,n,docs-examples-test,bash go.sh -,,n,docs-examples-test,bash test_R.sh diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index a78d0aa4a4..5dc1cef673 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -77,7 +77,7 @@ md5sum /usr/lib/libtaos.so.1 md5sum /home/TDinternal/debug/build/lib/libtaos.so #get python connector and update: taospy 2.7.16 taos-ws-py 0.3.5 -pip3 install taospy==2.7.16 +pip3 install taospy==2.7.21 pip3 install taos-ws-py==0.3.5 $TIMEOUT_CMD $cmd RET=$? diff --git a/tests/perf-test/stream.py b/tests/perf-test/stream.py new file mode 100644 index 0000000000..a34fe5381b --- /dev/null +++ b/tests/perf-test/stream.py @@ -0,0 +1,286 @@ +import json +import subprocess + +import psutil +import time +import taos + + +class MonitorSystemLoad: + + def __init__(self, name, count) -> None: + self.pid = self.get_pid_by_name(name) + self.count = count + + def get_pid_by_name(self, name): + for proc in psutil.process_iter(['pid', 'name']): + if proc.info['name'] == name: + return proc.info['pid'] + return None + + def get_proc_status(self): + process = psutil.Process(self.pid) + + while True: + cpu_percent = process.cpu_percent(interval=1) + + memory_info = process.memory_info() + memory_percent = process.memory_percent() + + io_counters = process.io_counters() + sys_load = psutil.getloadavg() + + print("load: %s, CPU:%s, Mem:%.2f MiB(%.2f%%), Read: %.2fMiB(%d), Write: %.2fMib (%d)" % ( + sys_load, cpu_percent, memory_info.rss / 1048576.0, + memory_percent, io_counters.read_bytes / 1048576.0, io_counters.read_count, + io_counters.write_bytes / 1048576.0, io_counters.write_count)) + + time.sleep(1) + self.count -= 1 + + if self.count <= 0: + break + + +class StreamStarter: + def __init__(self) -> None: + self.sql = None + self.host='127.0.0.1' + self.user = 'root' + self.passwd = 'taosdata' + self.conf = '/etc/taos/taos.cfg' + self.tz = 'Asia/Shanghai' + + def prepare_data(self) -> dict: + json_data = { + "filetype": "insert", + "cfgdir": "/etc/taos/cfg", + "host": "127.0.0.1", + "port": 6030, + "rest_port": 6041, + "user": "root", + "password": "taosdata", + "thread_count": 20, + "create_table_thread_count": 40, + "result_file": "/tmp/taosBenchmark_result.log", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 10000, + "max_sql_len": 1024000, + "databases": [ + { + "dbinfo": { + "name": "stream_test", + "drop": "yes", + "replica": 1, + "duration": 10, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "vgroups": 10, + "stt_trigger": 1, + "WAL_RETENTION_PERIOD": 86400 + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "yes", + "childtable_count": 500, + "childtable_prefix": "ctb0_", + "escape_character": "no", + "auto_create_table": "yes", + "batch_create_tbl_num": 1000, + "data_source": "rand", + "insert_mode": "taosc", + "interlace_rows": 400, + "tcp_transfer": "no", + "insert_rows": 10000, + "partial_col_num": 0, + "childtable_limit": 0, + "childtable_offset": 0, + "rows_per_tbl": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "keep_trying": -1, + "timestamp_step": 1000, + "trying_interval": 10, + "start_timestamp": "2021-01-01 00:00:00", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + { + "type": "INT", + "count": 1 + }, + { + "type": "TINYINT", + "count": 0 + }, + { + "type": "DOUBLE", + "count": 0 + }, + { + "type": "VARCHAR", + "count": 0, + "len": 16 + }, + { + "type": "NCHAR", + "count": 0, + "len": 4 + }, + { + "type": "SMALLINT", + "count": 0 + }, + { + "type": "BIGINT", + "count": 0 + }, + { + "type": "UTINYINT", + "count": 0 + }, + { + "type": "USMALLINT", + "count": 0 + }, + { + "type": "UINT", + "count": 0 + }, + { + "type": "UBIGINT", + "count": 0 + }, + { + "type": "FLOAT", + "count": 2 + }, + { + "type": "BINARY", + "count": 0, + "len": 8 + }, + { + "type": "BOOL", + "count": 0 + }, + { + "type": "TIMESTAMP", + "count": 1 + } + ], + "tags": [ + { + "type": "INT", + "count": 0 + }, + { + "type": "TINYINT", + "count": 1 + }, + { + "type": "DOUBLE", + "count": 0 + }, + { + "type": "VARCHAR", + "count": 0, + "len": 8 + }, + { + "type": "NCHAR", + "count": 0, + "len": 16 + }, + { + "type": "SMALLINT", + "count": 0 + }, + { + "type": "BIGINT", + "count": 0 + }, + { + "type": "UTINYINT", + "count": 0 + }, + { + "type": "USMALLINT", + "count": 0 + }, + { + "type": "UINT", + "count": 0 + }, + { + "type": "UBIGINT", + "count": 0 + }, + { + "type": "FLOAT", + "count": 0 + }, + { + "type": "BINARY", + "count": 1, + "len": 16 + }, + { + "type": "BOOL", + "count": 0 + }, + { + "type": "TIMESTAMP", + "count": 0 + } + ] + } + ] + } + ], + "prepare_rand": 10000, + "chinese": "no", + "test_log": "/tmp/testlog/" + } + + with open('/tmp/stream.json', 'w+') as f: + json.dump(json_data, f, indent=4) + + def do_start(self): + self.prepare_data() + + try: + subprocess.Popen('taosBenchmark --f /tmp/stream.json', stdout=subprocess.PIPE, shell=True, text=True) + except subprocess.CalledProcessError as e: + print(f"Error running Bash command: {e}") + + conn = taos.connect( + host=self.host, user=self.user, password=self.passwd, config=self.conf, timezone=self.tz + ) + + time.sleep(10) + + print("start to connect db") + cursor = conn.cursor() + + cursor.execute('use stream_test') + + sql = "create stream str1 ignore update 0 into str1_dst as select _wstart as wstart, min(c1),max(c2), count(c3) from stream_test.stb partition by cast(t1 as int) t1,tbname interval(5s)" + cursor.execute(sql) + + print("create stream completed, start to monitor system load") + conn.close() + + loader = MonitorSystemLoad('taosd', 80) + loader.get_proc_status() + + +if __name__ == "__main__": + StreamStarter().do_start() diff --git a/tests/script/tsim/analytics/basic0.sim b/tests/script/tsim/analytics/basic0.sim index 77c9184e8f..3ac49b1fc3 100644 --- a/tests/script/tsim/analytics/basic0.sim +++ b/tests/script/tsim/analytics/basic0.sim @@ -3,7 +3,17 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print =============== create anode +print =============== failed to create anode on '127.0.0.1:1101' +sql_error create anode '127.0.0.1:1101' + +sql show anodes +if $rows != 0 then + return -1 +endi + +sql_error drop anode 1 + +print ================ create anode sql create anode '192.168.1.116:6050' sql show anodes @@ -30,7 +40,7 @@ print $data00 $data01 $data02 sql use d0 print =============== create super table, include column type for count/sum/min/max/first -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double, c4 tinyint, c5 bigint, c6 varchar(12)) tags (t1 int unsigned) sql show stables if $rows != 1 then @@ -42,10 +52,11 @@ sql create table ct1 using stb tags(1000) print ==================== insert data # input_list = [5, 14, 15, 15, 14, 19, 17, 16, 20, 22, 8, 21, 28, 11, 9, 29, 40] -sql insert into ct1(ts, c1) values(now-1a, 5)(now+1a, 14)(now+2a, 15)(now+3a, 15)(now+4a, 14) -sql insert into ct1(ts, c1) values(now+5a, 19)(now+6a, 17)(now+7a, 16)(now+8a, 20)(now+9a, 22) -sql insert into ct1(ts, c1) values(now+10a, 8)(now+11a, 21)(now+12a, 28)(now+13a, 11)(now+14a, 9) -sql insert into ct1(ts, c1) values(now+15a, 29)(now+16a, 40) +sql insert into ct1(ts, c1, c2, c3, c4, c5, c6) values(now-1a, 5, 5, 5, 5, 5, 'a')(now+1a, 14, 14, 14, 14, 14, 'a')(now+2a, 15, 15, 15, 15, 15, 'a') +sql insert into ct1 values(now+3a, 15, 15, 15, 15, 15, 'a')(now+4a, 14, 14, 14, 14, 14, 'a')(now+5a, 19, 19, 19, 19, 19, 'a')(now+6a, 17, 17, 17, 17, 17, 'a') +sql insert into ct1 values(now+7a, 16, 16, 16, 16, 16, 'a')(now+8a, 20, 20, 20, 20, 20, 'a')(now+9a, 22, 22, 22, 22, 22, 'a') +sql insert into ct1 values(now+10a, 8, 8, 8, 8, 8, 'a')(now+11a, 21, 21, 21, 21, 21, 'a')(now+12a, 28, 28, 28, 28, 28, 'a')(now+13a, 11, 11, 11, 11, 11, 'a')(now+14a, 9, 9, 9, 9, 9, 'a') +sql insert into ct1 values(now+15a, 29, 29, 29, 29, 29, 'a')(now+16a, 40, 40, 40, 40, 40, 'a') sql select count(*) from ct1 if $data00 != 17 then @@ -58,6 +69,87 @@ if $data00 != 1 then return -1 endi +print ================= try every loaded anomaly detection algorithm +sql select count(*) from ct1 anomaly_window(c1, 'algo=iqr'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=lof'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=shesd'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=grubbs'); + +print ================= try every column type of column +sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c2, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c3, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c4, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c5, 'algo=ksigma,k=2'); + +print =================== invalid column type +sql_error select count(*) from ct1 anomaly_window(c6, 'algo=ksigma,k=2'); +sql_error select forecast(c6, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 + + +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1 +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=0') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=-10') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, every=0') from ct1 + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters, conf=50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, ' algo=holtwinters , conf=50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, ' algo = holtwinters , conf = 50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ,') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, , ,') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, a =') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, = a ,') from ct1 + +print =================== valid column type +sql select forecast(c1, 'conf=50 ,algo = arima') from ct1 +sql select forecast(c1, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c2, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c3, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c4, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=2') from ct1 +if $rows != 10 then + return -1 +endi + +if $data03 != 28 then + return -1 +endi + +if $data00 != @23-11-15 06:13:20.000@ then + print expect 23-11-15 06:13:20.000 , actual $data00 + return -1 +endi + +if $data10 != @23-11-15 06:13:20.002@ then + print expect 23-11-15 06:13:20.002 , actual $data10 + return -1 +endi + +if $data20 != @23-11-15 06:13:20.004@ then + return -1 +endi + +print test the every option and rows option + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=100,rows=5') from ct1 +if $rows != 5 then + return -1 +endi + +if $data00 != @23-11-15 06:13:20.000@ then + return -1 +endi + +if $data10 != @23-11-15 06:13:20.100@ then + return -1 +endi sql drop anode 1 sql show anodes @@ -66,6 +158,13 @@ if $rows != 0 then return -1 endi +sleep 1000 + +print ===================== query without anodes +sql_error select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 +sql_error select count(*) from ct1 anomaly_window(c1, 'algo=iqr'); + + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check diff --git a/tests/script/tsim/stream/basic5.sim b/tests/script/tsim/stream/basic5.sim index d7dd603d3c..866fbd3ebe 100644 --- a/tests/script/tsim/stream/basic5.sim +++ b/tests/script/tsim/stream/basic5.sim @@ -2,13 +2,28 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c debugflag -v 135 system sh/cfg.sh -n dnode1 -c streamBufferSize -v 10 +system sh/cfg.sh -n dnode1 -c checkpointinterval -v 60 +system sh/cfg.sh -n dnode1 -c snodeAddress -v 127.0.0.1:873 system sh/exec.sh -n dnode1 -s start sleep 500 sql connect +print step1 ============= -print step1============= +print ================ create snode +sql show snodes +if $rows != 0 then + return -1 +endi + +sql create snode on dnode 1; +sql show snodes; +if $rows != 1 then + return -1 +endi + +print ============== snode created , create db sql create database test3 vgroups 1; sql use test3; @@ -57,7 +72,7 @@ loop8: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -87,7 +102,7 @@ loop9: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -127,7 +142,7 @@ loop10: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -162,7 +177,7 @@ loop11: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -194,7 +209,7 @@ loop11: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -239,7 +254,7 @@ loop12: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -315,7 +330,7 @@ loop13: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -369,7 +384,6 @@ if $data24 != 1 then endi print step4============= - sql create database test6 vgroups 4; sql use test6; sql create stable st(ts timestamp,a int,b int,c int,d int) tags(ta int,tb int,tc int); @@ -396,7 +410,7 @@ loop14: sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 30 then return -1 endi @@ -449,4 +463,21 @@ if $data25 != 2 then goto loop14 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print sleep for 1min for checkpoint generate +sleep 60000 + +print ================== restart to load checkpoint from snode + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +sleep 500 +sql connect + +sleep 30000 + +sql select start_ver, checkpoint_ver from information_schema.ins_stream_tasks where level='source'; +sleep 500 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT + diff --git a/tests/script/tsim/stream/tag.sim b/tests/script/tsim/stream/tag.sim new file mode 100644 index 0000000000..f293f4ac05 --- /dev/null +++ b/tests/script/tsim/stream/tag.sim @@ -0,0 +1,110 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect + +print step1 +print =============== create database +sql create database test vgroups 2; +sql use test; + +sql create table st1(ts timestamp, a int, b int , c int, d double) tags(x int); +sql create table t1 using st1 tags(1); +sql create table t2 using st1 tags(2); +sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1 from st1 where x>=2 interval(60s) ; + +run tsim/stream/checkTaskStatus.sim + + +sql insert into t2 values(1648791213000,0,1,1,1.0); +sql insert into t2 values(1648791213001,9,2,2,1.1); +sql insert into t2 values(1648791213009,0,3,3,1.0); + +sql insert into t1 values(1648791223000,0,1,1,1.0); +sql insert into t1 values(1648791223001,9,2,2,1.1); +sql insert into t1 values(1648791223009,0,3,3,1.0); + +sleep 300 + +sql select * from streamt; +if $data01 != 3 then + return -1 +endi + +sql alter table t1 set tag x=3; + +sql insert into t1 values(1648791233000,0,1,1,1.0); +sql insert into t1 values(1648791233001,9,2,2,1.1); +sql insert into t1 values(1648791233009,0,3,3,1.0); + +sleep 1000 +sql select * from streamt; + +if $data01 != 6 then + return -1 +endi + +sql alter table t1 set tag x=1; +sql alter table t2 set tag x=1; + +sql insert into t1 values(1648791243000,0,1,1,1.0); +sql insert into t1 values(1648791243001,9,2,2,1.1); + + +sql select * from streamt; +if $data01 != 6 then + return -1 +endi + +#$loop_count = 0 +#loop2: +# +#sleep 300 +#print 1 sql select * from streamt; +#sql select * from streamt; +# +#print $data00 $data01 $data02 $data03 +#print $data10 $data11 $data12 $data13 +# +#$loop_count = $loop_count + 1 +#if $loop_count == 10 then +# return -1 +#endi +# +## row 0 +#if $data01 != 3 then +# print ======data01=$data01 +# goto loop2 +#endi +# +#if $data02 != 6 then +# print ======data02=$data02 +# goto loop2 +#endi +# +#if $data03 != 3 then +# print ======data03=$data03 +# goto loop2 +#endi +# +## row 1 +#if $data11 != 3 then +# print ======data11=$data11 +# goto loop2 +#endi +# +#if $data12 != 6 then +# print ======data12=$data12 +# goto loop2 +#endi +# +#if $data13 != 3 then +# print ======data13=$data13 +# goto loop2 +#endi +# + +print tag end +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index ec52b8c234..67b84c24df 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -111,6 +111,10 @@ run tsim/stream/distributeInterval0.sim run tsim/stream/distributeSession0.sim run tsim/stream/state0.sim run tsim/stream/basic2.sim +run tsim/stream/basic3.sim +run tsim/stream/basic4.sim +run tsim/stream/basic5.sim +run tsim/stream/tag.sim run tsim/stream/concurrentcheckpt.sim run tsim/insert/basic1.sim run tsim/insert/commit-merge0.sim diff --git a/tests/system-test/0-others/compact_auto.py b/tests/system-test/0-others/compact_auto.py new file mode 100644 index 0000000000..813be3e9f0 --- /dev/null +++ b/tests/system-test/0-others/compact_auto.py @@ -0,0 +1,170 @@ +import taos +import sys +import time +import socket +import os +import threading +import psutil +import platform +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.default_compact_options = [ "0d", "0d,0d", "0h"] + self.compact_options = [["db00", "0m", "-0d,0", "0", "0d", "0d,0d", "0h"], + ["db01", "0m", "-2d,-1", "0", "0d", "-2d,-1d", "0h"], + ["db02", "2880m", "-61d,-1", "0", "2d", "-61d,-1d", "0h"], + ["db03", "48h", "-87840m,-60", "1h", "2d", "-61d,-60d", "1h"], + ["db04", "2d", "-87840m,-1440h", "12", "2d", "-61d,-60d", "12h"], + ["db05", "2", "-61,-1440h", "23h", "2d", "-61d,-60d", "23h"], + ] + + def create_db_compact(self): + tdLog.info("create db compact options") + for item in self.compact_options: + tdSql.execute(f'create database {item[0]} compact_interval {item[1]} compact_time_range {item[2]} compact_time_offset {item[3]} duration 1d') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + tdSql.query(f'show create database {item[0]}') + tdSql.checkEqual(tdSql.queryResult[0][0], item[0]) + tdSql.checkEqual(True, f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}' in tdSql.queryResult[0][1]) + tdSql.execute(f'drop database {item[0]}') + + def checkShowCreateWithTimeout(self, db, expectResult, timeout=30): + result = False + for i in range(timeout): + tdSql.query(f'show create database `%s`' %(db)) + tdSql.checkEqual(tdSql.queryResult[0][0], db) + if expectResult in tdSql.queryResult[0][1]: + result = True + break + time.sleep(1) + if result == False: + raise Exception(f"Unexpected result of 'show create database `{db}`':{tdSql.queryResult[0][1]}, expect:{expectResult}") + + def alter_db_compact(self): + tdLog.info("alter db compact options together") + for item in self.compact_options: + tdSql.execute(f'create database {item[0]} duration 1d') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], self.default_compact_options[0]) + tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[1]) + tdSql.checkEqual(tdSql.queryResult[0][36], self.default_compact_options[2]) + tdSql.query(f'show create database {item[0]}') + tdSql.checkEqual(tdSql.queryResult[0][0], item[0]) + tdSql.checkEqual(True, f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {self.default_compact_options[2]}' in tdSql.queryResult[0][1]) + tdSql.execute(f'alter database {item[0]} compact_interval {item[1]} compact_time_range {item[2]} compact_time_offset {item[3]}') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + for item in self.compact_options: + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}') + tdSql.execute(f'drop database {item[0]}') + + tdLog.info("alter db compact options separately") + compact_separate_options = [["db100", "0m", "-0d,0", "0", "0d", "0d,0d", "0h"], + ["db101", "10m", "-2d,-1", "1", "10m", "-2d,-1d", "1h"]] + for item in compact_separate_options: + tdSql.execute(f'create database {item[0]} duration 1d') + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], self.default_compact_options[0]) + tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[1]) + tdSql.checkEqual(tdSql.queryResult[0][36], self.default_compact_options[2]) + tdSql.query(f'show create database {item[0]}') + tdSql.checkEqual(tdSql.queryResult[0][0], item[0]) + tdSql.checkEqual(True, f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {self.default_compact_options[2]}' in tdSql.queryResult[0][1]) + tdSql.execute(f'alter database {item[0]} compact_time_offset {item[3]}', queryTimes=10) + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], self.default_compact_options[0]) + tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[1]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {item[6]}') + tdSql.execute(f'alter database {item[0]} compact_interval {item[1]}', queryTimes=10) + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[1]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {item[6]}') + tdSql.execute(f'alter database {item[0]} compact_time_range {item[2]}', queryTimes=10) + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}') + tdSql.execute(f'alter database {item[0]} compact_time_offset {item[3]}', queryTimes=10) + tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"') + tdSql.checkEqual(tdSql.queryResult[0][34], item[4]) + tdSql.checkEqual(tdSql.queryResult[0][35], item[5]) + tdSql.checkEqual(tdSql.queryResult[0][36], item[6]) + self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}') + for item in compact_separate_options: + tdSql.execute(f'drop database {item[0]}', queryTimes=10) + + def compact_error(self): + compact_err_list = [["compact_time_range 86400m,61d", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range 60,61", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range 60d,61d", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -60,-60", "Invalid option compact_time_range: -86400m,-86400m, start time should be less than end time"], + ["compact_time_range -60,-1440h", "Invalid option compact_time_range: -86400m,-86400m, start time should be less than end time"], + ["compact_time_range -60d,-61d", "Invalid option compact_time_range: -86400m,-87840m, start time should be less than end time"], + ["compact_time_range -5256001m,-1", "Invalid option compact_time_range: -5256001m, start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -199999999999m,-199999999998m", "start time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -99999999999999999999m,-99999999999999999998m", "Invalid value type: -99999999999999999999"], + ["compact_time_range -60d,-1", "Invalid option compact_time_range: -1440m, end time should be in range: [-5256000m, -14400m]"], + ["compact_interval 24h compact_time_range -60,61", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"], + ["compact_interval 100 compact_time_range -60d,61d", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -60d,87840m", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"], + ["compact_time_range -100,-90s", "Invalid option compact_time_range end unit: s, only m, h, d allowed"], + ["compact_interval 10m compact_time_range -120d,-14400m compact_time_offset -1", "syntax error near"], + ["compact_time_range -100,-99d compact_interval -1", "syntax error near"], + ["compact_time_range 0", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100,-90,-80", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100;-90", "Invalid option compact_time_range, should have 2 value"], + ["compact_time_range -100:-90", "syntax error near"], + ["compact_time_range -100 -90", "syntax error near"], + ["compact_interval 1m", "Invalid option compact_interval: 1, valid range: [10, 5256000]"], + ["compact_interval 199999999999m", "valid range: [10, 5256000]"], + ["compact_interval 9999999999999m", "Invalid option compact_interval: 9999999999999, valid range: [10, 5256000]"], + ["compact_interval 5256001m", "Invalid option compact_interval: 5256001, valid range: [10, 5256000]"], + ["compact_interval 3651", "Invalid option compact_interval: 5257440, valid range: [10, 5256000]"], + ["compact_interval 86400s", "Invalid option compact_interval unit: s, only m, h, d allowed"], + ["compact_interval -1", "syntax error near"], + ["compact_time_offset -1", "syntax error near"], + ["compact_time_offset 3600s", "Invalid option compact_time_offset unit: s, only h allowed"], + ["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], + ["compact_time_offset 24", "Invalid option compact_time_offset: 24, valid range: [0, 23]"], + ["compact_time_offset 24h", "Invalid option compact_time_offset: 24, valid range: [0, 23]"], + ["compact_time_offset 9999999999999", "valid range: [0, 23]"], + ["compact_time_offset 199999999999", "valid range: [0, 23]"], + ["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], + ["compact_interval 10m compact_time_range -120d,-60 compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"], + ] + tdSql.execute('create database if not exists db') + for item in compact_err_list: + tdSql.error(f"create database db {item[0]}", expectErrInfo=item[1], fullMatched=False) + tdSql.error(f"alter database db {item[0]}", expectErrInfo=item[1], fullMatched=False) + tdSql.execute('drop database db') + + def run(self): + self.create_db_compact() + self.alter_db_compact() + self.compact_error() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index af0dd6d949..d4a1aa938d 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(303, 304)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(306, 307)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(60, len(tdSql.queryResult)) diff --git a/tests/system-test/0-others/ins_filesets.py b/tests/system-test/0-others/ins_filesets.py new file mode 100644 index 0000000000..e5bc4b61b7 --- /dev/null +++ b/tests/system-test/0-others/ins_filesets.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.execute('create database db vgroups 1') + tdSql.execute('use db') + tdSql.execute('create table t1 (ts timestamp, a int, b int)') + tdSql.execute('insert into t1 values(\'2024-12-27 14:00:00\', 1, 2)') + tdSql.execute('flush database db') + + tdLog.sleep(5) + + rows = tdSql.query('select * from information_schema.ins_filesets') + tdSql.checkRows(1) + tdSql.checkEqual(tdSql.getData(0, 0), 'db') + tdSql.checkEqual(tdSql.getData(0, 1), 2) + tdSql.checkEqual(tdSql.getData(0, 2), 2008) + # tdSql.CheckEqual(str(tdSql.getData(0, 3)), '2024-12-23 08:00:00.000') + # tdSql.CheckEqual(str(tdSql.getData(0, 4)), '2025-01-02 07:59:59.999') + # tdSql.CheckEqual(tdSql.getData(0, 6), '1970-01-01 08:00:00.000') + # tdSql.CheckEqual(tdSql.getData(0, 7), False) + + tdDnodes.stopAll() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index f58bb7517e..17a0664926 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -81,8 +81,8 @@ class TDTestCase: tdSql.query('select * from information_schema.ins_databases where name = "db"') db_options_items = ["replica","keep","buffer","pages","minrows","cachemodel","cachesize","wal_level","wal_fsync_period", - "wal_retention_period","wal_retention_size","stt_trigger"] - db_options_result_idx = [4,7,8,10,11,18,19,20,21,22,23,24] + "wal_retention_period","wal_retention_size","stt_trigger", "compact_interval", "compact_time_range", "compact_time_offset"] + db_options_result_idx = [4,7,8,10,11,18,19,20,21,22,23,24,34,35,36] self.option_result = [] for idx in db_options_result_idx: diff --git a/tests/system-test/1-insert/insert_from_csv.py b/tests/system-test/1-insert/insert_from_csv.py new file mode 100644 index 0000000000..38f6a8b09d --- /dev/null +++ b/tests/system-test/1-insert/insert_from_csv.py @@ -0,0 +1,47 @@ +import taos +import sys +import datetime +import inspect +import random +from util.dnodes import TDDnode +from util.dnodes import tdDnodes + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + self.testcasePath = os.path.split(__file__)[0] + self.testcasePath = self.testcasePath.replace('\\', '//') + self.database = "test_insert_csv_db" + self.table = "test_insert_csv_tbl" + + def insert_from_csv(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") + tdSql.execute(f"use {self.database}") + tdSql.execute(f"create table {self.table} (ts timestamp, c1 nchar(16), c2 double, c3 int)") + tdSql.execute(f"insert into {self.table} file '{self.testcasePath}//test_insert_from_csv.csv'") + tdSql.query(f"select count(*) from {self.table}") + tdSql.checkData(0, 0, 5) + + def run(self): + tdSql.prepare() + + startTime_all = time.time() + self.insert_from_csv() + endTime_all = time.time() + print("total time %ds" % (endTime_all - startTime_all)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/test_insert_from_csv.csv b/tests/system-test/1-insert/test_insert_from_csv.csv new file mode 100644 index 0000000000..966af8c27a --- /dev/null +++ b/tests/system-test/1-insert/test_insert_from_csv.csv @@ -0,0 +1,5 @@ +'2024-12-13 09:30:00.050','ABCDEF68',24.774736842805263,200 +'2024-12-13 09:30:00.060','ABCDEF68',24.774736842805263,201 +'2024-12-13 09:30:00.080','ABCDEF68',24.774736842805263,202 +'2024-12-13 09:30:00.100','ABCDEF68',24.774736842805263,203 +'2024-12-13 09:30:00.110','ABCDEF68',24.774736842805263,204 diff --git a/tests/system-test/2-query/elapsed.py b/tests/system-test/2-query/elapsed.py index dc89cd513d..c1191c3441 100644 --- a/tests/system-test/2-query/elapsed.py +++ b/tests/system-test/2-query/elapsed.py @@ -1542,7 +1542,7 @@ class TDTestCase: def query_precision(self): def generate_data(precision="ms"): - tdSql.execute("create database if not exists db_%s precision '%s';" %(precision, precision)) + tdSql.execute("create database if not exists db_%s keep 36500 precision '%s';" %(precision, precision)) tdSql.execute("use db_%s;" %precision) tdSql.execute("create stable db_%s.st (ts timestamp , id int) tags(ind int);"%precision) tdSql.execute("create table db_%s.tb1 using st tags(1);"%precision) diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index 006215956b..f5dba2df4f 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -395,7 +395,7 @@ class TDTestCase: def query_precision(self): def generate_data(precision="ms"): dbname = f"db_{precision}" - tdSql.execute(f"create database if not exists db_%s precision '%s';" %(precision, precision)) + tdSql.execute(f"create database if not exists db_%s keep 36500 precision '%s';" %(precision, precision)) tdSql.execute("use db_%s;" %precision) tdSql.execute(f"create stable db_%s.st (ts timestamp , id int) tags(ind int);"%precision) tdSql.execute(f"create table db_%s.tb1 using {dbname}.st tags(1);"%precision) @@ -449,7 +449,7 @@ class TDTestCase: def check_boundary_values(self, dbname="bound_test"): tdSql.execute(f"drop database if exists {dbname}") - tdSql.execute(f"create database if not exists {dbname}") + tdSql.execute(f"create database if not exists {dbname} keep 36500") tdSql.execute( f"create table {dbname}.stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" ) diff --git a/tests/system-test/simpletest.bat b/tests/system-test/simpletest.bat index a1f7273ad4..5948c7fc80 100644 --- a/tests/system-test/simpletest.bat +++ b/tests/system-test/simpletest.bat @@ -13,6 +13,7 @@ python3 .\test.py -f 0-others\cachemodel.py @REM python3 .\test.py -f 0-others\fsync.py python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py +python3 .\test.py -f 1-insert\insert_from_csv.py @REM python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py @REM python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py @REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index d058d7a52f..1ee2bc4ce6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -251,7 +251,7 @@ IF(TD_BUILD_KEEPER) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND go build -a -ldflags "-X 'github.com/taosdata/taoskeeper/version.Version=${taos_version}' -X 'github.com/taosdata/taoskeeper/version.CommitID=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" + COMMAND go build -a -ldflags "-X 'github.com/taosdata/taoskeeper/version.Version=${taos_version}' -X 'github.com/taosdata/taoskeeper/version.Gitinfo=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.CommitID=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND COMMAND cmake -E echo "Comparessing taoskeeper.exe" COMMAND cmake -E time upx taoskeeper.exe @@ -278,7 +278,7 @@ IF(TD_BUILD_KEEPER) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND go build -a -ldflags "-X 'github.com/taosdata/taoskeeper/version.Version=${taos_version}' -X 'github.com/taosdata/taoskeeper/version.CommitID=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" + COMMAND go build -a -ldflags "-X 'github.com/taosdata/taoskeeper/version.Version=${taos_version}' -X 'github.com/taosdata/taoskeeper/version.Gitinfo=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.CommitID=${taoskeeper_commit_sha1}' -X 'github.com/taosdata/taoskeeper/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND COMMAND cmake -E echo "Copy taoskeeper" COMMAND cmake -E copy taoskeeper ${CMAKE_BINARY_DIR}/build/bin