diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 0f65684027..2179200bb3 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -6,6 +6,7 @@ on: - 'main' - '3.0' - '3.1' + - '3.3.6' - 'enh/cmake-TD-33848' paths-ignore: diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 36576eb187..31eda597ad 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -9,17 +9,12 @@ on: paths-ignore: - 'packaging/**' - 'docs/**' - repository_dispatch: - types: [run-tests] concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.event.client_payload.ref}}-${{ github.event_name == 'repository_dispatch' && 'dispatch' || ''}} + group: ${{ github.workflow }}-${{ github.ref }}-TDengine cancel-in-progress: true env: - CONTAINER_NAME: 'taosd-test' - WKDIR: '/var/lib/jenkins/workspace' - WK: '/var/lib/jenkins/workspace/TDinternal' WKC: '/var/lib/jenkins/workspace/TDinternal/community' jobs: @@ -28,430 +23,71 @@ jobs: group: CI labels: [self-hosted, Linux, X64, testing] outputs: - tdinternal: ${{ steps.parameters.outputs.tdinternal }} run_function_test: ${{ steps.parameters.outputs.run_function_test }} run_tdgpt_test: ${{ steps.parameters.outputs.run_tdgpt_test }} - source_branch: ${{ steps.parameters.outputs.source_branch }} - target_branch: ${{ steps.parameters.outputs.target_branch }} - pr_number: ${{ steps.parameters.outputs.pr_number }} steps: - name: Determine trigger source and fetch parameters id: parameters run: | set -euo pipefail - # check the trigger source and get branch information - if [ "${{ github.event_name }}" == "repository_dispatch" ]; then - tdinternal="true" - source_branch=${{ github.event.client_payload.tdinternal_source_branch }} - target_branch=${{ github.event.client_payload.tdinternal_target_branch }} - pr_number=${{ github.event.client_payload.tdinternal_pr_number }} + target_branch=${{ github.event.pull_request.base.ref }} + + # Check whether to run tdgpt test cases + cd ${{ env.WKC }} + changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) + echo "changed files exclude doc: ${changed_files_non_doc}" + + if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then run_tdgpt_test="true" + else + run_tdgpt_test="false" + fi + echo "run tdgpt test: ${run_tdgpt_test}" + + # Check whether to run function test cases + changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ + grep -v "^docs/en/" | \ + grep -v "^docs/zh/" | \ + grep -v ".md$" | \ + grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ + tr '\n' ' ' || :) + echo "changed files exclude tdgpt: ${changed_files_non_tdgpt}" + + if [ -n "$changed_files_non_tdgpt" ]; then run_function_test="true" else - tdinternal="false" - source_branch=${{ github.event.pull_request.head.ref }} - target_branch=${{ github.event.pull_request.base.ref }} - pr_number=${{ github.event.pull_request.number }} - - # check whether to run tdgpt test cases - cd ${{ env.WKC }} - changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD $target_branch`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :) - - if [[ "$changed_files_non_doc" != '' && "$changed_files_non_doc" =~ /forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics|tdgpt/ ]]; then - run_tdgpt_test="true" - else - run_tdgpt_test="false" - fi - - # check whether to run function test cases - changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD $target_branch`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -Ev "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics|tdgpt" | tr '\n' ' ' ||:) - if [ "$changed_files_non_tdgpt" != '' ]; then - run_function_test="true" - else - run_function_test="false" - fi + run_function_test="false" fi - echo "tdinternal=$tdinternal" >> $GITHUB_OUTPUT + echo "run function test: ${run_function_test}" + + # Output the results for GitHub Actions echo "run_function_test=$run_function_test" >> $GITHUB_OUTPUT echo "run_tdgpt_test=$run_tdgpt_test" >> $GITHUB_OUTPUT - echo "source_branch=$source_branch" >> $GITHUB_OUTPUT - echo "target_branch=$target_branch" >> $GITHUB_OUTPUT - echo "pr_number=$pr_number" >> $GITHUB_OUTPUT + + echo ${{ github.event.pull_request.head.ref }} + echo ${{ github.event.pull_request.base.ref }} + echo ${{ github.event.pull_request.number }} run-tests-on-linux: + uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@main needs: fetch-parameters - runs-on: - group: CI - labels: [self-hosted, Linux, X64, testing] - timeout-minutes: 200 - env: - IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }} - RUN_RUNCTION_TEST: ${{ needs.fetch-parameters.outputs.run_function_test }} - RUN_TDGPT_TEST: ${{ needs.fetch-parameters.outputs.run_tdgpt_test }} - SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }} - TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }} - PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }} - steps: - - name: Output the environment information - run: | - echo "::group::Environment Info" - date - hostname - env - echo "Runner: ${{ runner.name }}" - echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}" - echo "Workspace: ${{ env.WKDIR }}" - git --version - echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}" - echo "::endgroup::" - - - name: Prepare repositories - run: | - set -euo pipefail - prepare_environment() { - cd "$1" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "$2" - } - prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}" - prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}" - - - name: Get latest codes and logs for TDinternal PR - if: ${{ env.IS_TDINTERNAL == 'true' }} - run: | - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Get latest codes and logs for TDengine PR - if: ${{ env.IS_TDINTERNAL == 'false' }} - run: | - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Update submodule - run: | - cd ${{ env.WKC }} - git submodule update --init --recursive - - name: Output the 'file_no_doc_changed' information to the file - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - mkdir -p ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }} - changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ env.TARGET_BRANCH }}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :) - echo $changed_files_non_doc > ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt - - name: Check assert testing - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - ./run_check_assert_container.sh -d ${{ env.WKDIR }} - - name: Check void function testing - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - ./run_check_void_container.sh -d ${{ env.WKDIR }} - - name: Build docker container - if: ${{ env.RUN_RUNCTION_TEST == 'true' }} - run: | - date - rm -rf ${{ env.WKC }}/debug - cd ${{ env.WKC }}/tests/parallel_test - time ./container_build.sh -w ${{ env.WKDIR }} -e - - name: Get parameters for testing - id: get_param - run: | - log_server_file="/home/log_server.json" - timeout_cmd="" - extra_param="" - - if [ -f "$log_server_file" ]; then - log_server_enabled=$(jq '.enabled' "$log_server_file") - timeout_param=$(jq '.timeout' "$log_server_file") - if [ "$timeout_param" != "null" ] && [ "$timeout_param" != "0" ]; then - timeout_cmd="timeout $timeout_param" - fi - - if [ "$log_server_enabled" == "1" ]; then - log_server=$(jq '.server' "$log_server_file" | sed 's/\\\"//g') - if [ "$log_server" != "null" ] && [ "$log_server" != "" ]; then - extra_param="-w $log_server" - fi - fi - fi - echo "timeout_cmd=$timeout_cmd" >> $GITHUB_OUTPUT - echo "extra_param=$extra_param" >> $GITHUB_OUTPUT - - name: Run function returns with a null pointer scan testing - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - ./run_scan_container.sh -d ${{ env.WKDIR }} -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -f ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt ${{ steps.get_param.outputs.extra_param }} - - name: Run tdgpt test cases - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' && env.RUN_TDGPT_TEST == 'true' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - export DEFAULT_RETRY_TIME=2 - date - timeout 600 time ./run.sh -e -m /home/m.json -t tdgpt_cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 300 ${{ steps.get_param.outputs.extra_param }} - - name: Run function test cases - if: ${{ env.RUN_RUNCTION_TEST == 'true'}} - run: | - cd ${{ env.WKC }}/tests/parallel_test - export DEFAULT_RETRY_TIME=2 - date - ${{ steps.get_param.outputs.timeout_cmd }} time ./run.sh -e -m /home/m.json -t cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 1200 ${{ steps.get_param.outputs.extra_param }} + if: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || needs.fetch-parameters.outputs.run_function_test == 'true' }} + with: + tdinternal: false + run_function_test: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} + run_tdgpt_test: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' }} run-tests-on-mac: + uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml@main needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} - runs-on: - group: CI - labels: [self-hosted, macOS, testing] - timeout-minutes: 60 - env: - IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }} - SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }} - TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }} - PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }} - steps: - - name: Output the environment information - run: | - echo "::group::Environment Info" - date - hostname - env - echo "Runner: ${{ runner.name }}" - echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}" - echo "Workspace: ${{ env.WKDIR }}" - git --version - echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}" - echo "::endgroup::" - - name: Prepare repositories - run: | - set -euo pipefail - prepare_environment() { - cd "$1" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "$2" - } - prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}" - prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}" - - name: Get latest codes and logs for TDinternal PR - if: ${{ env.IS_TDINTERNAL == 'true' }} - run: | - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Get latest codes and logs for TDengine PR - if: ${{ env.IS_TDINTERNAL == 'false' }} - run: | - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Update submodule - run: | - cd ${{ env.WKC }} - git submodule update --init --recursive - - name: Run tests - run: | - date - cd ${{ env.WK }} - rm -rf debug - mkdir debug - cd ${{ env.WK }}/debug - echo $PATH - echo "PATH=/opt/homebrew/bin:$PATH" >> $GITHUB_ENV - cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false -DCMAKE_BUILD_TYPE=Release - make -j10 - ctest -j10 || exit 7 - date + with: + tdinternal: false run-tests-on-windows: + uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml@main needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} - runs-on: - group: CI - labels: [self-hosted, Windows, X64, testing] - timeout-minutes: 126 - env: - IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }} - SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }} - TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }} - PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }} - WIN_INTERNAL_ROOT: "C:\\workspace\\0\\TDinternal" - WIN_COMMUNITY_ROOT: "C:\\workspace\\0\\TDinternal\\community" - WIN_SYSTEM_TEST_ROOT: "C:\\workspace\\0\\TDinternal\\community\\tests\\system-test" - WIN_VS_PATH: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" - WIN_CPU_TYPE: "x64" - steps: - - name: Output the environment information - run: | - hostname - taskkill /f /t /im python.exe - taskkill /f /t /im bash.exe - taskkill /f /t /im taosd.exe - ipconfig - set - date /t - time /t - rd /s /Q "%WIN_INTERNAL_ROOT%\debug" || exit 0 - shell: cmd - - name: Prepare repositories - run: | - :: Prepare internal repository - if exist "%WIN_INTERNAL_ROOT%" ( - cd /d "%WIN_INTERNAL_ROOT%" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "%TARGET_BRANCH%" - ) else ( - echo Directory does not exist: "%WIN_INTERNAL_ROOT%" - exit 1 - ) - - :: Prepare community repository - if exist "%WIN_COMMUNITY_ROOT%" ( - cd /d "%WIN_COMMUNITY_ROOT%" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "%TARGET_BRANCH%" - ) else ( - echo Directory does not exist: "%WIN_COMMUNITY_ROOT%" - exit 1 - ) - shell: cmd - - name: Get latest codes and logs for TDinternal PR - if: ${{ env.IS_TDINTERNAL == 'true' }} - run: | - cd %WIN_INTERNAL_ROOT% - git pull origin %TARGET_BRANCH% - git fetch origin +refs/pull/%PR_NUMBER%/merge - git checkout -qf FETCH_HEAD - cd %WIN_COMMUNITY_ROOT% - git remote prune origin - git pull - shell: cmd - - name: Get latest codes and logs for TDengine PR - if: ${{ env.IS_TDINTERNAL == 'false' }} - run: | - cd %WIN_INTERNAL_ROOT% - git pull origin %TARGET_BRANCH% - cd %WIN_COMMUNITY_ROOT% - git remote prune origin - git pull origin %TARGET_BRANCH% - git fetch origin +refs/pull/%PR_NUMBER%/merge - git checkout -qf FETCH_HEAD - shell: cmd - - name: Output branch and log information - run: | - cd %WIN_INTERNAL_ROOT% - git branch - git log -5 - - cd %WIN_COMMUNITY_ROOT% - git branch - git log -5 - shell: cmd - - name: Update submodule - run: | - cd %WIN_COMMUNITY_ROOT% - git submodule update --init --recursive - shell: cmd - - name: Build on windows - run: | - echo "building ..." - time /t - cd %WIN_INTERNAL_ROOT% - mkdir debug - cd debug - time /t - call "%WIN_VS_PATH%" %WIN_CPU_TYPE% - set CL=/MP8 - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" - time /t - cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true || exit 7 - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" - time /t - jom -j 6 || exit 8 - time /t - - cd %WIN_COMMUNITY_ROOT%/tests/ci - pip3 install taospy==2.7.21 - pip3 install taos-ws-py==0.3.8 - xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 - shell: cmd - - name: Run ctest - run: | - echo "windows ctest ..." - time /t - cd %WIN_INTERNAL_ROOT%\\debug - ctest -j 1 || exit 7 - time /t - shell: cmd - - name: Run function test - run: | - echo "windows test ..." - xcopy /e/y/i/f "%WIN_INTERNAL_ROOT%\debug\build\lib\taos.dll" C:\Windows\System32 - ls -l "C:\Windows\System32\taos.dll" - time /t - cd %WIN_SYSTEM_TEST_ROOT% - echo "testing ..." - test-all.bat ci - time /t - shell: cmd + with: + tdinternal: false diff --git a/.github/workflows/tdgpt-ci.yml b/.github/workflows/tdgpt-ci.yml new file mode 100644 index 0000000000..4a438938dc --- /dev/null +++ b/.github/workflows/tdgpt-ci.yml @@ -0,0 +1,51 @@ +name: TDgpt CI + +on: + pull_request: + branches: + - '3.0' + paths: + - 'tools/tdgpt/**' + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + + defaults: + run: + working-directory: ${{ github.workspace }}/tools/tdgpt + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest pylint + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Checking the code with pylint + run: | + pylint $(git ls-files '*.py') --exit-zero + + - name: Checking the code with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Run test cases with pytest + run: | + pytest diff --git a/.github/workflows/tdgpt-update-service.yml b/.github/workflows/tdgpt-update-service.yml new file mode 100644 index 0000000000..fa7344ce74 --- /dev/null +++ b/.github/workflows/tdgpt-update-service.yml @@ -0,0 +1,41 @@ +name: TDgpt Update Service + +on: + schedule: + - cron: '30 00 * * *' + +env: + WKC: "/root/TDengine" + +jobs: + update-service: + runs-on: + group: CI + labels: [self-hosted, Linux, X64, tdgpt-anode-service] + steps: + - name: Update TDengine codes + run: | + set -euo pipefail + cd ${{ env.WKC }} + git checkout 3.0 + + - name: Package the TDGpt Anode Service + run: | + set -euo pipefail + cd ${{ env.WKC }}/tools/tdgpt/script && ./release.sh + + - name: Reinstall and restart the TDGpt Anode Service + run: | + set -euo pipefail + cd ${{ env.WKC }}/tools/tdgpt/release + if [[ -f "TDengine-enterprise-anode-1.0.1.tar.gz" ]]; then + tar -xzf TDengine-enterprise-anode-1.0.1.tar.gz + cd TDengine-enterprise-anode-1.0.1 + ./install.sh + fi + systemctl restart taosanoded + + - name: Clean up + if: always() + run: | + if [[ -f ${{ env.WKC }}/tools/tdgpt/release/TDengine-enterprise-anode-1.0.1 ]] then rm -rf ${{ env.WKC }}/tools/tdgpt/release/TDengine-enterprise-anode-1.0.1; fi diff --git a/docs/en/05-basic/03-query.md b/docs/en/05-basic/03-query.md index 463f58fd6e..d0aef17f4d 100644 --- a/docs/en/05-basic/03-query.md +++ b/docs/en/05-basic/03-query.md @@ -145,20 +145,19 @@ Query OK, 10 row(s) in set (2.415961s) In TDengine, you can use the window clause to perform aggregation queries by time window partitioning, which is particularly suitable for scenarios requiring analysis of large amounts of time-series data, such as smart meters collecting data every 10s but needing to query the average temperature every 1min. -The window clause allows you to partition the queried data set by windows and aggregate the data within each window, including: - -- Time window (time window) -- State window (status window) -- Session window (session window) -- Event window (event window) - -The logic of window partitioning is shown in the following image: +The window clause allows you to partition the queried data set by windows and aggregate the data within each window. The logic of window partitioning is shown in the following image:
Windowing description
Figure 1. Windowing logic
+- Time Window: Data is divided based on time intervals, supporting sliding and tumbling time windows, suitable for data aggregation over fixed time periods. +- Status Window: Windows are divided based on changes in device status values, with data of the same status value grouped into one window, which closes when the status value changes. +- Session Window: Sessions are divided based on the differences in record timestamps, with records having a timestamp interval less than the predefined value belonging to the same session. +- Event Window: Windows are dynamically divided based on the start and end conditions of events, opening when the start condition is met and closing when the end condition is met. +- Count Window: Windows are divided based on the number of data rows, with each window consisting of a specified number of rows for aggregation calculations. + The syntax for the window clause is as follows: ```sql diff --git a/docs/en/06-advanced/05-data-in/07-mqtt.md b/docs/en/06-advanced/05-data-in/07-mqtt.md index 73ef3b534c..47ffa82bf2 100644 --- a/docs/en/06-advanced/05-data-in/07-mqtt.md +++ b/docs/en/06-advanced/05-data-in/07-mqtt.md @@ -86,9 +86,15 @@ The keep alive interval is the time interval negotiated between the client and t In **Clean Session**, choose whether to clear the session. The default value is true. -Fill in the Topic names to be consumed in **Subscription Topics and QoS Configuration**. Use the following format: `topic1::0,topic2::1`. +In the **Topics Qos Config**, fill in the topic name and QoS to subscribe. Use the following format: `{topic_name}::{qos}` (e.g., `my_topic::0`). MQTT protocol 5.0 supports shared subscriptions, allowing multiple clients to subscribe to the same topic for load balancing. Use the following format: `$share/{group_name}/{topic_name}::{qos}`, where `$share` is a fixed prefix indicating the enablement of shared subscription, and `group_name` is the client group name, similar to Kafka's consumer group. -Click the **Check Connectivity** button to check if the data source is available. +In the **Topic Analysis**, fill in the MQTT topic parsing rules. The format is the same as the MQTT Topic, parsing each level of the MQTT Topic into corresponding variable names, with `_` indicating that the current level is ignored during parsing. For example: if the MQTT Topic `a/+/c` corresponds to the parsing rule `v1/v2/_`, it means assigning the first level `a` to variable `v1`, the value of the second level (where the wildcard `+` represents any value) to variable `v2`, and ignoring the value of the third level `c`, which will not be assigned to any variable. In the `payload parsing` below, the variables obtained from Topic parsing can also participate in various transformations and calculations. + +In the **Compression**, configure the message body compression algorithm. After receiving the message, taosX uses the corresponding compression algorithm to decompress the message body and obtain the original data. Options include none (no compression), gzip, snappy, lz4, and zstd, with the default being none. + +In the **Char Encoding**, configure the message body encoding format. After receiving the message, taosX uses the corresponding encoding format to decode the message body and obtain the original data. Options include UTF_8, GBK, GB18030, and BIG5, with the default being UTF_8. + +Click the **Check Connection** button to check if the data source is available.
diff --git a/docs/en/08-operation/03-deployment/03-kubernetes.md b/docs/en/08-operation/03-deployment/03-kubernetes.md index dbb6022ce6..fdec50b9d6 100644 --- a/docs/en/08-operation/03-deployment/03-kubernetes.md +++ b/docs/en/08-operation/03-deployment/03-kubernetes.md @@ -339,7 +339,7 @@ Helm operates Kubernetes using kubectl and kubeconfig configurations, which can The TDengine Chart has not yet been released to the Helm repository, it can currently be downloaded directly from GitHub: ```shell -wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-enterpise-3.5.0.tgz +wget https://github.com/taosdata/TDengine-Operator/raw/refs/heads/3.0/helm/tdengine-enterprise-3.5.0.tgz ``` Note that it's for the enterprise edition, and the community edition is not yet available. diff --git a/docs/en/14-reference/01-components/01-taosd.md b/docs/en/14-reference/01-components/01-taosd.md index 4527a7fcac..5ce9a40ecb 100644 --- a/docs/en/14-reference/01-components/01-taosd.md +++ b/docs/en/14-reference/01-components/01-taosd.md @@ -43,7 +43,7 @@ After modifying configuration file parameters, you need to restart the *taosd* s |resolveFQDNRetryTime | Cancelled after 3.x |Not supported |Number of retries when FQDN resolution fails| |timeToGetAvailableConn | Cancelled after 3.3.4.x |Maximum waiting time to get an available connection, range 10-50000000, in milliseconds, default value 500000| |maxShellConns | Cancelled after 3.x |Supported, effective after restart|Maximum number of connections allowed| -|maxRetryWaitTime | |Supported, effective after restart|Maximum timeout for reconnection,calculated from the time of retry,range is 0-86400000,in milliseconds, default value 10000| +|maxRetryWaitTime | |Supported, effective after restart|Maximum timeout for reconnection,calculated from the time of retry,range is 3000-86400000,in milliseconds, default value 10000| |shareConnLimit |Added in 3.3.4.0 |Supported, effective after restart|Number of requests a connection can share, range 1-512, default value 10| |readTimeout |Added in 3.3.4.0 |Supported, effective after restart|Minimum timeout for a single request, range 64-604800, in seconds, default value 900| diff --git a/docs/en/14-reference/01-components/02-taosc.md b/docs/en/14-reference/01-components/02-taosc.md index 7b8d3ae78c..83cb3d765b 100644 --- a/docs/en/14-reference/01-components/02-taosc.md +++ b/docs/en/14-reference/01-components/02-taosc.md @@ -44,6 +44,7 @@ The TDengine client driver provides all the APIs needed for application programm |enableQueryHb | |Supported, effective immediately |Internal parameter, whether to send query heartbeat messages| |minSlidingTime | |Supported, effective immediately |Internal parameter, minimum allowable value for sliding| |minIntervalTime | |Supported, effective immediately |Internal parameter, minimum allowable value for interval| +|compareAsStrInGreatest | v3.3.6.0 |Supported, effective immediately |When the greatest and least functions have both numeric and string types as parameters, the comparison type conversion rules are as follows: Integer; 1: uniformly converted to string comparison, 0: uniformly converted to numeric type comparison.| ### Writing Related diff --git a/docs/en/14-reference/02-tools/10-taosbenchmark.md b/docs/en/14-reference/02-tools/10-taosbenchmark.md index cfc92b4e0b..19f498eab1 100644 --- a/docs/en/14-reference/02-tools/10-taosbenchmark.md +++ b/docs/en/14-reference/02-tools/10-taosbenchmark.md @@ -188,9 +188,12 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) The parameters listed in this section apply to all functional modes. -- **filetype**: The function to test, possible values are `insert`, `query`, and `subscribe`. Corresponding to insert, query, and subscribe functions. Only one can be specified in each configuration file. +- **filetype**: The function to test, possible values are `insert`, `query`, `subscribe` and `csvfile`. Corresponding to insert, query, subscribe and generate csv file functions. Only one can be specified in each configuration file. + - **cfgdir**: Directory where the TDengine client configuration file is located, default path is /etc/taos. +- **output_dir**: The directory specified for output files. When the feature category is csvfile, it refers to the directory where the generated csv files will be saved. The default value is ./output/. + - **host**: Specifies the FQDN of the TDengine server to connect to, default value is localhost. - **port**: The port number of the TDengine server to connect to, default value is 6030. @@ -283,6 +286,27 @@ Parameters related to supertable creation are configured in the `super_tables` s - **repeat_ts_max** : Numeric type, when composite primary key is enabled, specifies the maximum number of records with the same timestamp to be generated - **sqls** : Array of strings type, specifies the array of sql to be executed after the supertable is successfully created, the table name specified in sql must be prefixed with the database name, otherwise an unspecified database error will occur +- **csv_file_prefix**: String type, sets the prefix for the names of the generated csv files. Default value is "data". + +- **csv_ts_format**: String type, sets the format of the time string in the names of the generated csv files, following the `strftime` format standard. If not set, files will not be split by time intervals. Supported patterns include: + - %Y: Year as a four-digit number (e.g., 2025) + - %m: Month as a two-digit number (01 to 12) + - %d: Day of the month as a two-digit number (01 to 31) + - %H: Hour in 24-hour format as a two-digit number (00 to 23) + - %M: Minute as a two-digit number (00 to 59) + - %S: Second as a two-digit number (00 to 59) + +- **csv_ts_interval**: String type, sets the time interval for splitting generated csv file names. Supports daily, hourly, minute, and second intervals such as 1d/2h/30m/40s. The default value is "1d". + +- **csv_output_header**: String type, sets whether the generated csv files should contain column header descriptions. The default value is "yes". + +- **csv_tbname_alias**: String type, sets the alias for the tbname field in the column header descriptions of csv files. The default value is "device_id". + +- **csv_compress_level**: String type, sets the compression level for generating csv-encoded data and automatically compressing it into gzip file. This process directly encodes and compresses the data, rather than first generating a csv file and then compressing it. Possible values are: + - none: No compression + - fast: gzip level 1 compression + - balance: gzip level 6 compression + - best: gzip level 9 compression #### Tag and Data Columns @@ -478,6 +502,17 @@ Note: Data types in the taosBenchmark configuration file must be in lowercase to +### Export CSV File Example + +
+csv-export.json + +```json +{{#include /TDengine/tools/taos-tools/example/csv-export.json}} +``` + +
+ Other json examples see [here](https://github.com/taosdata/TDengine/tree/main/tools/taos-tools/example) ## Output Performance Indicators diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index 7875501e7e..993b1176fe 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -124,7 +124,39 @@ FLOOR(expr) ``` **Function Description**: Gets the floor of the specified field. - Other usage notes see CEIL function description. + Other usage notes see [CEIL](#ceil) function description. + +#### GREATEST +```sql +GREATEST(expr1, expr2[, expr]...) +``` + +**Function Description**: Get the maximum value of all input parameters. The minimum number of parameters for this function is 2. + +**Version**：ver-3.3.6.0 + +**Return Type**：Refer to the comparison rules. The comparison type is the final return type. + +**Applicable Data Types**: +- Numeric types: timestamp, bool, integer and floating point types +- Strings types: nchar and varchar types. + +**Comparison rules**: The following rules describe the conversion method of the comparison operation: +- If any parameter is NULL, the comparison result is NULL. +- If all parameters in the comparison operation are string types, compare them as string types +- If all parameters are numeric types, compare them as numeric types. +- If there are both string types and numeric types in the parameters, according to the `compareAsStrInGreatest` configuration item, they are uniformly compared as strings or numeric values. By default, they are compared as strings. +- In all cases, when different types are compared, the comparison type will choose the type with a larger range for comparison. For example, when comparing integer types, if there is a BIGINT type, BIGINT will definitely be selected as the comparison type. + +**Related configuration items**: Client configuration, compareAsStrInGreatest is 1, which means that both string types and numeric types are converted to string comparisons, and 0 means that they are converted to numeric types. The default is 1. + + +#### LEAST +```sql +LEAST(expr1, expr2[, expr]...) +``` + +**Function Description**：Get the minimum value of all input parameters. The rest of the description is the same as the [GREATEST](#greatest) function. #### LOG diff --git a/docs/en/26-tdinternal/09-stream.md b/docs/en/26-tdinternal/09-stream.md index 2c8d41cccb..69424652dc 100644 --- a/docs/en/26-tdinternal/09-stream.md +++ b/docs/en/26-tdinternal/09-stream.md @@ -94,7 +94,7 @@ The sink task is responsible for receiving the output results from the agg task The above three types of tasks each play their roles in the stream computing architecture, distributed at different levels. Clearly, the number of source tasks directly depends on the number of vnodes, with each source task independently handling the data in its vnode without interference from other source tasks, and there are no sequential constraints. However, it is worth noting that if the final stream computing results converge to one table, then only one sink task will be deployed on the vnode where that table is located. The collaborative relationship between these three types of tasks is shown in the following diagram, together forming the complete execution process of stream computing tasks.
-Relationships between tasks +Relationships between tasks
Figure 3. Relationships between tasks
diff --git a/docs/en/27-train-faq/02-dst.md b/docs/en/27-train-faq/02-dst.md new file mode 100644 index 0000000000..6fda8f8a16 --- /dev/null +++ b/docs/en/27-train-faq/02-dst.md @@ -0,0 +1,293 @@ +--- +title: DST(Daylight Saving Time) Usage +description: Explanation and suggestions for using DST(Daylight Saving Time) in TDengine +--- + +## Background + +In the use of time-series databases, there are times when Daylight Saving Time (DST) is encountered. We analyze and explain the use and issues of DST in TDengine to help you use TDengine more smoothly. + +## Definitions + +### Time Zone + +A time zone is a region on Earth that uses the same standard time. Due to the Earth's rotation, to ensure that the time in each place is coordinated with the local sunrise and sunset, the world is divided into multiple time zones. + +### IANA Time Zone + +The IANA (Internet Assigned Numbers Authority) time zone database, also known as the tz database, provides a standard reference for global time zone information. It is the basis for modern systems and software to handle time zone-related operations. + +IANA uses the "Region/City" format (e.g., Europe/Berlin) to clearly identify time zones. + +TDengine supports the use of IANA time zones in different components (except for the time zone settings in Windows taos.cfg). + +### Standard Time and Local Time + +Standard time is the time determined based on a fixed meridian on Earth. It provides a unified reference point for each time zone. + +- Greenwich Mean Time (GMT): Historically used reference time, located at the 0° meridian. +- Coordinated Universal Time (UTC): The modern time standard, similar to GMT but more precise. + +The relationship between standard time and time zones is as follows: + +- Reference: Standard time (e.g., UTC) is the reference point for setting time zones. +- Offset: Different time zones are defined by their offset from standard time. For example, UTC+1 means 1 hour ahead of UTC. +- Regional Division: The world is divided into multiple time zones, each using one or more standard times. + +Relative to standard time, each region sets its local time based on its time zone: + +- Time Zone Offset: Local time equals standard time plus the offset of the time zone. For example, UTC+2 means 2 hours ahead of UTC. +- Daylight Saving Time (DST): Some regions adjust their local time during specific periods, such as moving the clock forward by one hour. See the next section for details. + +### Daylight Saving Time + +Daylight Saving Time (DST) is a system that advances the time by one hour to make better use of daylight and save energy. It usually starts in spring and ends in autumn. The specific start and end times of DST vary by region. The following explanation uses Berlin time as an example to illustrate DST and its effects. + +![DST Berlin](./02-dst/dst-berlin.png) + +According to this rule, you can see: + +- The time between 02:00:00 and 03:00:00 (excluding 03:00:00) on March 31, 2024, in Berlin local time does not exist (jump). +- The time between 02:00:00 and 03:00:00 (excluding 03:00:00) on October 27, 2024, in Berlin local time appears twice. + +#### DST and the IANA Time Zone Database + +- Recording Rules: The IANA time zone database records detailed DST rules for each region, including the start and end dates and times. +- Automatic Adjustment: Many operating systems and software use the IANA database to automatically handle DST adjustments. +- Historical Changes: The IANA database also tracks historical DST changes to ensure accuracy. + +#### DST and Timestamp Conversion + +- Converting a timestamp to local time is deterministic. For example, 1729990654 is Berlin time DST 2024-10-27 02:57:34, and 1729994254 is Berlin time standard time 2024-10-27 02:57:34 (these two local times are the same except for the time offset). +- Without specifying the time offset, converting local time to a timestamp is indeterminate. The time skipped during DST does not exist and cannot be converted to a timestamp, such as Berlin time 2024-03-31 02:34:56 does not exist and cannot be converted to a timestamp. The repeated time during the end of DST cannot determine which timestamp it is, such as 2024-10-27 02:57:34 without specifying the time offset cannot determine whether it is 1729990654 or 1729994254. Specifying the time offset can determine the timestamp, such as 2024-10-27 02:57:34 CEST(+02:00), specifying DST 2024-10-27 02:57:34 timestamp 1729990654. + +### RFC3339 Time Format + +RFC 3339 is an internet time format standard used to represent dates and times. It is based on the ISO 8601 standard but specifies some format details more specifically. + +The format is as follows: + +- Basic Format: `YYYY-MM-DDTHH:MM:SSZ` +- Time Zone Representation: + - Z represents Coordinated Universal Time (UTC). + - Offset format, such as +02:00, represents the time difference from UTC. + +With explicit time zone offsets, the RFC 3339 format can accurately parse and compare times globally. + +The advantages of RFC 3339 include: + +- Standardization: Provides a unified format for easy cross-system data exchange. +- Clarity: Clearly indicates time zone information, avoiding time misunderstandings. + +TDengine uses the RFC3339 format for display in REST API and Explorer UI. In SQL statements, you can use the RFC3339 format to write timestamp data: + +```sql +insert into t1 values('2024-10-27T01:59:59.000Z', 0); +select * from t1 where ts >= '2024-10-27T01:59:59.000Z'; +``` + +### Undefined Behavior + +Undefined behavior refers to specific code or operations that do not have a clearly defined result and do not guarantee compatibility with that result. TDengine may modify the current behavior in a future version without notifying users. Therefore, users should not rely on the current undefined behavior for judgment or application in TDengine. + +## Writing and Querying DST in TDengine + +We use the following table to show the impact of DST on writing and querying. + +![DST Table](./02-dst/dst-table.png) + +### Table Explanation + +- **TIMESTAMP**: TDengine uses a 64-bit integer to store raw timestamps. +- **UTC**: The UTC time representation corresponding to the timestamp. +- **Europe/Berlin**: The RFC3339 format time corresponding to the Europe/Berlin time zone. +- **Local**: The local time corresponding to the Europe/Berlin time zone (without time zone). + +### Table Analysis + +- At the **start of DST** (Berlin time March 31, 02:00), the time jumps directly from 02:00 to 03:00 (one hour forward). + - Light green is the timestamp one hour before the start of DST; + - Dark green is the timestamp one hour after the start of DST; + - Red indicates that the nonexistent local time was inserted into the TDengine database: + - Using SQL `INSERT INTO t1 values('2024-03-31 02:59:59',..)` to insert data from `2024-03-31 02:00:00` to `2024-03-31 02:59:59` will be automatically adjusted to -1000 (in TDengine, this is undefined behavior, currently this value is related to the database precision, millisecond database is -1000, microsecond database is -1000000, nanosecond database is -1000000000), because that moment does not exist in local time; +- At the **end of DST** (Berlin time October 27, 03:00), the time jumps from 03:00 to 02:00 (one hour back). + - Light blue indicates the timestamp one hour before the clock jump; + - Dark blue indicates the timestamp within one hour after the clock jump, its local time without time zone is the same as the previous hour. + - Purple indicates the timestamp one hour after the clock jump; +- **Local Time Changes**: It can be seen that due to the adjustment of DST, local time changes, which may cause some time periods to appear repeated or missing. +- **UTC Time Unchanged**: UTC time remains unchanged, ensuring the consistency and order of time. +- **RFC3339**: The RFC3339 format time shows the change in time offset, changing to +02:00 after the start of DST and to +01:00 after the end of DST. +- **Conditional Query**: + - At the **start of DST**, the skipped time (`[03-31 02:00:00,03-31 03:00:00)`) does not exist, so using that time for queries results in undefined behavior: `SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'` (the nonexistent local timestamp is converted to `-1000`): + + ```sql + taos> SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'; + ts | + ================= + -1000 | + Query OK, 1 row(s) in set (0.003635s) + ``` + + When the nonexistent timestamp is used together with the existing timestamp, the result is also not as expected, as shown below where the start local time does not exist: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 03:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + -1000 | 1969-12-31T23:59:59.000Z | + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + 1711846800000 | 2024-03-31T01:00:00.000Z | + 1711846801000 | 2024-03-31T01:00:01.000Z | + Query OK, 5 row(s) in set (0.003339s) + ``` + + In the following statements, the first SQL query end time does not exist, and the second end time exists. The first SQL query result is not as expected: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 02:00:00'; + Query OK, 0 row(s) in set (0.000930s) + + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 01:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + Query OK, 2 row(s) in set (0.001227s) + ``` + + - At the end of DST, the repeated time (`[10-27 02:00:00,10-27 03:00:00)` excluding `10-27 03:00:00`) appears twice, and using that time range for queries in TDengine is also undefined behavior. + - Querying the data between `[2024-10-27 02:00:00, 2024-10-27 03:00:00]` includes the repeated timestamps and the data at `2024-10-27 03:00:00`: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts BETWEEN '2024-10-27 02:00:00' AND '2024-10-27 03:00:00'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + 1729994400000 | 2024-10-27T02:00:00.000Z | 2024-10-27 03:00:00 | + Query OK, 5 row(s) in set (0.001370s) + ``` + + - However, the following query for the range [2024-10-27 02:00:00.000,2024-10-27 02:57:34.999] can only find the data at the first 2024-10-27 02:00:00 time point: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:00.999'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 1 row(s) in set (0.004480s) + ``` + + - The following query for the range [2024-10-27 02:00:01,2024-10-27 02:57:35] can find 3 rows of data (including one row of local time data at 02:59:59): + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:35';; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + =============================================================================================== + 2024-10-27 02:00:00.000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 2024-10-27 02:59:59.000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 2024-10-27 02:00:00.000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 3 row(s) in set (0.004428s) + ``` + +## Summary and Suggestions + +### Summary + +This explanation only addresses the impact of using local time. Using UNIX timestamps or RFC3339 has no impact. + +- Writing: + - It is not possible to write data for nonexistent times during the DST transition. + - Writing data for repeated times during the DST transition is undefined behavior. +- Querying: + - Querying with conditions that specify the skipped time during the start of DST results in undefined behavior. + - Querying with conditions that specify the repeated time during the end of DST results in undefined behavior. +- Display: + - Displaying with time zones is not affected. + - Displaying local time is accurate, but repeated times during the end of DST cannot be distinguished. + - Users should be cautious when using time without time zones for display and application. + +### Suggestions + +To avoid unnecessary impacts of DST on querying and writing in TDengine, it is recommended to use explicit time offsets for writing and querying. + +- Use UNIX Timestamps: Using UNIX timestamps can avoid time zone issues. + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1711846799000 | 2024-03-31T00:59:59.000Z | 2024-03-31T01:59:59.000+01:00 | 2024-03-31 01:59:59 | + | 1711846800000 | 2024-03-31T01:00:00.000Z | 2024-03-31T03:00:00.000+02:00 | 2024-03-31 03:00:00 | + + ```sql + taos> insert into t1 values(1711846799000, 1)(1711846800000, 2); + Insert OK, 2 row(s) affected (0.001434s) + + taos> select * from t1 where ts between 1711846799000 and 1711846800000; + ts | v1 | + =============================== + 1711846799000 | 1 | + 1711846800000 | 2 | + Query OK, 2 row(s) in set (0.003503s) + ``` + +- Use RFC3339 Time Format: The RFC3339 time format with time zone offsets can effectively avoid the uncertainty of DST. + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27T02:00:00.000+02:00 | 2024-10-27 02:00:00 | + | 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27T02:59:59.000+02:00 | 2024-10-27 02:59:59 | + | 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27T02:00:00.000+01:00 | 2024-10-27 02:00:00 | + | 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27T02:59:59.000+01:00 | 2024-10-27 02:59:59 | + + ```sql + taos> insert into t1 values ('2024-10-27T02:00:00.000+02:00', 1) + ('2024-10-27T02:59:59.000+02:00', 2) + ('2024-10-27T02:00:00.000+01:00', 3) + ('2024-10-27T02:59:59.000+01:00', 4); + Insert OK, 4 row(s) affected (0.001514s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+01:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 3 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 4 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 4 row(s) in set (0.004275s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+02:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 2 row(s) in set (0.004275s) + ``` + +- Pay Attention to Time Zone Settings When Querying: When querying and displaying, if local time is needed, be sure to consider the impact of DST. + - taosAdapter: When using the REST API, it supports setting the IANA time zone, and the result is returned in RFC3339 format. + + ```shell + $ curl -uroot:taosdata 'localhost:6041/rest/sql?tz=Europe/Berlin'\ + -d "select ts from tz1.t1" + {"code":0,"column_meta":[["ts","TIMESTAMP",8]],"data":[["1970-01-01T00:59:59.000+01:00"],["2024-03-31T01:00:00.000+01:00"],["2024-03-31T01:59:59.000+01:00"],["2024-03-31T03:00:00.000+02:00"],["2024-03-31T03:00:01.000+02:00"],["2024-10-27T02:00:00.000+02:00"],["2024-10-27T02:59:59.000+02:00"],["2024-10-27T02:00:00.000+01:00"],["2024-10-27T02:59:59.000+01:00"],["2024-10-27T03:00:00.000+01:00"]],"rows":10} + ``` + + - Explorer: When using the Explorer page for SQL queries, users can configure the client time zone to display in RFC3339 format. + + ![Explorer DST](./02-dst/explorer-with-tz.png) + +## Reference Documents + +- IANA Time Zone Database: [https://www.iana.org/time-zones](https://www.iana.org/time-zones) +- RFC3339: [https://datatracker.ietf.org/doc/html/rfc3339](https://datatracker.ietf.org/doc/html/rfc3339) diff --git a/docs/en/27-train-faq/02-dst/dst-berlin.png b/docs/en/27-train-faq/02-dst/dst-berlin.png new file mode 100644 index 0000000000..fa1125de25 Binary files /dev/null and b/docs/en/27-train-faq/02-dst/dst-berlin.png differ diff --git a/docs/en/27-train-faq/02-dst/dst-table.png b/docs/en/27-train-faq/02-dst/dst-table.png new file mode 100644 index 0000000000..f5eddbe010 Binary files /dev/null and b/docs/en/27-train-faq/02-dst/dst-table.png differ diff --git a/docs/en/27-train-faq/02-dst/explorer-with-tz.png b/docs/en/27-train-faq/02-dst/explorer-with-tz.png new file mode 100644 index 0000000000..95758f117a Binary files /dev/null and b/docs/en/27-train-faq/02-dst/explorer-with-tz.png differ diff --git a/docs/en/assets/architecture-01.png b/docs/en/assets/architecture-01.png index a9e2bf9f70..71f6b06f5a 100644 Binary files a/docs/en/assets/architecture-01.png and b/docs/en/assets/architecture-01.png differ diff --git a/docs/en/assets/architecture-02.png b/docs/en/assets/architecture-02.png index e558a830ae..92580995c1 100644 Binary files a/docs/en/assets/architecture-02.png and b/docs/en/assets/architecture-02.png differ diff --git a/docs/en/assets/architecture-03.png b/docs/en/assets/architecture-03.png index 467580248c..60d786a782 100644 Binary files a/docs/en/assets/architecture-03.png and b/docs/en/assets/architecture-03.png differ diff --git a/docs/en/assets/architecture-04.png b/docs/en/assets/architecture-04.png index e034f97f89..06ecf72a26 100644 Binary files a/docs/en/assets/architecture-04.png and b/docs/en/assets/architecture-04.png differ diff --git a/docs/en/assets/data-compression-01.png b/docs/en/assets/data-compression-01.png index 64a36f4a42..31abb35926 100644 Binary files a/docs/en/assets/data-compression-01.png and b/docs/en/assets/data-compression-01.png differ diff --git a/docs/en/assets/data-querying-01.png b/docs/en/assets/data-querying-01.png index 381d6ec88e..36cc854fe2 100644 Binary files a/docs/en/assets/data-querying-01.png and b/docs/en/assets/data-querying-01.png differ diff --git a/docs/en/assets/data-querying-03.png b/docs/en/assets/data-querying-03.png index 416a14a0a9..b1f4024322 100644 Binary files a/docs/en/assets/data-querying-03.png and b/docs/en/assets/data-querying-03.png differ diff --git a/docs/en/assets/data-querying-04.png b/docs/en/assets/data-querying-04.png index fc83f00193..a249dac3c3 100644 Binary files a/docs/en/assets/data-querying-04.png and b/docs/en/assets/data-querying-04.png differ diff --git a/docs/en/assets/data-subscription-engine-01.png b/docs/en/assets/data-subscription-engine-01.png index d9306cdb74..088d0070be 100644 Binary files a/docs/en/assets/data-subscription-engine-01.png and b/docs/en/assets/data-subscription-engine-01.png differ diff --git a/docs/en/assets/data-subscription-engine-02.png b/docs/en/assets/data-subscription-engine-02.png index 0c9a061107..0bbd9ff0af 100644 Binary files a/docs/en/assets/data-subscription-engine-02.png and b/docs/en/assets/data-subscription-engine-02.png differ diff --git a/docs/en/assets/data-subscription-engine-03.png b/docs/en/assets/data-subscription-engine-03.png index 0ddb005d66..df61f32208 100644 Binary files a/docs/en/assets/data-subscription-engine-03.png and b/docs/en/assets/data-subscription-engine-03.png differ diff --git a/docs/en/assets/data-subscription-engine-04.png b/docs/en/assets/data-subscription-engine-04.png index 1bb4bb3014..921d6d7de6 100644 Binary files a/docs/en/assets/data-subscription-engine-04.png and b/docs/en/assets/data-subscription-engine-04.png differ diff --git a/docs/en/assets/data-subscription-engine-05.png b/docs/en/assets/data-subscription-engine-05.png index fbc2147c91..5e7867170c 100644 Binary files a/docs/en/assets/data-subscription-engine-05.png and b/docs/en/assets/data-subscription-engine-05.png differ diff --git a/docs/en/assets/data-subscription-engine-06.png b/docs/en/assets/data-subscription-engine-06.png index dd2b472890..256c6abc9c 100644 Binary files a/docs/en/assets/data-subscription-engine-06.png and b/docs/en/assets/data-subscription-engine-06.png differ diff --git a/docs/en/assets/data-subscription-engine-07.png b/docs/en/assets/data-subscription-engine-07.png index 6cd1670b64..978fa022ee 100644 Binary files a/docs/en/assets/data-subscription-engine-07.png and b/docs/en/assets/data-subscription-engine-07.png differ diff --git a/docs/en/assets/mqtt-05.png b/docs/en/assets/mqtt-05.png index 5345b3923f..5644330e14 100644 Binary files a/docs/en/assets/mqtt-05.png and b/docs/en/assets/mqtt-05.png differ diff --git a/docs/en/assets/query-engine-01.png b/docs/en/assets/query-engine-01.png index 08d42dc038..da850c3116 100644 Binary files a/docs/en/assets/query-engine-01.png and b/docs/en/assets/query-engine-01.png differ diff --git a/docs/en/assets/query-engine-02.png b/docs/en/assets/query-engine-02.png index ae7ab92c0a..dcb5360f62 100644 Binary files a/docs/en/assets/query-engine-02.png and b/docs/en/assets/query-engine-02.png differ diff --git a/docs/en/assets/storage-engine-01.png b/docs/en/assets/storage-engine-01.png index 40cba22780..f5cc5867bc 100644 Binary files a/docs/en/assets/storage-engine-01.png and b/docs/en/assets/storage-engine-01.png differ diff --git a/docs/en/assets/storage-engine-02.png b/docs/en/assets/storage-engine-02.png index c6ff24bccb..f201b4dbc4 100644 Binary files a/docs/en/assets/storage-engine-02.png and b/docs/en/assets/storage-engine-02.png differ diff --git a/docs/en/assets/storage-engine-03.png b/docs/en/assets/storage-engine-03.png index fdc252a4ab..d083906367 100644 Binary files a/docs/en/assets/storage-engine-03.png and b/docs/en/assets/storage-engine-03.png differ diff --git a/docs/en/assets/storage-engine-04.png b/docs/en/assets/storage-engine-04.png index 606bd4eaeb..d265fcd6a7 100644 Binary files a/docs/en/assets/storage-engine-04.png and b/docs/en/assets/storage-engine-04.png differ diff --git a/docs/en/assets/storage-engine-05.png b/docs/en/assets/storage-engine-05.png index 4211ac2343..36febdfc85 100644 Binary files a/docs/en/assets/storage-engine-05.png and b/docs/en/assets/storage-engine-05.png differ diff --git a/docs/en/assets/storage-engine-06.png b/docs/en/assets/storage-engine-06.png index d310929b31..4f8eaa1547 100644 Binary files a/docs/en/assets/storage-engine-06.png and b/docs/en/assets/storage-engine-06.png differ diff --git a/docs/en/assets/storage-engine-07.png b/docs/en/assets/storage-engine-07.png index 7c82e017e9..bfbf0c03d3 100644 Binary files a/docs/en/assets/storage-engine-07.png and b/docs/en/assets/storage-engine-07.png differ diff --git a/docs/en/assets/storage-engine-08.png b/docs/en/assets/storage-engine-08.png index ecd1262e94..a2c4201fe7 100644 Binary files a/docs/en/assets/storage-engine-08.png and b/docs/en/assets/storage-engine-08.png differ diff --git a/docs/en/assets/storage-engine-09.png b/docs/en/assets/storage-engine-09.png index d07acee6ae..304926b04e 100644 Binary files a/docs/en/assets/storage-engine-09.png and b/docs/en/assets/storage-engine-09.png differ diff --git a/docs/en/assets/storage-engine-10.png b/docs/en/assets/storage-engine-10.png index 475dbae2fb..deba103a86 100644 Binary files a/docs/en/assets/storage-engine-10.png and b/docs/en/assets/storage-engine-10.png differ diff --git a/docs/en/assets/storage-engine-11.png b/docs/en/assets/storage-engine-11.png index 4d942ad728..43a3d625c8 100644 Binary files a/docs/en/assets/storage-engine-11.png and b/docs/en/assets/storage-engine-11.png differ diff --git a/docs/en/assets/stream-processing-engine-01.png b/docs/en/assets/stream-processing-engine-01.png index c8a8087273..407eb3c542 100644 Binary files a/docs/en/assets/stream-processing-engine-01.png and b/docs/en/assets/stream-processing-engine-01.png differ diff --git a/docs/en/assets/stream-processing-engine-02.png b/docs/en/assets/stream-processing-engine-02.png index 16c3be56b9..04ecf582de 100644 Binary files a/docs/en/assets/stream-processing-engine-02.png and b/docs/en/assets/stream-processing-engine-02.png differ diff --git a/docs/en/assets/stream-processing-engine-03.png b/docs/en/assets/stream-processing-engine-03.png index d9fae4908d..138f4eda39 100644 Binary files a/docs/en/assets/stream-processing-engine-03.png and b/docs/en/assets/stream-processing-engine-03.png differ diff --git a/docs/zh/05-basic/03-query.md b/docs/zh/05-basic/03-query.md index 0b2f290667..52b825c47c 100644 --- a/docs/zh/05-basic/03-query.md +++ b/docs/zh/05-basic/03-query.md @@ -4,6 +4,10 @@ title: TDengine 数据查询 toc_max_heading_level: 4 --- +import win from './window.png'; +import swin from './session-window.png'; +import ewin from './event-window.png'; + 相较于其他众多时序数据库和实时数据库，TDengine 的一个独特优势在于，自其首个版本发布之初便支持标准的 SQL 查询功能。这一特性极大地降低了用户在使用过程中的学习难度。本章将以智能电表的数据模型为例介绍如何在 TDengine 中运用 SQL 查询来处理时序数据。如果需要进一步了解 SQL 语法的细节和功能，建议参阅 TDengine 的官方文档。通过本章的学习，你将能够熟练掌握 TDengine 的 SQL 查询技巧，进而高效地对时序数据进行操作和分析。 ## 基本查询 @@ -136,16 +140,15 @@ Query OK, 10 row(s) in set (2.415961s) 在 TDengine 中，你可以使用窗口子句来实现按时间窗口切分方式进行聚合结果查询，这种查询方式特别适用于需要对大量时间序列数据进行分析的场景，例如智能电表每 10s 采集一次数据，但需要查询每隔 1min 的温度平均值。 -窗口子句允许你针对查询的数据集合按照窗口进行切分，并对每个窗口内的数据进行聚合，包含： -- 时间窗口（time window） -- 状态窗口（status window） -- 会话窗口（session window） -- 事件窗口（event window） -- 计数窗口（count window） +窗口子句允许你针对查询的数据集合按照窗口进行切分，并对每个窗口内的数据进行聚合。窗口划分逻辑如下图所示。 -窗口划分逻辑如下图所示： +常用窗口划分逻辑 -![常用窗口划分逻辑](./window.png) +- 时间窗口（time window）：根据时间间隔划分数据，支持滑动时间窗口和翻转时间窗口，适用于按固定时间周期进行数据聚合。 +- 状态窗口（status window）：基于设备状态值的变化划分窗口，相同状态值的数据归为一个窗口，状态值改变时窗口关闭。 +- 会话窗口（session window）：根据记录的时间戳差异划分会话，时间戳间隔小于预设值的记录属于同一会话。 +- 事件窗口（event window）：基于事件的开始条件和结束条件动态划分窗口，满足开始条件时窗口开启，满足结束条件时窗口关闭。 +- 计数窗口（count window）：根据数据行数划分窗口，每达到指定行数即为一个窗口，并进行聚合计算。 窗口子句语法如下： @@ -408,7 +411,8 @@ Query OK, 22 row(s) in set (0.153403s) 会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示，如果设置时间戳的连续的间隔小于等于 12 秒，则以下 6 条记录构成 2 个会话窗口，分别是：[2019-04-28 14:22:10，2019-04-28 14:22:30] 和 [2019-04-28 14:23:10，2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒，超过了连续时间间隔（12 秒）。 -![会话窗口示意图](./session-window.png) +会话窗口示意图 + 在 tol_value 时间间隔范围内的结果都认为归属于同一个窗口，如果连续的两条记录的时间超过 tol_val，则自动开启下一个窗口。 @@ -461,7 +465,7 @@ Query OK, 10 row(s) in set (0.043489s) select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c2 < 10 ``` -![事件窗口示意图](./event-window.png) +事件窗口示意图 示例 SQL: diff --git a/docs/zh/05-basic/event-window.png b/docs/zh/05-basic/event-window.png index 3a2a628ffe..a249dac3c3 100644 Binary files a/docs/zh/05-basic/event-window.png and b/docs/zh/05-basic/event-window.png differ diff --git a/docs/zh/05-basic/session-window.png b/docs/zh/05-basic/session-window.png index 416a14a0a9..b1f4024322 100644 Binary files a/docs/zh/05-basic/session-window.png and b/docs/zh/05-basic/session-window.png differ diff --git a/docs/zh/05-basic/window.png b/docs/zh/05-basic/window.png index 5e3efb8a3b..a6f5883917 100644 Binary files a/docs/zh/05-basic/window.png and b/docs/zh/05-basic/window.png differ diff --git a/docs/zh/06-advanced/06-TDgpt/07-faq.md b/docs/zh/06-advanced/06-TDgpt/07-faq.md index 9747dc18f4..4cfb5abf06 100644 --- a/docs/zh/06-advanced/06-TDgpt/07-faq.md +++ b/docs/zh/06-advanced/06-TDgpt/07-faq.md @@ -3,7 +3,12 @@ title: "常见问题" sidebar_label: "常见问题" --- -1. 创建 anode 失败，返回指定服务无法访问 +### 1. 安装过程中编译 uWSGI 失败，如何处理 +TDgpt 安装过程中需要在本地编译 uWSGI，某些环境的 Python（例如：anaconda）安装 uWSGI 会出现冲突导致编译失败，安装流程因此无法继续下去。这种情况下可以尝试在安装过程中忽略 uWSGI的安装。 +由于忽略了 uWSGI 安装，后续启动 taosasnode 服务的时候，需要手动输入命令进行启动 `python3.10 /usr/local/taos/taosanode/lib/taosanalytics/app.py` 。 执行该命令的时候请确保使用了虚拟环境中的 Python 程序才能加载依赖库。 + + +### 2. 创建 anode 失败，返回指定服务无法访问 ```bash taos> create anode '127.0.0.1:6090'; @@ -26,7 +31,7 @@ curl: (7) Failed to connect to 127.0.0.1 port 6090: Connection refused >请勿使用 systemctl status taosanode 检查 taosanode 是否正常 -2. 服务正常，查询过程返回服务不可用 +### 3. 服务正常，查询过程返回服务不可用 ```bash taos> select _frowts,forecast(current, 'algo=arima, alpha=95, wncheck=0, rows=20') from d1 where ts<='2017-07-14 10:40:09.999'; @@ -34,7 +39,7 @@ DB error: Analysis service can't access[0x80000441] (60.195613s) ``` 数据分析默认超时时间是 60s，出现这个问题的原因是输入数据分析过程超过默认的最长等待时间，请尝试采用限制数据输入范围的方式将输入数据规模减小或者更换分析算法再次尝试。 -3. 返回结果出现非法 JSON 格式错误 (Invalid json format) +### 4. 返回结果出现非法 JSON 格式错误 (Invalid json format) 从 anode 返回到 TDengine 的分析结果有误，请检查 anode 运行日志 `/var/log/taos/taosanode/taosanode.app.log`，以便于获得具体的错误信息。 diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 2216cab915..1e6667b31f 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -147,7 +147,7 @@ taosd 命令行参数如下 - 类型：整数 - 单位：毫秒 - 默认值：10000 -- 最小值：0 +- 最小值：3000 - 最大值：86400000 - 动态修改：支持通过 SQL 修改，重启后生效 - 支持版本：v3.3.4.0 引入 diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md index 262027bff6..8952e35664 100755 --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -221,6 +221,12 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 - 动态修改：不支持 - 支持版本：从 v3.0.0.0 版本开始引入 +#### compareAsStrInGreatest +- 说明：用于决定 greatest、least 函数的参数既有数值类型又有字符串类型时，比较类型的转换规则。 +- 类型：整数；1：统一转为字符串比较，0：统一转为数值类型比较。 +- 动态修改：支持通过 SQL 修改，立即生效 +- 支持版本：从 v3.3.6.0 版本开始引入 + ### 写入相关 #### smlChildTableName diff --git a/docs/zh/14-reference/02-tools/10-taosbenchmark.md b/docs/zh/14-reference/02-tools/10-taosbenchmark.md index 56f9e5b122..1f97b0702a 100644 --- a/docs/zh/14-reference/02-tools/10-taosbenchmark.md +++ b/docs/zh/14-reference/02-tools/10-taosbenchmark.md @@ -93,14 +93,17 @@ taosBenchmark -f 本节所列参数适用于所有功能模式。 -- **filetype**：功能分类，可选值为 `insert`、`query` 和 `subscribe`。分别对应插入、查询和订阅功能。每个配置文件中只能指定其中之一。 +- **filetype**：功能分类，可选值为 `insert`、`query`、`subscribe` 和 `csvfile`。分别对应插入、查询、订阅和生成 csv 文件功能。每个配置文件中只能指定其中之一。 + - **cfgdir**：TDengine 客户端配置文件所在的目录，默认路径是 /etc/taos 。 +- **output_dir**：指定输出文件的目录，当功能分类是 `csvfile` 时，指生成的 csv 文件的保存目录，默认值为 ./output/ 。 + - **host**：指定要连接的 TDengine 服务端的 FQDN，默认值为 localhost 。 - **port**：要连接的 TDengine 服务器的端口号，默认值为 6030 。 -- **user**：用于连接 TDengine 服务端的用户名，默认为 root 。 +- **user**：用于连接 TDengine 服务端的用户名，默认值为 root 。 - **password**：用于连接 TDengine 服务端的密码，默认值为 taosdata。 @@ -184,10 +187,34 @@ taosBenchmark -f - **tags_file**：仅当 insert_mode 为 taosc，rest 的模式下生效。最终的 tag 的数值与 childtable_count 有关，如果 csv 文件内的 tag 数据行小于给定的子表数量，那么会循环读取 csv 文件数据直到生成 childtable_count 指定的子表数量；否则则只会读取 childtable_count 行 tag 数据。也即最终生成的子表数量为二者取小。 - **primary_key**：指定超级表是否有复合主键，取值 1 和 0，复合主键列只能是超级表的第二列，指定生成复合主键后要确保第二列符合复合主键的数据类型，否则会报错。 + - **repeat_ts_min**：数值类型，复合主键开启情况下指定生成相同时间戳记录的最小个数，生成相同时间戳记录的个数是在范围[repeat_ts_min, repeat_ts_max] 内的随机值，最小值等于最大值时为固定个数。 + - **repeat_ts_max**：数值类型，复合主键开启情况下指定生成相同时间戳记录的最大个数。 + - **sqls**：字符串数组类型，指定超级表创建成功后要执行的 sql 数组，sql 中指定表名前面要带数据库名，否则会报未指定数据库错误。 +- **csv_file_prefix**：字符串类型，设置生成的 csv 文件名称的前缀，默认值为 data 。 + +- **csv_ts_format**：字符串类型，设置生成的 csv 文件名称中时间字符串的格式，格式遵循 `strftime` 格式标准，如果没有设置表示不按照时间段切分文件。支持的模式有： + - %Y: 年份，四位数表示（例如：2025） + - %m: 月份，两位数表示（01到12） + - %d: 一个月中的日子，两位数表示（01到31） + - %H: 小时，24小时制，两位数表示（00到23） + - %M: 分钟，两位数表示（00到59） + - %S: 秒，两位数表示（00到59） + +- **csv_ts_interval**：字符串类型，设置生成的 csv 文件名称中时间段间隔，支持天、小时、分钟、秒级间隔，如 1d/2h/30m/40s，默认值为 1d 。 + +- **csv_output_header**：字符串类型，设置生成的 csv 文件是否包含列头描述，默认值为 yes 。 + +- **csv_tbname_alias**：字符串类型，设置 csv 文件列头描述中 tbname 字段的别名，默认值为 device_id 。 + +- **csv_compress_level**：字符串类型，设置生成 csv 编码数据并自动压缩成 gzip 格式文件的压缩等级。此过程直接编码并压缩，而非先生成 csv 文件再压缩。可选值为： + - none：不压缩 + - fast：gzip 1级压缩 + - balance：gzip 6级压缩 + - best：gzip 9级压缩 #### 标签列与数据列 @@ -383,6 +410,17 @@ interval 控制休眠时间，避免持续查询慢查询消耗 CPU，单位为 +### 生成 CSV 文件 JSON 示例 + +
+csv-export.json + +```json +{{#include /TDengine/tools/taos-tools/example/csv-export.json}} +``` + +
+ 查看更多 json 配置文件示例可 [点击这里](https://github.com/taosdata/TDengine/tree/main/tools/taos-tools/example) ## 输出性能指标 diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 9c8e2ebb20..a8959d1d22 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -65,7 +65,6 @@ ASIN(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - #### ATAN ```sql @@ -84,7 +83,6 @@ ATAN(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - #### CEIL ```sql @@ -121,6 +119,66 @@ COS(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 +#### DEGREES + +```sql +DEGREES(expr) +``` + +**功能说明**：计算指定参数由弧度值转为角度后的值。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，则返回 NULL。 +- degree = radian * 180 / π。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select degrees(PI()); + degrees(pi()) | +============================ + 180.000000000000000 | +``` + +#### EXP + +```sql +EXP(expr) +``` +**功能说明**：返回 e（自然对数的底）的指定乘方后的值。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select exp(2); + exp(2) | +============================ + 7.389056098930650 | +``` + #### FLOOR ```sql @@ -128,7 +186,69 @@ FLOOR(expr) ``` **功能说明**：获得指定字段的向下取整数的结果。 - 其他使用说明参见 CEIL 函数描述。 + 其他使用说明参见 [CEIL](#ceil) 函数描述。 + +#### GREATEST +```sql +GREATEST(expr1, expr2[, expr]...) +``` + +**功能说明**：获得输入的所有参数中的最大值。该函数最小参数个数为 2 个。 + +**使用说明**：ver-3.3.6.0 + +**返回结果类型**：参考比较规则，比较类型即为最终返回类型。 + +**适用数据类型**： +- 数值类型：包括 bool 型，整型和浮点型 +- 字符串类型：包括 nchar 和 varchar 类型。 + +**比较规则**：以下规则描述了比较操作的转换方式： +- 如果有任何一个参数为 NULL，则比较结果为 NULL。 +- 如果比较操作中的所有参数都是字符串类型，按照字符串类型比较 +- 如果所有参数都是数值类型，则将它们作为数值类型进行比较。 +- 如果参数中既有字符串类型，也有数值类型，根据 compareAsStrInGreatest 配置项，统一作为字符串或者数值进行比较。默认按照字符串比较。 +- 在所有情况下，不同类型比较，比较类型会选择范围更大的类型进行比较，例如作为整数类型比较时，如果存在 BIGINT 类型，必定会选择 BIGINT 作为比较类型。 + +**相关配置项**：客户端配置，compareAsStrInGreatest 为 1 表示同时存在字符串类型和数值类型统一转为字符串比较，为 0 表示统一转为数值类型比较。默认为 1。 + +#### LEAST +```sql +LEAST(expr1, expr2[, expr]...) +``` + +**功能说明**：获得输入的所有参数中的最小值。其余部分说明同 [GREATEST](#greatest) 函数。 + +#### LN + +```sql +LN(expr) +``` + +**功能说明**：返回指定参数的自然对数。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 如果 `epxr` 小于等于 0，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select ln(10); + ln(10) | +============================ + 2.302585092994046 | +``` #### LOG @@ -148,6 +268,71 @@ LOG(expr1[, expr2]) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 +#### MOD + +```sql +MOD(expr1, expr2) +``` + +**功能说明**：计算 expr1 % expr2 的结果。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr2` 为 0 则返回 NULL。 +- 如果 `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +``` sql +taos> select mod(10,3); + mod(10,3) | +============================ + 1.000000000000000 | + +taos> select mod(1,0); + mod(1,0) | +============================ + NULL | +``` + +#### PI + +```sql +PI() +``` + +**功能说明**：返回圆周率 π 的值。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：无。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- π ≈ 3.141592653589793。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select pi(); + pi() | +============================ + 3.141592653589793 | +``` #### POW @@ -167,8 +352,85 @@ POW(expr1, expr2) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 +#### RADIANS + +```sql +RADIANS(expr) +``` + +**功能说明**：计算指定参数由角度值转为弧度后的值。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，则返回 NULL。 +- radian = degree * π / 180。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select radians(180); + radians(180) | +============================ + 3.141592653589793 | +``` + +#### RAND + +```sql +RAND([seed]) +``` + +**功能说明**：返回一个从0到1均匀分布的随机数。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**： +- `seed`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果指定了 `seed` 值，那么将会用指定的 `seed` 作为随机种子，确保生成的随机数序列具有确定性。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +``` sql +taos> select rand(); + rand() | +============================ + 0.202092426923147 | + +taos> select rand(); + rand() | +============================ + 0.131537788143166 | + +taos> select rand(1); + rand(1) | +============================ + 0.000007826369259 | + +taos> select rand(1); + rand(1) | +============================ + 0.000007826369259 | +``` #### ROUND + ```sql ROUND(expr[, digits]) ``` @@ -208,6 +470,49 @@ taos> select round(8888.88,-1); 8890.000000000000000 | ``` +#### SIGN + +```sql +SIGN(expr) +``` + +**功能说明**：返回指定参数的符号。 + +**版本**：v3.3.3.0 + +**返回结果类型**：与指定字段的原始数据类型一致。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为负，返回 -1。 +- 如果 `expr` 为正，返回 1。 +- 如果 `expr` 为 0，返回 0。 +- 如果 `expr` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select sign(-1); + sign(-1) | +======================== + -1 | + +taos> select sign(1); + sign(1) | +======================== + 1 | + +taos> select sign(0); + sign(0) | +======================== + 0 | +``` + #### SIN ```sql @@ -262,36 +567,8 @@ TAN(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -#### PI -```sql -PI() -``` - -**功能说明**：返回圆周率 π 的值。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：无。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- π ≈ 3.141592653589793。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select pi(); - pi() | -============================ - 3.141592653589793 | -``` - ##### TRUNCATE + ```sql TRUNCATE(expr, digits) ``` @@ -332,249 +609,88 @@ taos> select truncate(8888.88, -1); 8880.000000000000000 | ``` -#### EXP -```sql -EXP(expr) -``` -**功能说明**：返回 e（自然对数的底）的指定乘方后的值。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，返回 NULL。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select exp(2); - exp(2) | -============================ - 7.389056098930650 | -``` - -#### LN -```sql -LN(expr) -``` - -**功能说明**：返回指定参数的自然对数。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，返回 NULL。 -- 如果 `epxr` 小于等于 0，返回 NULL。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select ln(10); - ln(10) | -============================ - 2.302585092994046 | -``` - -#### MOD -```sql -MOD(expr1, expr2) -``` - -**功能说明**：计算 expr1 % expr2 的结果。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr2` 为 0 则返回 NULL。 -- 如果 `expr1` 或 `expr2` 为 NULL，返回 NULL。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -``` sql -taos> select mod(10,3); - mod(10,3) | -============================ - 1.000000000000000 | - -taos> select mod(1,0); - mod(1,0) | -============================ - NULL | -``` - -#### RAND -```sql -RAND([seed]) -``` - -**功能说明**：返回一个从0到1均匀分布的随机数。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**： -- `seed`：INTEGER。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果指定了 `seed` 值，那么将会用指定的 `seed` 作为随机种子，确保生成的随机数序列具有确定性。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -``` sql -taos> select rand(); - rand() | -============================ - 0.202092426923147 | - -taos> select rand(); - rand() | -============================ - 0.131537788143166 | - -taos> select rand(1); - rand(1) | -============================ - 0.000007826369259 | - -taos> select rand(1); - rand(1) | -============================ - 0.000007826369259 | -``` - -#### SIGN -```sql -SIGN(expr) -``` - -**功能说明**：返回指定参数的符号。 - -**版本**：v3.3.3.0 - -**返回结果类型**：与指定字段的原始数据类型一致。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为负，返回 -1。 -- 如果 `expr` 为正，返回 1。 -- 如果 `expr` 为 0，返回 0。 -- 如果 `expr` 为 NULL，返回 NULL。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select sign(-1); - sign(-1) | -======================== - -1 | - -taos> select sign(1); - sign(1) | -======================== - 1 | - -taos> select sign(0); - sign(0) | -======================== - 0 | -``` - -#### DEGREES -```sql -DEGREES(expr) -``` - -**功能说明**：计算指定参数由弧度值转为角度后的值。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，则返回 NULL。 -- degree = radian * 180 / π。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select degrees(PI()); - degrees(pi()) | -============================ - 180.000000000000000 | -``` - -#### RADIANS -```sql -RADIANS(expr) -``` - -**功能说明**：计算指定参数由角度值转为弧度后的值。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，则返回 NULL。 -- radian = degree * π / 180。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select radians(180); - radians(180) | -============================ - 3.141592653589793 | -``` ### 字符串函数 字符串函数的输入参数为字符串类型，返回结果为数值类型或字符串类型。 +#### ASCII + +```sql +ASCII(expr) +``` + +**功能说明**：返回字符串第一个字符的 ASCII 码。 + +**版本**：v3.3.3.0 + +**返回结果数据类型**：BIGINT。 + +**适用数据类型**：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 如果 `expr` 的第一个字符为多字节字符，只会返回该字符第一个字节的值对应的 ASCII 码。 + +**举例**： +```sql +taos> select ascii('testascii'); + ascii('testascii') | +===================== + 116 | +``` + +#### CHAR + +```sql +CHAR(expr1 [, expr2] [, epxr3] ...) +``` + +**功能说明**：将输入参数当作整数，并返回这些整数在 ASCII 编码中对应的字符。 + +**版本**：v3.3.3.0 + +**返回结果类型**：VARCHAR。 + +**适用数据类型**：整数类型,VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 输入的值超过 255 会被转化成多字节的结果，如 `CHAR(256)` 等同于 `CHAR(1,0)`、`CHAR(256 * 256)` 等同于 `CHAR(1,0,0)`。 +- 输入参数的 NULL 值会被跳过。 +- 输入参数若为字符串类型，会将其转换为数值类型处理。 +- 若输入的参数对应的字符为不可打印字符，返回值中仍有该参数对应的字符，但是可能无法显示出来。 +- 输入参数的个数上限为 2^31 - 1 个。 + +**举例**： +```sql +taos> select char(77); + char(77) | +=========== + M | + +taos> select char(77,77); + char(77,77) | +============== + MM | + +taos> select char(77 * 256 + 77); + char(77 * 256 + 77) | +====================== + MM | + +taos> select char(77,NULL,77); + char(77,null,77) | +=================== + MM | +``` + #### CHAR_LENGTH ```sql @@ -628,7 +744,6 @@ CONCAT(expr1, expr2 [, expr] ... ) **适用于**：表和超级表。 - #### CONCAT_WS ```sql @@ -645,7 +760,6 @@ CONCAT_WS(separator_expr, expr1, expr2 [, expr] ...) **适用于**：表和超级表。 - #### LENGTH ```sql @@ -662,7 +776,6 @@ LENGTH(expr) **适用于**：表和超级表。 - #### LOWER ```sql @@ -696,6 +809,120 @@ LTRIM(expr) **适用于**：表和超级表。 +#### POSITION + +```sql +POSITION(expr1 IN expr2) +``` + +**功能说明**：计算字符串 `expr1` 在字符串 `expr2` 中的位置。 + +**版本**：v3.3.3.0 + +**返回结果类型**：BIGINT。 + +**适用数据类型**： +- `expr1`：VARCHAR、NCHAR。 +- `expr2`：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 若 `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- 若 `expr1` 在 `expr2` 中不存在，返回 0。 +- 若 `expr1` 为空串，认为 `expr1` 在 `expr2` 中总能匹配成功，返回 1。 +- 返回的位置是 1-base 的。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select position('a' in 'cba'); + position('a' in 'cba') | +========================= + 3 | + + +taos> select position('' in 'cba'); + position('' in 'cba') | +======================== + 1 | + +taos> select position('d' in 'cba'); + position('d' in 'cba') | +========================= + 0 | +``` + +#### REPEAT + +```sql +REPEAT(expr, count) +``` +**功能说明**：返回将字符串重复指定次数得到的字符串。 + +**版本**：v3.3.3.0 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR、NCHAR。 +- `count`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 若 `count < 1`，返回空串。 +- 若 `expr` 或 `count` 为 NULL，返回 NULL。 + +**举例**： +```sql +taos> select repeat('abc',5); + repeat('abc',5) | +============================ + abcabcabcabcabc | + +taos> select repeat('abc',-1); + repeat('abc',-1) | +=================== + | +``` + +#### REPLACE + +```sql +REPLACE(expr, from_str, to_str) +``` +**功能说明**：将字符串中的 `from_str` 全部替换为 `to_str`。 + +**版本**：v3.3.3.0 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR、NCHAR。 +- `from_str`：VARCHAR、NCHAR。 +- `to_str`：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 该函数是大小写敏感的。 +- 任意参数为 NULL，返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select replace('aabbccAABBCC', 'AA', 'DD'); + replace('aabbccAABBCC', 'AA', 'DD') | +====================================== + aabbccDDBBCC | +``` #### RTRIM @@ -713,62 +940,8 @@ RTRIM(expr) **适用于**：表和超级表。 -#### TRIM -```sql -TRIM([{LEADING | TRAILING | BOTH} [remstr] FROM] expr) -TRIM([remstr FROM] expr) -``` - -**功能说明**：返回去掉了所有 remstr 前缀或后缀的字符串 epxr。 - -**版本**：v3.3.3.0 - -**返回结果类型**：与输入字段 epxr 的原始类型相同。 - -**适用数据类型**： -- remstr：VARCHAR、NCHAR。 -- epxr：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 第一个可选变量 [LEADING | BOTH | TRAILING] 指定要剪裁字符串的哪一侧： - - LEADING 将移除字符串开头的指定字符。 - - TRAILING 将移除字符串末尾的指定字符。 - - BOTH（默认值）将移除字符串开头和末尾的指定字符。 -- 第二个可选变量[remstr]指定要裁剪掉的字符串： - - 如果不指定 remstr，默认裁剪空格。 - - remstr 可以指定多个字符，如 trim('ab' from 'abacd')，此时会将 'ab' 看做一个整体来裁剪，得到裁剪结果 'acd'。 -- 若 expr 为 NULL，返回 NULL。 -- 该函数是多字节安全的。 - -**举例**： -```sql -taos> select trim(' a '); - trim(' a ') | -============================= - a | - -taos> select trim(leading from ' a '); - trim(leading from ' a ') | -========================================== - a | - - -taos> select trim(leading 'b' from 'bbbbbbbba '); - trim(leading 'b' from 'bbbbbbbba ') | -============================================== - a | - -taos> select trim(both 'b' from 'bbbbbabbbbbb'); - trim(both 'b' from 'bbbbbabbbbbb') | -===================================== - a | -``` - #### SUBSTRING/SUBSTR + ```sql SUBSTRING/SUBSTR(expr, pos [, len]) SUBSTRING/SUBSTR(expr FROM pos [FOR len]) @@ -826,6 +999,7 @@ taos> select substring('tdengine', -3,-3); ``` #### SUBSTRING_INDEX + ```sql SUBSTRING_INDEX(expr, delim, count) ``` @@ -864,6 +1038,62 @@ taos> select substring_index('www.taosdata.com','.',-2); taosdata.com | ``` +#### TRIM + +```sql +TRIM([{LEADING | TRAILING | BOTH} [remstr] FROM] expr) +TRIM([remstr FROM] expr) +``` + +**功能说明**：返回去掉了所有 remstr 前缀或后缀的字符串 epxr。 + +**版本**：v3.3.3.0 + +**返回结果类型**：与输入字段 epxr 的原始类型相同。 + +**适用数据类型**： +- remstr：VARCHAR、NCHAR。 +- epxr：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 第一个可选变量[LEADING | BOTH | TRAILING]指定要剪裁字符串的哪一侧： + - LEADING 将移除字符串开头的指定字符。 + - TRAILING 将移除字符串末尾的指定字符。 + - BOTH（默认值）将移除字符串开头和末尾的指定字符。 +- 第二个可选变量[remstr]指定要裁剪掉的字符串： + - 如果不指定 remstr，默认裁剪空格。 + - remstr 可以指定多个字符，如trim('ab' from 'abacd')，此时会将 'ab' 看做一个整体来裁剪，得到裁剪结果 'acd'。 +- 若 expr 为 NULL，返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select trim(' a '); + trim(' a ') | +============================= + a | + +taos> select trim(leading from ' a '); + trim(leading from ' a ') | +========================================== + a | + + +taos> select trim(leading 'b' from 'bbbbbbbba '); + trim(leading 'b' from 'bbbbbbbba ') | +============================================== + a | + +taos> select trim(both 'b' from 'bbbbbabbbbbb'); + trim(both 'b' from 'bbbbbabbbbbb') | +===================================== + a | +``` + #### UPPER ```sql @@ -880,193 +1110,6 @@ UPPER(expr) **适用于**：表和超级表。 -#### CHAR -```sql -CHAR(expr1 [, expr2] [, epxr3] ...) -``` - -**功能说明**：将输入参数当作整数，并返回这些整数在 ASCII 编码中对应的字符。 - -**版本**：v3.3.3.0 - -**返回结果类型**：VARCHAR。 - -**适用数据类型**：整数类型,VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 输入的值超过 255 会被转化成多字节的结果，如 `CHAR(256)` 等同于 `CHAR(1,0)`、`CHAR(256 * 256)` 等同于 `CHAR(1,0,0)`。 -- 输入参数的 NULL 值会被跳过。 -- 输入参数若为字符串类型，会将其转换为数值类型处理。 -- 若输入的参数对应的字符为不可打印字符，返回值中仍有该参数对应的字符，但是可能无法显示出来。 -- 输入参数的个数上限为 2^31 - 1 个。 - -**举例**： -```sql -taos> select char(77); - char(77) | -=========== - M | - -taos> select char(77,77); - char(77,77) | -============== - MM | - -taos> select char(77 * 256 + 77); - char(77 * 256 + 77) | -====================== - MM | - -taos> select char(77,NULL,77); - char(77,null,77) | -=================== - MM | -``` - -#### ASCII -```sql -ASCII(expr) -``` - -**功能说明**：返回字符串第一个字符的 ASCII 码。 - -**版本**：v3.3.3.0 - -**返回结果数据类型**：BIGINT。 - -**适用数据类型**：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，返回 NULL。 -- 如果 `expr` 的第一个字符为多字节字符，只会返回该字符第一个字节的值对应的 ASCII 码。 - -**举例**： -```sql -taos> select ascii('testascii'); - ascii('testascii') | -===================== - 116 | -``` - -#### POSITION -```sql -POSITION(expr1 IN expr2) -``` - -**功能说明**：计算字符串 `expr1` 在字符串 `expr2` 中的位置。 - -**版本**：v3.3.3.0 - -**返回结果类型**：BIGINT。 - -**适用数据类型**： -- `expr1`：VARCHAR、NCHAR。 -- `expr2`：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 若 `expr1` 或 `expr2` 为 NULL，返回 NULL。 -- 若 `expr1` 在 `expr2` 中不存在，返回 0。 -- 若 `expr1` 为空串，认为 `expr1` 在 `expr2` 中总能匹配成功，返回 1。 -- 返回的位置是 1-base 的。 -- 该函数是多字节安全的。 - -**举例**： -```sql -taos> select position('a' in 'cba'); - position('a' in 'cba') | -========================= - 3 | - - -taos> select position('' in 'cba'); - position('' in 'cba') | -======================== - 1 | - -taos> select position('d' in 'cba'); - position('d' in 'cba') | -========================= - 0 | -``` - -#### REPLACE -```sql -REPLACE(expr, from_str, to_str) -``` -**功能说明**：将字符串中的 `from_str` 全部替换为 `to_str`。 - -**版本**：v3.3.3.0 - -**返回结果类型**：与输入字段 `expr` 的原始类型相同。 - -**适用数据类型**： -- `expr`：VARCHAR、NCHAR。 -- `from_str`：VARCHAR、NCHAR。 -- `to_str`：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 该函数是大小写敏感的。 -- 任意参数为 NULL，返回 NULL。 -- 该函数是多字节安全的。 - -**举例**： -```sql -taos> select replace('aabbccAABBCC', 'AA', 'DD'); - replace('aabbccAABBCC', 'AA', 'DD') | -====================================== - aabbccDDBBCC | -``` - -#### REPEAT -```sql -REPEAT(expr, count) -``` -**功能说明**：返回将字符串重复指定次数得到的字符串。 - -**版本**：v3.3.3.0 - -**返回结果类型**：与输入字段 `expr` 的原始类型相同。 - -**适用数据类型**： -- `expr`：VARCHAR、NCHAR。 -- `count`：INTEGER。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 若 `count < 1`，返回空串。 -- 若 `expr` 或 `count` 为 NULL，返回 NULL。 - -**举例**： -```sql -taos> select repeat('abc',5); - repeat('abc',5) | -============================ - abcabcabcabcabc | - -taos> select repeat('abc',-1); - repeat('abc',-1) | -=================== - | -``` ### 转换函数 转换函数将值从一种数据类型转换为另一种数据类型。 @@ -1095,72 +1138,6 @@ CAST(expr AS type_name) - 转换到数值类型时，数值大于 type_name 可表示的范围时，则会溢出，但不会报错。 - 转换到字符串类型时，如果转换后长度超过 type_name 中指定的长度，则会截断，但不会报错。 -#### TO_ISO8601 - -```sql -TO_ISO8601(expr [, timezone]) -``` - -**功能说明**：将时间戳转换成为 ISO8601 标准的日期时间格式，并附加时区信息。timezone 参数允许用户为输出结果指定附带任意时区信息。如果 timezone 参数省略，输出结果则附带当前客户端的系统时区信息。 - -**返回结果数据类型**：VARCHAR 类型。 - -**适用数据类型**：INTEGER、TIMESTAMP。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： - -- timezone 参数允许输入的时区格式为：[z/Z, +/-hhmm, +/-hh, +/-hh:mm]。例如，TO_ISO8601(1, "+00:00")。 -- 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒. - - -#### TO_JSON - -```sql -TO_JSON(str_literal) -``` - -**功能说明**：将字符串常量转换为 JSON 类型。 - -**返回结果数据类型**：JSON。 - -**适用数据类型**：JSON 字符串，形如 '\{ "literal" : literal }'。'\{}'表示空值。键必须为字符串字面量，值可以为数值字面量、字符串字面量、布尔字面量或空值字面量。str_literal 中不支持转义符。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - - -#### TO_UNIXTIMESTAMP - -```sql -TO_UNIXTIMESTAMP(expr [, return_timestamp]) - -return_timestamp: { - 0 - | 1 -} -``` - -**功能说明**：将日期时间格式的字符串转换成为时间戳。 - -**返回结果数据类型**：BIGINT、TIMESTAMP。 - -**应用字段**：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： - -- 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 NULL。 -- 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 -- return_timestamp 指定函数返回值是否为时间戳类型，设置为 1 时返回 TIMESTAMP 类型，设置为 0 时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。 - #### TO_CHAR ```sql @@ -1222,6 +1199,43 @@ TO_CHAR(ts, format_str_literal) - 推荐在时间格式中带时区信息，如果不带则默认输出的时区为服务端或客户端所配置的时区。 - 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒。 +#### TO_ISO8601 + +```sql +TO_ISO8601(expr [, timezone]) +``` + +**功能说明**：将时间戳转换成为 ISO8601 标准的日期时间格式，并附加时区信息。timezone 参数允许用户为输出结果指定附带任意时区信息。如果 timezone 参数省略，输出结果则附带当前客户端的系统时区信息。 + +**返回结果数据类型**：VARCHAR 类型。 + +**适用数据类型**：INTEGER、TIMESTAMP。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： + +- timezone 参数允许输入的时区格式为：[z/Z, +/-hhmm, +/-hh, +/-hh:mm]。例如，TO_ISO8601(1, "+00:00")。 +- 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒. + +#### TO_JSON + +```sql +TO_JSON(str_literal) +``` + +**功能说明**：将字符串常量转换为 JSON 类型。 + +**返回结果数据类型**：JSON。 + +**适用数据类型**：JSON 字符串，形如 '\{ "literal" : literal }'。'\{}'表示空值。键必须为字符串字面量，值可以为数值字面量、字符串字面量、布尔字面量或空值字面量。str_literal 中不支持转义符。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + #### TO_TIMESTAMP ```sql @@ -1252,6 +1266,32 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal) - `to_timestamp` 转换具有一定的容错机制，在格式串和时间戳串不完全对应时，有时也可转换，如 `to_timestamp('200101/2', 'yyyyMM1/dd')`，格式串中多出来的1会被丢弃。格式串与时间戳串中多余的空格字符(空格、tab 等)也会被自动忽略，如 `to_timestamp(' 23 年 - 1 月 - 01 日 ', 'yy 年-MM月-dd日')` 可以被成功转换。虽然 `MM` 等字段需要两个数字对应(只有一位时前面补 0), 在 `to_timestamp` 时，一个数字也可以成功转换。 - 输出时间戳的精度与查询表的精度相同，若查询未指定表，则输出精度为毫秒，如 `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')` 的输出将会把微秒和纳秒进行截断、如果指定一张纳秒表，那么就不会发生截断，如 `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`。 +#### TO_UNIXTIMESTAMP + +```sql +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} +``` + +**功能说明**：将日期时间格式的字符串转换成为时间戳。 + +**返回结果数据类型**：BIGINT、TIMESTAMP。 + +**应用字段**：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： + +- 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 NULL。 +- 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 +- return_timestamp 指定函数返回值是否为时间戳类型，设置为 1 时返回 TIMESTAMP 类型，设置为 0 时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。 ### 时间和日期函数 @@ -1259,6 +1299,35 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal) 所有返回当前时间的函数，如 NOW、TODAY 和 TIMEZONE，在一条 SQL 语句中不论出现多少次都只会被计算一次。 +#### DAYOFWEEK +```sql +DAYOFWEEK(expr) +``` +**功能说明**：返回输入日期是周几。 + +**版本**：v3.3.3.0 + +**返回结果类型**：BIGINT。 + +**适用数据类型**：表示时间戳的 BIGINT、TIMESTAMP 类型，或符合 ISO8601/RFC3339 标准的日期时间格式的 VARCHAR、NCHAR 类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 返回值 1 代表周日，2 代表周一 ... 7 代表周六 +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒。 + +**举例**： +```sql +taos> select dayofweek('2000-01-01'); + dayofweek('2000-01-01') | +========================== + 7 | +``` + #### NOW ```sql @@ -1281,7 +1350,6 @@ NOW() b(纳秒)、u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 - #### TIMEDIFF ```sql @@ -1355,7 +1423,6 @@ use_current_timezone: { - 当将时间值截断到一周（1w）时，timetruncate 的计算是基于 Unix 时间戳（1970年1月1日00:00:00 UTC）进行的。Unix 时间戳始于星期四， 因此所有截断后的日期都是星期四。 - #### TIMEZONE ```sql @@ -1370,7 +1437,6 @@ TIMEZONE() **适用于**：表和超级表。 - #### TODAY ```sql @@ -1392,6 +1458,7 @@ TODAY() - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 #### WEEK + ```sql WEEK(expr [, mode]) ``` @@ -1454,35 +1521,6 @@ taos> select week('2000-01-01',3); 52 | ``` -#### WEEKOFYEAR -```sql -WEEKOFYEAR(expr) -``` -**功能说明**：返回输入日期的周数。 - -**版本**：v3.3.3.0 - -**返回结果类型**：BIGINT。 - -**适用数据类型**：表示时间戳的 BIGINT、TIMESTAMP 类型，或符合 ISO8601/RFC3339 标准的日期时间格式的 VARCHAR、NCHAR 类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 等同于`WEEK(expr, 3)`，即在每周第一天是周一，返回值范围为 1-53，第一个包含四天及以上的周为第 1 周的条件下判断输入日期的周数。 -- 若 `expr` 为 NULL，返回 NULL。 -- 输入时间戳的精度由所查询表的精度确定，未未指定表，则精度为毫秒。 - -**举例**： -```sql -taos> select weekofyear('2000-01-01'); - weekofyear('2000-01-01') | -=========================== - 52 | -``` - #### WEEKDAY ```sql WEEKDAY(expr) @@ -1512,11 +1550,12 @@ taos> select weekday('2000-01-01'); 5 | ``` -#### DAYOFWEEK +#### WEEKOFYEAR + ```sql -DAYOFWEEK(expr) +WEEKOFYEAR(expr) ``` -**功能说明**：返回输入日期是周几。 +**功能说明**：返回输入日期的周数。 **版本**：v3.3.3.0 @@ -1529,19 +1568,18 @@ DAYOFWEEK(expr) **适用于**：表和超级表。 **使用说明**： -- 返回值 1 代表周日，2 代表周一 ... 7 代表周六。 +- 等同于`WEEK(expr, 3)`，即在每周第一天是周一，返回值范围为 1 - 53，第一个包含四天及以上的周为第 1 周的条件下判断输入日期的周数。 - 若 `expr` 为 NULL，返回 NULL。 -- 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒。 - +- 输入时间戳的精度由所查询表的精度确定，未指定表，则精度为毫秒。 + **举例**： ```sql -taos> select dayofweek('2000-01-01'); - dayofweek('2000-01-01') | -========================== - 7 | +taos> select weekofyear('2000-01-01'); + weekofyear('2000-01-01') | +=========================== + 52 | ``` - ## 聚合函数 聚合函数为查询结果集的每一个分组返回单个结果行。可以由 GROUP BY 或窗口切分子句指定分组，如果没有，则整个查询结果集视为一个分组。 @@ -1586,7 +1624,6 @@ AVG(expr) **适用于**：表和超级表。 - ### COUNT ```sql @@ -1606,7 +1643,6 @@ COUNT({* | expr}) - 可以使用星号 (\*) 来替代具体的字段，使用星号 (\*) 返回全部记录数量。 - 如果统计字段是具体的列，则返回该列中非 NULL 值的记录数量。 - ### ELAPSED ```sql @@ -1632,6 +1668,50 @@ ELAPSED(ts_primary_key [, time_unit]) - 对于嵌套查询，仅当内层查询会输出隐式时间戳列时有效。例如 `select elapsed(ts) from (select diff(value) from sub1)` 语句，diff 函数会让内层查询输出隐式时间戳列，此为主键列，可以用于 elapsed 函数的第一个参数。相反，例如 `select elapsed(ts) from (select * from sub1)` 语句，ts 列输出到外层时已经没有了主键列的含义，无法使用 elapsed 函数。此外，elapsed 函数作为一个与时间线强依赖的函数，形如 `select elapsed(ts) from (select diff(value) from st group by tbname)` 尽管会返回一条计算结果，但并无实际意义，这种用法后续也将被限制。 - 不支持与 leastsquares、diff、derivative、top、bottom、last_row、interp 等函数混合使用。 +### HISTOGRAM + +```sql +HISTOGRAM(expr，bin_type, bin_description, normalized) +``` + +**功能说明**：统计数据按照用户指定区间的分布。 + +**返回结果类型**：如归一化参数 normalized 设置为 1，返回结果为 DOUBLE 类型，否则为 BIGINT 类型。 + +**适用数据类型**：数值型字段。 + +**适用于**：表和超级表。 + +**详细说明**： +- bin_type 用户指定的分桶类型，有效输入类型为 "user_input"、"linear_bin"、"log_bin"。 +- bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)： + - "user_input": "[1, 3, 5, 7]" + 用户指定 bin 的具体数值。 + + - "linear_bin": "\{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" + "start" 表示数据起始点，"width" 表示每次 bin 偏移量，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， + 生成区间为[-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]。 + + - "log_bin": "\{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" + "start" 表示数据起始点，"factor" 表示按指数递增的因子，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， + 生成区间为[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]。 +- normalized 是否将返回结果归一化到 0~1 之间。有效输入为 0 和 1。 + +### HYPERLOGLOG + +```sql +HYPERLOGLOG(expr) +``` + +**功能说明**： + - 采用 hyperloglog 算法，返回某列的基数。该算法在数据量很大的情况下，可以明显降低内存的占用，求出来的基数是个估算值，标准误差（标准误差是多次实验，每次的平均数的标准差，不是与真实结果的误差）为 0.81%。 + - 在数据量较少的时候该算法不是很准确，可以使用 `select count(data) from (select unique(col) as data from table)` 的方法。 + +**返回结果类型**：INTEGER。 + +**适用数据类型**：任何类型。 + +**适用于**：表和超级表。 ### LEASTSQUARES @@ -1648,6 +1728,26 @@ LEASTSQUARES(expr, start_val, step_val) **适用于**：表。 +### PERCENTILE + +```sql +PERCENTILE(expr, p [, p1] ... ) +``` + +**功能说明**：统计表中某列的值百分比分位数。 + +**返回数据类型**：该函数最小参数个数为 2 个，最大参数个数为 11 个。可以最多同时返回 10 个百分比分位数。当参数个数为 2 时，返回一个分位数，类型为DOUBLE，当参数个数大于 2 时，返回类型为VARCHAR，格式为包含多个返回值的JSON数组。 + +**应用字段**：数值类型。 + +**适用于**：表。 + +**使用说明**： + +- *P* 值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX; +- 同时计算针对同一列的多个分位数时，建议使用一个 PERCENTILE 函数和多个参数的方式，能很大程度上降低查询的响应时间。 + 比如，使用查询 `SELECT percentile(col, 90, 95, 99) FROM table`，性能会优于 `SELECT percentile(col, 90), percentile(col, 95), percentile(col, 99) from table`。 + ### SPREAD ```sql @@ -1696,7 +1796,23 @@ taos> select stddev_pop(id) from test_stddev; ============================ 1.414213562373095 | ``` + +### SUM + +```sql +SUM(expr) +``` + +**功能说明**：统计表/超级表中某列的和。 + +**返回数据类型**：DOUBLE、BIGINT。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + ### VAR_POP + ```sql VAR_POP(expr) ``` @@ -1727,88 +1843,6 @@ taos> select var_pop(id) from test_var; ============================ 2.000000000000000 | ``` -### SUM - -```sql -SUM(expr) -``` - -**功能说明**：统计表/超级表中某列的和。 - -**返回数据类型**：DOUBLE、BIGINT。 - -**适用数据类型**：数值类型。 - -**适用于**：表和超级表。 - - -### HYPERLOGLOG - -```sql -HYPERLOGLOG(expr) -``` - -**功能说明**： - - 采用 hyperloglog 算法，返回某列的基数。该算法在数据量很大的情况下，可以明显降低内存的占用，求出来的基数是个估算值，标准误差（标准误差是多次实验，每次的平均数的标准差，不是与真实结果的误差）为 0.81%。 - - 在数据量较少的时候该算法不是很准确，可以使用 `select count(data) from (select unique(col) as data from table)` 的方法。 - -**返回结果类型**：INTEGER。 - -**适用数据类型**：任何类型。 - -**适用于**：表和超级表。 - - -### HISTOGRAM - -```sql -HISTOGRAM(expr，bin_type, bin_description, normalized) -``` - -**功能说明**：统计数据按照用户指定区间的分布。 - -**返回结果类型**：如归一化参数 normalized 设置为 1，返回结果为 DOUBLE 类型，否则为 BIGINT 类型。 - -**适用数据类型**：数值型字段。 - -**适用于**：表和超级表。 - -**详细说明**： -- bin_type 用户指定的分桶类型，有效输入类型为 "user_input"、"linear_bin"、"log_bin"。 -- bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)： - - "user_input": "[1, 3, 5, 7]" - 用户指定 bin 的具体数值。 - - - "linear_bin": "\{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" - "start" 表示数据起始点，"width" 表示每次 bin 偏移量，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， - 生成区间为[-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]。 - - - "log_bin": "\{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" - "start" 表示数据起始点，"factor" 表示按指数递增的因子，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， - 生成区间为[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]。 -- normalized 是否将返回结果归一化到 0~1 之间。有效输入为 0 和 1。 - - -### PERCENTILE - -```sql -PERCENTILE(expr, p [, p1] ... ) -``` - -**功能说明**：统计表中某列的值百分比分位数。 - -**返回数据类型**：该函数最小参数个数为 2 个，最大参数个数为 11 个。可以最多同时返回 10 个百分比分位数。当参数个数为 2 时，返回一个分位数，类型为DOUBLE，当参数个数大于 2 时，返回类型为VARCHAR，格式为包含多个返回值的JSON数组。 - -**应用字段**：数值类型。 - -**适用于**：表。 - -**使用说明**： - -- *P* 值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX; -- 同时计算针对同一列的多个分位数时，建议使用一个 PERCENTILE 函数和多个参数的方式，能很大程度上降低查询的响应时间。 - 比如，使用查询 `SELECT percentile(col, 90, 95, 99) FROM table`，性能会优于 `SELECT percentile(col, 90), percentile(col, 95), percentile(col, 99) from table`。 - ## 选择函数 @@ -1912,7 +1946,6 @@ LAST(expr) - 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 - 对于存在复合主键的表的查询，若最大时间戳的数据有多条，则只有对应的复合主键最大的数据被返回。 - ### LAST_ROW ```sql @@ -1981,7 +2014,6 @@ MODE(expr) **适用于**：表和超级表。 - ### SAMPLE ```sql @@ -1998,7 +2030,6 @@ SAMPLE(expr, k) **适用于**：表和超级表。 - ### TAIL ```sql @@ -2015,7 +2046,6 @@ TAIL(expr, k [, offset_rows]) **适用于**：表、超级表。 - ### TOP ```sql @@ -2093,11 +2123,9 @@ CSUM(expr) **适用于**：表和超级表。 **使用说明**： - - 不支持 +、-、*、/ 运算，如 csum(col1) + csum(col2)。 - 只能与聚合（Aggregation）函数一起使用。该函数可以应用在普通表和超级表上。 - ### DERIVATIVE ```sql @@ -2173,7 +2201,6 @@ IRATE(expr) **适用于**：表和超级表。 - ### MAVG ```sql @@ -2195,7 +2222,6 @@ MAVG(expr, k) - 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1); - 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用； - ### STATECOUNT ```sql @@ -2221,7 +2247,6 @@ STATECOUNT(expr, oper, val) - 不能和窗口操作一起使用，例如 `interval/state_window/session_window`。 - ### STATEDURATION ```sql @@ -2248,7 +2273,6 @@ STATEDURATION(expr, oper, val, unit) - 不能和窗口操作一起使用，例如 interval、state_window、session_window。 - ### TWA ```sql @@ -2263,18 +2287,8 @@ TWA(expr) **适用于**：表和超级表。 - ## 系统信息函数 -### DATABASE - -```sql -SELECT DATABASE(); -``` - -**说明**：返回当前登录的数据库。如果登录的时候没有指定默认数据库，且没有使用 USE 命令切换数据库，则返回 NULL。 - - ### CLIENT_VERSION ```sql @@ -2283,13 +2297,21 @@ SELECT CLIENT_VERSION(); **说明**：返回客户端版本。 -### SERVER_VERSION +### CURRENT_USER ```sql -SELECT SERVER_VERSION(); +SELECT CURRENT_USER(); ``` -**说明**：返回服务端版本。 +**说明**：获取当前用户。 + +### DATABASE + +```sql +SELECT DATABASE(); +``` + +**说明**：返回当前登录的数据库。如果登录的时候没有指定默认数据库，且没有使用 USE 命令切换数据库，则返回 NULL。 ### SERVER_STATUS @@ -2299,14 +2321,13 @@ SELECT SERVER_STATUS(); **说明**：检测服务端是否所有 dnode 都在线，如果是则返回成功，否则返回无法建立连接的错误。如果想要查询集群的状态，推荐使用 `SHOW CLUSTER ALIVE` 与 `SELECT SERVER_STATUS()` 不同，当集群中的部分节点不可用时，它不会返回错误，而是返回不同的状态码，详见：[SHOW CLUSTER ALIVE](https://docs.taosdata.com/reference/taos-sql/show/#show-cluster-alive) -### CURRENT_USER +### SERVER_VERSION ```sql -SELECT CURRENT_USER(); +SELECT SERVER_VERSION(); ``` -**说明**：获取当前用户。 - +**说明**：返回服务端版本。 ## Geometry 函数 @@ -2348,70 +2369,6 @@ ST_AsText(GEOMETRY geom) ### Geometry 关系函数 -#### ST_Intersects - -```sql -ST_Intersects(GEOMETRY geomA, GEOMETRY geomB) -``` - -##功能说明**：比较两个几何对象，并在它们相交时返回 true。 - -**返回值类型**：BOOL。 - -**适用数据类型**：GEOMETRY、GEOMETRY。 - -**适用表类型**：标准表和超表。 - -**使用说明**：如果两个几何对象有任何一个共享点，则它们相交。 - -#### ST_Equals - -```sql -ST_Equals(GEOMETRY geomA, GEOMETRY geomB) -``` - -**功能说明**：如果给定的几何对象是“空间相等”的，则返回 TRUE。 - -**返回值类型**：BOOL。 - -**适用数据类型**：GEOMETRY、GEOMETRY。 - -**适用表类型**：标准表和超表。 - -**使用说明**："空间相等"意味着 ST_Contains(A,B) = true 和 ST_Contains(B,A) = true，并且点的顺序可能不同，但表示相同的几何结构。 - -#### ST_Touches - -```sql -ST_Touches(GEOMETRY geomA, GEOMETRY geomB) -``` - -**功能说明**：如果 A 和 B 相交，但它们的内部不相交，则返回 TRUE。 - -**返回值类型**：BOOL。 - -**适用数据类型**：GEOMETRY、GEOMETRY。 - -**适用表类型**：标准表和超表。 - -**使用说明**：A 和 B 至少有一个公共点，并且这些公共点位于至少一个边界中。对于点/点输入，关系始终为 FALSE，因为点没有边界。 - -#### ST_Covers - -```sql -ST_Covers(GEOMETRY geomA, GEOMETRY geomB) -``` - -**功能说明**：如果 B 中的每个点都位于几何形状 A 内部（与内部或边界相交），则返回 TRUE。 - -**返回值类型**：BOOL。 - -**适用数据类型**：GEOMETRY、GEOMETRY。 - -**适用表类型**：标准表和超表。 - -**使用说明**：A 包含 B 意味着 B 中的没有点位于 A 的外部（在外部）。 - #### ST_Contains ```sql @@ -2443,3 +2400,68 @@ ST_ContainsProperly(GEOMETRY geomA, GEOMETRY geomB) **适用表类型**：标准表和超表。 **使用说明**：B 的没有点位于 A 的边界或外部。 + +#### ST_Covers + +```sql +ST_Covers(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果 B 中的每个点都位于几何形状 A 内部（与内部或边界相交），则返回 TRUE。 + +**返回值类型**：BOOL。 + +**适用数据类型**：GEOMETRY、GEOMETRY。 + +**适用表类型**：标准表和超表。 + +**使用说明**：A 包含 B 意味着 B 中的没有点位于 A 的外部（在外部）。 + +#### ST_Equals + +```sql +ST_Equals(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果给定的几何对象是"空间相等"的，则返回 TRUE。 + +**返回值类型**：BOOL。 + +**适用数据类型**：GEOMETRY、GEOMETRY。 + +**适用表类型**：标准表和超表。 + +**使用说明**："空间相等"意味着 ST_Contains(A,B) = true 和 ST_Contains(B,A) = true，并且点的顺序可能不同，但表示相同的几何结构。 + + +#### ST_Intersects + +```sql +ST_Intersects(GEOMETRY geomA, GEOMETRY geomB) +``` + +##功能说明**：比较两个几何对象，并在它们相交时返回 true。 + +**返回值类型**：BOOL。 + +**适用数据类型**：GEOMETRY、GEOMETRY。 + +**适用表类型**：标准表和超表。 + +**使用说明**：如果两个几何对象有任何一个共享点，则它们相交。 + +#### ST_Touches + +```sql +ST_Touches(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果 A 和 B 相交，但它们的内部不相交，则返回 TRUE。 + +**返回值类型**：BOOL。 + +**适用数据类型**：GEOMETRY、GEOMETRY。 + +**适用表类型**：标准表和超表。 + +**使用说明**：A 和 B 至少有一个公共点，并且这些公共点位于至少一个边界中。对于点/点输入，关系始终为 FALSE，因为点没有边界。 diff --git a/docs/zh/27-train-faq/02-dst.md b/docs/zh/27-train-faq/02-dst.md new file mode 100644 index 0000000000..5c430fd42f --- /dev/null +++ b/docs/zh/27-train-faq/02-dst.md @@ -0,0 +1,291 @@ +--- +title: 夏令时使用指南 +description: TDengine 中关于夏令时使用问题的解释和建议 +--- + +## 背景 + +在时序数据库的使用中，有时会遇到使用夏令时的情况。我们将 TDengine 中使用夏令时的情况和问题进行分析说明，以便您在 TDengine 的使用中更加顺利。 + +## 定义 + +### 时区 + +时区是地球上使用相同标准时间的区域。由于地球的自转，为了保证各地的时间与当地的日出日落相协调，全球划分为多个时区。 + +### IANA 时区 + +IANA（Internet Assigned Numbers Authority）时区数据库，也称为 tz database，提供全球时区信息的标准参考。它是现代各类系统和软件处理时区相关操作的基础。 + +IANA 使用“区域/城市”格式（如 Europe/Berlin）来明确标识时区。 + +TDengine 在不同组件中均支持使用 IANA 时区（除 Windows taos.cfg 时区设置外）。 + +### 标准时间与当地时间 + +标准时间是根据地球上某个固定经线确定的时间。它为各个时区提供了一个统一的参考点。 + +- 格林尼治标准时间（GMT）：历史上使用的参考时间，位于 0° 经线。 +- 协调世界时（UTC）：现代的时间标准，类似于GMT，但更加精确。 + +标准时间与时区的关系如下： + +- 基准：标准时间（如 UTC）是时区设定的基准点。 +- 偏移量：不同时区通过相对于标准时间的偏移量来定义。例如，UTC+1 表示比 UTC 快 1 小时。 +- 区域划分：全球被划分为多个时区，每个时区使用一个或多个标准时间。 + +相对于标准时间，每个地区根据其所在时区设定其当地时间： + +- 时区偏移：当地时间等于标准时间加上该时区的偏移量。例如，UTC+2 表示比 UTC 时间快 2 小时。 +- 夏令时（DST）：某些地区在特定时间段调整当地时间，例如将时钟拨快一小时。详见下节。 + +### 夏令时 + +夏令时（Daylight Saving Time，DST）是一种通过将时间提前一小时，以充分利用日光、节约能源的制度。通常在春季开始，秋季结束。夏令时的具体开始和结束时间因地区而异。以下均以柏林时间为例，对夏令时和夏令时的影响做说明。 + +按照这个规则，可以看到： + +- 柏林当地时间 2024 年 03 月 31 日 02:00:00 到 03:00:00 （不含 03:00:00）之间的时间不存在（跳变）。 +- 柏林当地时间 2024 年 10 月 27 日 02:00:00 到 03:00:00 （不含 03:00:00）之间的时间出现了两次。 + +#### 夏令时与 IANA 时区数据库 + +- 记录规则：IANA 时区数据库详细记录了各地的夏令时规则，包括开始和结束的日期与时间。 +- 自动调整：许多操作系统和软件利用 IANA 数据库来自动处理夏令时的调整。 +- 历史变更：IANA 数据库还追踪历史上的夏令时变化，以确保准确性。 + +#### 夏令时与时间戳转换 + +- 时间戳转为当地时间是确定的。例如，1729990654 为柏林时间**夏令时** `2024-10-27 02:57:34`，1729994254 为柏林时间**冬令时** `2024-10-27 02:57:34`（这两个本地时间除时间偏移量外是一样的）。 +- 不指定时间偏移量时，当地时间转为时间戳是不确定的。夏令时跳过的时间不存在会造成无法转换成时间戳，如 **柏林时间** `2024-03-31 02:34:56` 不存在，所以无法转换为时间戳。夏令时结束时重复导致无法确定是哪个时间戳，如 `2024-10-27 02:57:34` 不指定时间偏移量无法确定 是 1729990654 还是 1729994254。指定时间偏移量才能确定时间戳，如 `2024-10-27 02:57:34 CEST(+02:00) `，指定了夏令时 `2024-10-27 02:57:34` 时间戳 1729990654 。 + +### RFC3339 时间格式 + +RFC 3339 是一种互联网时间格式标准，用于表示日期和时间。它基于 ISO 8601 标准，但更具体地规定了一些格式细节。 + +其格式如下： + +- 基本格式：`YYYY-MM-DDTHH:MM:SSZ` +- 时区表示： + - Z 表示协调世界时（UTC）。 + - 偏移量格式，例如 +02:00，表示与 UTC 的时差。 + +通过明确的时区偏移，RFC 3339 格式可以在全球范围内准确地解析和比较时间。 + +RFC 3339 的优势包括： + +- 标准化：提供统一的格式，方便跨系统数据交换。 +- 清晰性：明确时区信息，避免时间误解。 + +TDengine 在 REST API 和 Explorer UI 中，均使用 RFC3339 格式进行展示。在 SQL 语句中，可使用 RFC3339 格式写入时间戳数据： + +```sql +insert into t1 values('2024-10-27T01:59:59.000Z', 0); +select * from t1 where ts >= '2024-10-27T01:59:59.000Z'; +``` + +### 未定义行为 + +未定义行为（Undefined Behavior）是指特定代码或操作没有明确规定的结果，也不会对该结果作出兼容性的保证，TDengine 可能在某个版本后对当前的行为作出修改而不会通知用户。所以，在 TDengine 中，用户不可依赖当前未定义的行为进行判断或应用。 + +## 夏令时在 TDengine 中的写入与查询 + +我们使用下表来展示夏令时在写入和查询中的影响。 + +![DST Berlin](./02-dst/dst-berlin.png) + +### 表格说明 + +- **TIMESTAMP**：TDengine 中使用 64位整数来存储原始时间戳。 +- **UTC**：时间戳对应的 UTC 时间表示。 +- **Europe/Berlin**：表示时区 Europe/Berlin 对应的 RFC3339 格式时间。 +- **Local**：表示时区 Europe/Berlin 对应的当地时间（不含时区）。 + +### 表格分析 + +- 在**夏令时开始**（柏林时间 3 月 31 日 02:00）时，时间直接从 02:00 跳到 03:00（往后跳一小时）。 + - 浅绿色是夏令时开始前一小时的时间戳； + - 深绿色是夏令时开始后一小时的时间戳； + - 红色为 TDengine 数据库中插入了不存在的当地时间： + - 使用 SQL `INSERT INTO t1 values('2024-03-31 02:59:59',..)` 插入 `2024-03-31 02:00:00` 到 `2024-03-31 02:59:59` 的数据会被自动调整为 -1000（在 TDengine 中属于未定义行为，当前该值与数据库精度 precision 有关，毫秒数据库为 -1000，微秒数据库为 -1000000，纳秒数据库为 -1000000000），因为那一时刻在本地时间中不存在； +- 在**夏令时结束**（柏林时间 10 月 27 日 03:00）时，时间从 03:00 跳到 02:00 （往前跳一小时）。 + - 浅蓝色表示时钟跳变前一小时的时间戳； + - 深蓝色表示时钟跳变后一小时内的时间戳，其无时区的当地时间与上一小时一致。 + - 紫色表示时钟跳变一小时后的时间戳； +- **当地时间变化**：可见，由于夏令时的调整而导致了当地时间的变化，可能导致某些时间段出现重复或缺失。 +- **UTC 时间不变**：UTC 时间保持不变，确保了时间的一致性和顺序性。 +- **RFC3339**：RFC3339 格式时间显示了时间偏移量的变化，在夏令时开始后变为 +02:00，结束后变为 +01:00 。 +- **条件查询**： + - **夏令时开始**时，跳过的时间（`[03-31 02:00:00,03-31 03:00:00)`）不存在，所以在使用该时间进行查询时，行为不确定：`SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'`（不存在的本地时间戳被转换为 `-1000`）: + + ```sql + taos> SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'; + ts | + ================= + -1000 | + Query OK, 1 row(s) in set (0.003635s) + ``` + + 当不存在的时间戳与存在的时间戳共同使用时，其结果同样不符合预期，以下为起始本地时间不存在： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 03:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + -1000 | 1969-12-31T23:59:59.000Z | + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + 1711846800000 | 2024-03-31T01:00:00.000Z | + 1711846801000 | 2024-03-31T01:00:01.000Z | + Query OK, 5 row(s) in set (0.003339s) + ``` + + 以下语句中第一个 SQL 查询截止时间不存在，第二个截止时间存在，第一个 SQL 查询结果不符合预期： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 02:00:00'; + Query OK, 0 row(s) in set (0.000930s) + + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 01:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + Query OK, 2 row(s) in set (0.001227s) + ``` + + - 夏令时结束时，跳变的时间（`[10-27 02:00:00,10-27 03:00:00)` 不包含 `10-27 03:00:00`）重复了两次，TDengine 在使用该区间内的时间戳进行查询时，也属于未定义行为。 + - 查询 `[2024-10-27 02:00:00, 2024-10-27 03:00:00]` 之间的数据结果，包含了两次重复的时间戳和 `2024-10-27 03:00:00` 这个时间点的数据： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts BETWEEN '2024-10-27 02:00:00' AND '2024-10-27 03:00:00'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + 1729994400000 | 2024-10-27T02:00:00.000Z | 2024-10-27 03:00:00 | + Query OK, 5 row(s) in set (0.001370s) + ```` + + - 但以下查询 [2024-10-27 02:00:00.000,2024-10-27 02:57:34.999] 区间只能查询到第一个2024-10-27 02:00:00 时间点的数据： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:00.999'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 1 row(s) in set (0.004480s) + ``` + + - 以下查询 `[2024-10-27 02:00:01,2024-10-27 02:57:35]` 却能查到 3 条数据（包含一条 02:59:59 的当地时间数据）： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:35';; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ================================================================================================ + 2024-10-27 02:00:00.000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 2024-10-27 02:59:59.000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 2024-10-27 02:00:00.000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 3 row(s) in set (0.004428s) + ``` + +## 总结与建议 + +### 总结 + +仅针对使用当地时间带来的影响作说明，使用 UNIX 时间戳或 RFC3339 无影响。 + +- 写入： + - 无法写入夏令时跳变时不存在的时间数据。 + - 写入夏令时跳变时重复的时间是未定义行为。 +- 查询： + - 查询条件指定夏令时开始时跳变的时间，其查询结果为未定义行为。 + - 查询条件指定夏令时结束时重复的时间，其查询结果为未定义行为。 +- 显示： + - 带时区显示不受影响。 + - 显示当地时间是准确的，但夏令时结束时重复的时间会无法区分。 + - 用户应谨慎使用不带时区的时间进行展示和应用。 + +### 建议 + +为避免夏令时给查询和写入造成不必要的影响，在 TDengine 中，建议使用明确的时间偏移量进行写入和查询。 + +- 使用 UNIX 时间戳：使用 UNIX 时间戳可避免时区问题。 + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1711846799000 | 2024-03-31T00:59:59.000Z | 2024-03-31T01:59:59.000+01:00 | 2024-03-31 01:59:59 | + | 1711846800000 | 2024-03-31T01:00:00.000Z | 2024-03-31T03:00:00.000+02:00 | 2024-03-31 03:00:00 | + + ```sql + taos> insert into t1 values(1711846799000, 1)(1711846800000, 2); + Insert OK, 2 row(s) affected (0.001434s) + + taos> select * from t1 where ts between 1711846799000 and 1711846800000; + ts | v1 | + =============================== + 1711846799000 | 1 | + 1711846800000 | 2 | + Query OK, 2 row(s) in set (0.003503s) + ``` + +- 使用 RFC3339 时间格式：带时区偏移量的 RFC3339 时间格式可以有效避免夏令时的不确定性。 + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27T02:00:00.000+02:00 | 2024-10-27 02:00:00 | + | 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27T02:59:59.000+02:00 | 2024-10-27 02:59:59 | + | 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27T02:00:00.000+01:00 | 2024-10-27 02:00:00 | + | 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27T02:59:59.000+01:00 | 2024-10-27 02:59:59 | + + ```sql + taos> insert into t1 values ('2024-10-27T02:00:00.000+02:00', 1) + ('2024-10-27T02:59:59.000+02:00', 2) + ('2024-10-27T02:00:00.000+01:00', 3) + ('2024-10-27T02:59:59.000+01:00', 4); + Insert OK, 4 row(s) affected (0.001514s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+01:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 3 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 4 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 4 row(s) in set (0.004275s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+02:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 2 row(s) in set (0.004275s) + ``` + +- 查询时注意时区设定：在查询和显示时，如果需要本地时间，务必考虑夏令时的影响。 + - taosAdapter：使用 REST API 时，支持设置 IANA 时区，结果使用 RFC3339 格式返回。 + + ```shell + $ curl -uroot:taosdata 'localhost:6041/rest/sql?tz=Europe/Berlin'\ + -d "select ts from tz1.t1" + {"code":0,"column_meta":[["ts","TIMESTAMP",8]],"data":[["1970-01-01T00:59:59.000+01:00"],["2024-03-31T01:00:00.000+01:00"],["2024-03-31T01:59:59.000+01:00"],["2024-03-31T03:00:00.000+02:00"],["2024-03-31T03:00:01.000+02:00"],["2024-10-27T02:00:00.000+02:00"],["2024-10-27T02:59:59.000+02:00"],["2024-10-27T02:00:00.000+01:00"],["2024-10-27T02:59:59.000+01:00"],["2024-10-27T03:00:00.000+01:00"]],"rows":10} + ``` + + - Explorer：使用 Explorer 页面进行 SQL 查询时，用户可配置客户端时区，以 RFC3339 格式显示。 + + ![Explorer DST](./02-dst/explorer-with-tz.png) + +## 参考文档 + +- IANA Time Zone Database: [https://www.iana.org/time-zones](https://www.iana.org/time-zones) +- RFC3339: [https://datatracker.ietf.org/doc/html/rfc3339](https://datatracker.ietf.org/doc/html/rfc3339) diff --git a/docs/zh/27-train-faq/02-dst/dst-berlin.png b/docs/zh/27-train-faq/02-dst/dst-berlin.png new file mode 100644 index 0000000000..8c64626dd0 Binary files /dev/null and b/docs/zh/27-train-faq/02-dst/dst-berlin.png differ diff --git a/docs/zh/27-train-faq/02-dst/dst-table.png b/docs/zh/27-train-faq/02-dst/dst-table.png new file mode 100644 index 0000000000..f5eddbe010 Binary files /dev/null and b/docs/zh/27-train-faq/02-dst/dst-table.png differ diff --git a/docs/zh/27-train-faq/02-dst/explorer-with-tz.png b/docs/zh/27-train-faq/02-dst/explorer-with-tz.png new file mode 100644 index 0000000000..95758f117a Binary files /dev/null and b/docs/zh/27-train-faq/02-dst/explorer-with-tz.png differ diff --git a/include/common/tanalytics.h b/include/common/tanalytics.h index 42c3ce9391..0fb1d543f7 100644 --- a/include/common/tanalytics.h +++ b/include/common/tanalytics.h @@ -25,12 +25,13 @@ extern "C" { #endif -#define ANALY_FORECAST_DEFAULT_ROWS 10 -#define ANALY_FORECAST_DEFAULT_CONF 95 -#define ANALY_FORECAST_DEFAULT_WNCHECK 1 -#define ANALY_FORECAST_MAX_HISTORY_ROWS 40000 -#define ANALY_MAX_FC_ROWS 1024 -#define ANALY_ANOMALY_WINDOW_MAX_ROWS 40000 +#define ANALY_FORECAST_DEFAULT_ROWS 10 +#define ANALY_FORECAST_DEFAULT_CONF 95 +#define ANALY_FORECAST_DEFAULT_WNCHECK 1 +#define ANALY_FORECAST_MAX_ROWS 40000 +#define ANALY_ANOMALY_WINDOW_MAX_ROWS 40000 +#define ANALY_DEFAULT_TIMEOUT 60 +#define ANALY_MAX_TIMEOUT 600 typedef struct { EAnalAlgoType type; @@ -48,7 +49,7 @@ typedef enum { typedef enum { ANALYTICS_HTTP_TYPE_GET = 0, ANALYTICS_HTTP_TYPE_POST, -} EAnalHttpType; +} EAnalyHttpType; typedef struct { TdFilePtr filePtr; @@ -66,7 +67,7 @@ typedef struct { int32_t taosAnalyticsInit(); void taosAnalyticsCleanup(); -SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf *pBuf); +SJson *taosAnalySendReqRetJson(const char *url, EAnalyHttpType type, SAnalyticBuf *pBuf, int64_t timeout); int32_t taosAnalyGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen); bool taosAnalyGetOptStr(const char *option, const char *optName, char *optValue, int32_t optMaxLen); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 8acff4e04b..8e9367d7c2 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -299,6 +299,7 @@ extern bool tsStreamCoverage; extern int8_t tsS3EpNum; extern int32_t tsStreamNotifyMessageSize; extern int32_t tsStreamNotifyFrameSize; +extern bool tsCompareAsStrInGreatest; extern bool tsExperimental; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index cdff5aaba2..4af223b537 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -40,6 +40,7 @@ typedef enum { ARB_QUEUE, STREAM_CTRL_QUEUE, STREAM_LONG_EXEC_QUEUE, + STREAM_CHKPT_QUEUE, QUEUE_MAX, } EQueueType; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 13a26910c1..2b367939ff 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -355,6 +355,7 @@ TD_DEF_MSG_TYPE(TDMT_STREAM_DROP, "stream-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE_TRIGGER, "stream-retri-trigger", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_CONSEN_CHKPT, "stream-consen-chkpt", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_CHKPT_EXEC, "stream-exec-chkpt", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index d0cddd87a9..95fe14e572 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -276,6 +276,9 @@ typedef struct { #define IS_STR_DATA_TYPE(t) \ (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) +#define IS_COMPARE_STR_DATA_TYPE(t) \ + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) + #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) #define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 41b5d76371..1949081993 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -90,6 +90,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_DEGREES, FUNCTION_TYPE_RADIANS, FUNCTION_TYPE_TRUNCATE, + FUNCTION_TYPE_GREATEST, + FUNCTION_TYPE_LEAST, // string function FUNCTION_TYPE_LENGTH = 1500, diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index a98bf29afb..99f1759a7a 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -66,6 +66,8 @@ int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode * SNode **pOtherCond); int32_t filterIsMultiTableColsCond(SNode *pCond, bool *res); EConditionType filterClassifyCondition(SNode *pNode); +int32_t filterGetCompFunc(__compar_fn_t *func, int32_t type, int32_t optr); +bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); #ifdef __cplusplus } diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 67fd954ad7..d1dda544ae 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -44,6 +44,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type, int8_ int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int32_t *overflow, int32_t startIndex, int32_t numOfRows); +int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex, int32_t numOfRows); /* Math functions */ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); @@ -71,6 +72,8 @@ int32_t signFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t degreesFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t radiansFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t randFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t greatestFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t leastFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); /* String functions */ int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 372322c0b8..bdb8bde2da 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -702,8 +702,8 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamExecTask(SStreamTask* pTask); int32_t streamResumeTask(SStreamTask* pTask); -int32_t streamTrySchedExec(SStreamTask* pTask); -int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType); +int32_t streamTrySchedExec(SStreamTask* pTask, bool chkptExec); +int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType, bool chkptExec); void streamTaskResumeInFuture(SStreamTask* pTask); void streamTaskClearSchedIdleInfo(SStreamTask* pTask); void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 266bd4a618..986874c5bb 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -27,6 +27,7 @@ #include "scheduler.h" #include "tcache.h" #include "tcompare.h" +#include "tconv.h" #include "tglobal.h" #include "thttp.h" #include "tmsg.h" @@ -36,7 +37,6 @@ #include "tsched.h" #include "ttime.h" #include "tversion.h" -#include "tconv.h" #include "cus_name.h" @@ -63,13 +63,13 @@ } \ } while (0) -STscDbg tscDbg = {0}; -SAppInfo appInfo; -int64_t lastClusterId = 0; -int32_t clientReqRefPool = -1; -int32_t clientConnRefPool = -1; -int32_t clientStop = -1; -SHashObj* pTimezoneMap = NULL; +STscDbg tscDbg = {0}; +SAppInfo appInfo; +int64_t lastClusterId = 0; +int32_t clientReqRefPool = -1; +int32_t clientConnRefPool = -1; +int32_t clientStop = -1; +SHashObj *pTimezoneMap = NULL; int32_t timestampDeltaLimit = 900; // s @@ -964,7 +964,7 @@ void taos_init_imp(void) { ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg"); initQueryModuleMsgHandle(); - if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL){ + if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) { tscInitRes = terrno; tscError("failed to init conv"); return; diff --git a/source/common/src/tanalytics.c b/source/common/src/tanalytics.c index deb68af3ea..397accc0b1 100644 --- a/source/common/src/tanalytics.c +++ b/source/common/src/tanalytics.c @@ -276,7 +276,7 @@ _OVER: return code; } -static int32_t taosCurlPostRequest(const char *url, SCurlResp *pRsp, const char *buf, int32_t bufLen) { +static int32_t taosCurlPostRequest(const char *url, SCurlResp *pRsp, const char *buf, int32_t bufLen, int32_t timeout) { struct curl_slist *headers = NULL; CURL *curl = NULL; CURLcode code = 0; @@ -292,7 +292,7 @@ static int32_t taosCurlPostRequest(const char *url, SCurlResp *pRsp, const char if (curl_easy_setopt(curl, CURLOPT_URL, url) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, taosCurlWriteData) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, pRsp) != 0) goto _OVER; - if (curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 60000) != 0) goto _OVER; + if (curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_POST, 1) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, bufLen) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buf) != 0) goto _OVER; @@ -311,7 +311,7 @@ _OVER: return code; } -SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf *pBuf) { +SJson *taosAnalySendReqRetJson(const char *url, EAnalyHttpType type, SAnalyticBuf *pBuf, int64_t timeout) { int32_t code = -1; char *pCont = NULL; int64_t contentLen; @@ -329,7 +329,7 @@ SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf terrno = code; goto _OVER; } - if (taosCurlPostRequest(url, &curlRsp, pCont, contentLen) != 0) { + if (taosCurlPostRequest(url, &curlRsp, pCont, contentLen, timeout) != 0) { terrno = TSDB_CODE_ANA_URL_CANT_ACCESS; goto _OVER; } @@ -767,7 +767,7 @@ static int32_t taosAnalyBufGetCont(SAnalyticBuf *pBuf, char **ppCont, int64_t *p int32_t taosAnalyticsInit() { return 0; } void taosAnalyticsCleanup() {} -SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf *pBuf) { return NULL; } +SJson *taosAnalySendReqRetJson(const char *url, EAnalyHttpType type, SAnalyticBuf *pBuf, int64_t timeout) { return NULL; } int32_t taosAnalyGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen) { return 0; } bool taosAnalyGetOptStr(const char *option, const char *optName, char *optValue, int32_t optMaxLen) { return true; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 00f9504bc9..756d03fd8a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "tglobal.h" #include "cJSON.h" #include "defines.h" #include "os.h" #include "osString.h" #include "tconfig.h" -#include "tglobal.h" #include "tgrant.h" #include "tjson.h" #include "tlog.h" @@ -28,7 +28,6 @@ #include "tutil.h" - #define CONFIG_PATH_LEN (TSDB_FILENAME_LEN + 12) #define CONFIG_FILE_LEN (CONFIG_PATH_LEN + 32) @@ -117,9 +116,9 @@ bool tsMndSkipGrant = false; bool tsEnableWhiteList = false; // ip white list cfg // arbitrator -int32_t tsArbHeartBeatIntervalSec = 5; -int32_t tsArbCheckSyncIntervalSec = 10; -int32_t tsArbSetAssignedTimeoutSec = 30; +int32_t tsArbHeartBeatIntervalSec = 2; +int32_t tsArbCheckSyncIntervalSec = 3; +int32_t tsArbSetAssignedTimeoutSec = 6; // dnode int64_t tsDndStart = 0; @@ -131,6 +130,8 @@ uint32_t tsEncryptionKeyChksum = 0; int8_t tsEncryptionKeyStat = ENCRYPT_KEY_STAT_UNSET; int8_t tsGrant = 1; +bool tsCompareAsStrInGreatest = true; + // monitor bool tsEnableMonitor = true; int32_t tsMonitorInterval = 30; @@ -501,9 +502,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -struct SConfig *taosGetCfg() { - return tsCfg; -} +struct SConfig *taosGetCfg() { return tsCfg; } static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -692,7 +691,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL) != 0); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_SERVER, + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 3000, 86400000, CFG_SCOPE_SERVER, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( @@ -749,6 +748,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { TAOS_CHECK_RETURN( cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "compareAsStrInGreatest", tsCompareAsStrInGreatest, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT,CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -1483,6 +1484,9 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamCoverage"); tsStreamCoverage = pItem->bval; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "compareAsStrInGreatest"); + tsCompareAsStrInGreatest = pItem->bval; + TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -2786,7 +2790,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"numOfRpcSessions", &tsNumOfRpcSessions}, {"bypassFlag", &tsBypassFlag}, {"safetyCheckLevel", &tsSafetyCheckLevel}, - {"streamCoverage", &tsStreamCoverage}}; + {"streamCoverage", &tsStreamCoverage}, + {"compareAsStrInGreatest", &tsCompareAsStrInGreatest}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); diff --git a/source/dnode/mgmt/mgmt_snode/inc/smInt.h b/source/dnode/mgmt/mgmt_snode/inc/smInt.h index 9d519e88f0..0df4b1c58a 100644 --- a/source/dnode/mgmt/mgmt_snode/inc/smInt.h +++ b/source/dnode/mgmt/mgmt_snode/inc/smInt.h @@ -47,7 +47,7 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pMsg); int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg); -void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg); +int32_t smPutNodeMsgToChkptQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 024e2e4e99..11710d7b39 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -102,6 +102,8 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_CHKPT_EXEC, smPutNodeMsgToStreamQueue, 0) == NULL) goto _OVER; + code = 0; _OVER: if (code != 0) { diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 1e882fc656..1255542454 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -162,6 +162,9 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { case WRITE_QUEUE: code = smPutNodeMsgToWriteQueue(pMgmt, pMsg); break; + case STREAM_CHKPT_QUEUE: + code = smPutNodeMsgToStreamQueue(pMgmt, pMsg); + break; default: code = TSDB_CODE_INVALID_PARA; rpcFreeCont(pMsg->pCont); @@ -172,7 +175,6 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { } int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { - int32_t code = 0; SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0); if (pWorker == NULL) { return TSDB_CODE_INVALID_MSG; @@ -198,3 +200,10 @@ int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); return taosWriteQitem(pWorker->queue, pMsg); } + +//int32_t smPutNodeMsgToChkptQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { +// SSingleWorker *pWorker = &pMgmt->chkptWorker; +// +// dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); +// return taosWriteQitem(pWorker->queue, pMsg); +//} diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index e33730130d..cb41bc0ea7 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -34,6 +34,7 @@ typedef struct SVnodeMgmt { SAutoQWorkerPool streamPool; SAutoQWorkerPool streamLongExecPool; SWWorkerPool streamCtrlPool; + SWWorkerPool streamChkPool; SWWorkerPool fetchPool; SSingleWorker mgmtWorker; SSingleWorker mgmtMultiWorker; @@ -77,6 +78,7 @@ typedef struct { STaosQueue *pStreamQ; STaosQueue *pStreamCtrlQ; STaosQueue *pStreamLongExecQ; + STaosQueue *pStreamChkQ; STaosQueue *pFetchQ; STaosQueue *pMultiMgmQ; } SVnodeObj; @@ -141,6 +143,7 @@ int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToStreamCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToStreamLongExecQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutMsgToStreamChkQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index fc8ff3133a..8a18727b99 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -1022,6 +1022,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToStreamCtrlQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY, vmPutMsgToStreamLongExecQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_CHKPT_EXEC, vmPutMsgToStreamChkQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 8871cd575f..5da3b2ce9a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -407,6 +407,9 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal, pVnode->pStreamLongExecQ, taosQueueItemSize(pVnode->pStreamLongExecQ)); while (!taosQueueEmpty(pVnode->pStreamLongExecQ)) taosMsleep(50); + dInfo("vgId:%d, wait for vnode stream chkpt queue:%p is empty", pVnode->vgId, pVnode->pStreamChkQ); + while (!taosQueueEmpty(pVnode->pStreamChkQ)) taosMsleep(10); + dInfo("vgId:%d, all vnode queues is empty", pVnode->vgId); dInfo("vgId:%d, post close", pVnode->vgId); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 5acd06bbda..41f3c64e7d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -165,6 +165,34 @@ static void vmProcessStreamCtrlQueue(SQueueInfo *pInfo, STaosQall* pQall, int32_ } } +static void vmProcessStreamChkptQueue(SQueueInfo *pInfo, STaosQall* pQall, int32_t numOfItems) { + SVnodeObj *pVnode = pInfo->ahandle; + void *pItem = NULL; + int32_t code = 0; + + while (1) { + if (taosGetQitem(pQall, &pItem) == 0) { + break; + } + + SRpcMsg *pMsg = pItem; + const STraceId *trace = &pMsg->info.traceId; + + dGTrace("vgId:%d, msg:%p get from vnode-stream-chkpt queue", pVnode->vgId, pMsg); + code = vnodeProcessStreamChkptMsg(pVnode->pImpl, pMsg, pInfo); + if (code != 0) { + terrno = code; + dGError("vgId:%d, msg:%p failed to process stream chkpt msg %s since %s", pVnode->vgId, pMsg, + TMSG_INFO(pMsg->msgType), tstrerror(code)); + vmSendRsp(pMsg, code); + } + + dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } +} + static void vmProcessStreamLongExecQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeObj *pVnode = pInfo->ahandle; const STraceId *trace = &pMsg->info.traceId; @@ -301,6 +329,10 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp dGTrace("vgId:%d, msg:%p put into vnode-stream-long-exec queue", pVnode->vgId, pMsg); code = taosWriteQitem(pVnode->pStreamLongExecQ, pMsg); break; + case STREAM_CHKPT_QUEUE: + dGTrace("vgId:%d, msg:%p put into vnode-stream-chkpt queue", pVnode->vgId, pMsg); + code = taosWriteQitem(pVnode->pStreamChkQ, pMsg); + break; case FETCH_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg); code = taosWriteQitem(pVnode->pFetchQ, pMsg); @@ -361,6 +393,8 @@ int32_t vmPutMsgToStreamCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmP int32_t vmPutMsgToStreamLongExecQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, STREAM_LONG_EXEC_QUEUE); } +int32_t vmPutMsgToStreamChkQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, STREAM_CHKPT_QUEUE); } + int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; dGTrace("msg:%p, put into vnode-multi-mgmt queue", pMsg); @@ -439,6 +473,8 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { case STREAM_LONG_EXEC_QUEUE: size = taosQueueItemSize(pVnode->pStreamLongExecQ); break; + case STREAM_CHKPT_QUEUE: + size = taosQueueItemSize(pVnode->pStreamChkQ); default: break; } @@ -487,10 +523,11 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pStreamQ = tAutoQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue, 2); pVnode->pStreamCtrlQ = tWWorkerAllocQueue(&pMgmt->streamCtrlPool, pVnode, (FItems)vmProcessStreamCtrlQueue); pVnode->pStreamLongExecQ = tAutoQWorkerAllocQueue(&pMgmt->streamLongExecPool, pVnode, (FItem)vmProcessStreamLongExecQueue, 1); + pVnode->pStreamChkQ = tWWorkerAllocQueue(&pMgmt->streamChkPool, pVnode, (FItems)vmProcessStreamChkptQueue); if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL || pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL - || pVnode->pStreamCtrlQ == NULL || pVnode->pStreamLongExecQ == NULL) { + || pVnode->pStreamCtrlQ == NULL || pVnode->pStreamLongExecQ == NULL || pVnode->pStreamChkQ == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -509,6 +546,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { dInfo("vgId:%d, stream-long-exec-queue:%p is alloced", pVnode->vgId, pVnode->pStreamLongExecQ); dInfo("vgId:%d, stream-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pStreamCtrlQ, taosQueueGetThreadId(pVnode->pStreamCtrlQ)); + dInfo("vgId:%d, stream-chk-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pStreamChkQ, + taosQueueGetThreadId(pVnode->pStreamChkQ)); return 0; } @@ -517,6 +556,7 @@ void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { tAutoQWorkerFreeQueue(&pMgmt->streamPool, pVnode->pStreamQ); tAutoQWorkerFreeQueue(&pMgmt->streamLongExecPool, pVnode->pStreamLongExecQ); tWWorkerFreeQueue(&pMgmt->streamCtrlPool, pVnode->pStreamCtrlQ); + tWWorkerFreeQueue(&pMgmt->streamChkPool, pVnode->pStreamChkQ); tWWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); pVnode->pQueryQ = NULL; pVnode->pFetchQ = NULL; @@ -525,6 +565,8 @@ void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pStreamCtrlQ = NULL; pVnode->pStreamLongExecQ = NULL; + pVnode->pStreamChkQ = NULL; + pVnode->pFetchQ = NULL; dDebug("vgId:%d, queue is freed", pVnode->vgId); } @@ -554,6 +596,11 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { pStreamCtrlPool->max = 1; if ((code = tWWorkerInit(pStreamCtrlPool)) != 0) return code; + SWWorkerPool *pStreamChkPool = &pMgmt->streamChkPool; + pStreamChkPool->name = "vnode-stream-chkpt"; + pStreamChkPool->max = 1; + if ((code = tWWorkerInit(pStreamChkPool)) != 0) return code; + SWWorkerPool *pFPool = &pMgmt->fetchPool; pFPool->name = "vnode-fetch"; pFPool->max = tsNumOfVnodeFetchThreads; @@ -587,6 +634,7 @@ void vmStopWorker(SVnodeMgmt *pMgmt) { tAutoQWorkerCleanup(&pMgmt->streamPool); tAutoQWorkerCleanup(&pMgmt->streamLongExecPool); tWWorkerCleanup(&pMgmt->streamCtrlPool); + tWWorkerCleanup(&pMgmt->streamChkPool); tWWorkerCleanup(&pMgmt->fetchPool); dDebug("vnode workers are closed"); } diff --git a/source/dnode/mnode/impl/src/mndAnode.c b/source/dnode/mnode/impl/src/mndAnode.c index c08d4aead4..bd0a4f3138 100644 --- a/source/dnode/mnode/impl/src/mndAnode.c +++ b/source/dnode/mnode/impl/src/mndAnode.c @@ -789,7 +789,7 @@ static int32_t mndGetAnodeAlgoList(const char *url, SAnodeObj *pObj) { char anodeUrl[TSDB_ANALYTIC_ANODE_URL_LEN + 1] = {0}; snprintf(anodeUrl, TSDB_ANALYTIC_ANODE_URL_LEN, "%s/%s", url, "list"); - SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL); + SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL, 0); if (pJson == NULL) return terrno; int32_t code = mndDecodeAlgoList(pJson, pObj); @@ -805,7 +805,7 @@ static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status, int32_t statusLe char anodeUrl[TSDB_ANALYTIC_ANODE_URL_LEN + 1] = {0}; snprintf(anodeUrl, TSDB_ANALYTIC_ANODE_URL_LEN, "%s/%s", pObj->url, "status"); - SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL); + SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL, 0); if (pJson == NULL) return terrno; code = tjsonGetDoubleValue(pJson, "protocol", &tmp); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 851bf25665..dcdc70da68 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -92,7 +92,7 @@ FAIL: } int32_t sndInit(SSnode *pSnode) { - if (streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_START_ALL_TASKS) != 0) { + if (streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_START_ALL_TASKS, false) != 0) { sndError("failed to start all tasks"); } return 0; @@ -138,6 +138,8 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return tqStreamTaskProcessRetrieveTriggerReq(pSnode->pMeta, pMsg); case TDMT_STREAM_RETRIEVE_TRIGGER_RSP: return tqStreamTaskProcessRetrieveTriggerRsp(pSnode->pMeta, pMsg); + case TDMT_STREAM_CHKPT_EXEC: + return tqStreamTaskProcessRunReq(pSnode->pMeta, pMsg, true); default: sndError("invalid snode msg:%d", pMsg->msgType); return TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index d224f9a411..a85dca05d5 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -114,6 +114,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); int32_t vnodeProcessStreamCtrlMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); int32_t vnodeProcessStreamLongExecMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); +int32_t vnodeProcessStreamChkptMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit); diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index 6ed4991679..3b9857a6a2 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -14,10 +14,19 @@ */ #include "meta.h" +#include "vnd.h" static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { + SVBufPool *pool = (SVBufPool *)pPool; + SVnode *pVnode = pool->pVnode; + + if (pVnode->inUse && pVnode->inUse->size > pVnode->inUse->node.size) { + return NULL; + } + return vnodeBufPoolMallocAligned((SVBufPool *)pPool, size); } + static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } // begin a meta txn diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7d65673226..c80edd070d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -928,12 +928,6 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask // now the fill-history task starts to scan data from wal files. code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); -// if (code == TSDB_CODE_SUCCESS) { -// code = tqScanWalAsync(pTq, false); -// if (code) { -// tqError("vgId:%d failed to start scan wal file, code:%s", vgId, tstrerror(code)); -// } -// } } } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 08ca02e20e..6a6b3574db 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -148,6 +148,7 @@ static void doStartScanWal(void* param, void* tmrId) { return; } + // failed to lock, try 500ms later code = streamMetaTryRlock(pMeta); if (code == 0) { numOfTasks = taosArrayGetSize(pMeta->pTaskList); @@ -156,25 +157,23 @@ static void doStartScanWal(void* param, void* tmrId) { numOfTasks = 0; } - if (numOfTasks == 0) { - goto _end; - } + if (numOfTasks > 0) { + tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d", vgId, numOfTasks); - tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d", vgId, numOfTasks); - - #if 0 +#if 0 // wait for the vnode is freed, and invalid read may occur. taosMsleep(10000); - #endif +#endif - code = streamTaskSchedTask(&pParam->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); - if (code) { - tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code)); + code = streamTaskSchedTask(&pParam->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA, false); + if (code) { + tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code)); + } } _end: streamTmrStart(doStartScanWal, SCAN_WAL_IDLE_DURATION, pParam, pTimer, &pMeta->scanInfo.scanTimer, vgId, "scan-wal"); - tqDebug("vgId:%d scan-wal will start in %dms", vgId, SCAN_WAL_IDLE_DURATION*SCAN_WAL_WAIT_COUNT); + tqDebug("vgId:%d try scan-wal will start in %dms", vgId, SCAN_WAL_IDLE_DURATION*SCAN_WAL_WAIT_COUNT); code = taosReleaseRef(streamMetaRefPool, pParam->metaId); if (code) { @@ -192,7 +191,7 @@ void tqScanWalAsync(STQ* pTq) { // 1. the vnode should be the leader. // 2. the stream isn't disabled - if ((pMeta->role == NODE_ROLE_FOLLOWER) || tsDisableStream) { + if ((pMeta->role != NODE_ROLE_LEADER) || tsDisableStream) { tqInfo("vgId:%d follower node or stream disabled, not scan wal", vgId); return; } @@ -217,7 +216,7 @@ void tqScanWalAsync(STQ* pTq) { } int32_t tqStopStreamAllTasksAsync(SStreamMeta* pMeta, SMsgCb* pMsgCb) { - return streamTaskSchedTask(pMsgCb, pMeta->vgId, 0, 0, STREAM_EXEC_T_STOP_ALL_TASKS); + return streamTaskSchedTask(pMsgCb, pMeta->vgId, 0, 0, STREAM_EXEC_T_STOP_ALL_TASKS, false); } int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId) { @@ -323,7 +322,7 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) { // check whether input queue is full or not if (streamQueueIsFull(pTask->inputq.queue)) { tqTrace("s-task:%s input queue is full, launch task without scanning wal", pTask->id.idStr); - int32_t code = streamTrySchedExec(pTask); + int32_t code = streamTrySchedExec(pTask, false); if (code) { tqError("s-task:%s failed to start task while inputQ is full", pTask->id.idStr); } @@ -462,7 +461,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, int32_t* pNumOfTasks) { streamMutexUnlock(&pTask->lock); if ((numOfItems > 0) || hasNewData) { - code = streamTrySchedExec(pTask); + code = streamTrySchedExec(pTask, false); if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pStreamMeta, pTask); taosArrayDestroy(pTaskList); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 1afccd3d01..64072bbda1 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -131,7 +131,7 @@ int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { tqDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); int32_t type = restart ? STREAM_EXEC_T_RESTART_ALL_TASKS : STREAM_EXEC_T_START_ALL_TASKS; - return streamTaskSchedTask(cb, vgId, 0, 0, type); + return streamTaskSchedTask(cb, vgId, 0, 0, type, false); } int32_t tqStreamStartOneTaskAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t streamId, int32_t taskId) { @@ -143,7 +143,7 @@ int32_t tqStreamStartOneTaskAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t stream } tqDebug("vgId:%d start task:0x%x async", vgId, taskId); - return streamTaskSchedTask(cb, vgId, streamId, taskId, STREAM_EXEC_T_START_ONE_TASK); + return streamTaskSchedTask(cb, vgId, streamId, taskId, STREAM_EXEC_T_START_ONE_TASK, false); } // this is to process request from transaction, always return true. @@ -960,11 +960,6 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { streamMetaWUnLock(pMeta); -// if (scanWal && (vgId != SNODE_HANDLE)) { -// tqDebug("vgId:%d start scan wal for executing tasks", vgId); -// code = tqScanWalAsync(pMeta->ahandle, true); -// } - return code; } @@ -1227,7 +1222,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { // code = tqScanWalAsync((STQ*)handle, false); } else { - code = streamTrySchedExec(pTask); + code = streamTrySchedExec(pTask, false); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 8baf08ef94..ca3b82239b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -610,11 +610,12 @@ static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { code = taosThreadMutexTryLock(&pReader->readerMutex); if (code != TSDB_CODE_SUCCESS) { - tsdbError("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + // Failing to acquire the lock is reasonable, not an error + tsdbWarn("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); } else { tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex); } - TSDB_CHECK_CODE(code, lino, _end); + return code; _end: if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 22d9c2657d..ea4ca896ab 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -940,6 +940,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("vgId:%d, msg:%p in stream queue is processing", pVnode->config.vgId, pMsg); + + // todo: NOTE: some command needs to run on follower, such as, stop_all_tasks if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || pMsg->msgType == TDMT_VND_BATCH_META) && !syncIsReadyForRead(pVnode->sync)) { @@ -1016,6 +1018,24 @@ int32_t vnodeProcessStreamLongExecMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo } } +int32_t vnodeProcessStreamChkptMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { + vTrace("vgId:%d, msg:%p in stream chkpt queue is processing", pVnode->config.vgId, pMsg); + if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || + pMsg->msgType == TDMT_VND_BATCH_META) && + !syncIsReadyForRead(pVnode->sync)) { + vnodeRedirectRpcMsg(pVnode, pMsg, terrno); + return 0; + } + + switch (pMsg->msgType) { + case TDMT_STREAM_CHKPT_EXEC: + return tqProcessTaskRunReq(pVnode->pTq, pMsg); + default: + vError("unknown msg type:%d in stream chkpt queue", pMsg->msgType); + return TSDB_CODE_APP_ERROR; + } +} + void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { int32_t code = tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); if (code) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index a7e8a43fae..317fd3307e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -169,8 +169,9 @@ void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit) { rpcFreeCont(rpcMsg.pCont); rpcMsg.pCont = NULL; } else { - if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &rpcMsg) < 0) { - vTrace("vgId:%d, failed to put vnode commit to queue since %s", pVnode->config.vgId, terrstr()); + int32_t code = 0; + if ((code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &rpcMsg)) < 0) { + vError("vgId:%d, failed to put vnode commit to write_queue since %s", pVnode->config.vgId, tstrerror(code)); } } } @@ -449,7 +450,9 @@ static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsm pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, pMsg->info.conn.applyIndex, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType), pMsg->code); - return tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); + int32_t code = tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); + if (code < 0) vError("vgId:%d, failed to put into apply_queue since %s", pVnode->config.vgId, tstrerror(code)); + return code; } static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { @@ -594,7 +597,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) streamMetaWUnLock(pMeta); tqInfo("vgId:%d stream task already loaded, start them", vgId); - int32_t code = streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_START_ALL_TASKS); + int32_t code = streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_START_ALL_TASKS, false); if (code != 0) { tqError("vgId:%d failed to sched stream task, code:%s", vgId, tstrerror(code)); } diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index 46aae38ad4..379177bb06 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -47,6 +47,7 @@ typedef struct { char algoName[TSDB_ANALYTIC_ALGO_NAME_LEN]; char algoUrl[TSDB_ANALYTIC_ALGO_URL_LEN]; char anomalyOpt[TSDB_ANALYTIC_ALGO_OPTION_LEN]; + int64_t timeout; SAnomalyWindowSupp anomalySup; SWindowRowsSup anomalyWinRowSup; SColumn anomalyCol; @@ -89,6 +90,20 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p goto _error; } + bool hasTimeout = taosAnalyGetOptInt(pAnomalyNode->anomalyOpt, "timeout", &pInfo->timeout); + if (!hasTimeout) { + qDebug("not set the timeout val, set default:%d", ANALY_DEFAULT_TIMEOUT); + pInfo->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + if (pInfo->timeout <= 0 || pInfo->timeout > ANALY_MAX_TIMEOUT) { + qDebug("timeout val:%" PRId64 "s is invalid (greater than 10min or less than 1s), use default:%dms", + pInfo->timeout, ANALY_DEFAULT_TIMEOUT); + pInfo->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + qDebug("timeout val is set to: %" PRId64 "s", pInfo->timeout); + } + } + pOperator->exprSupp.hasWindowOrGroup = true; pInfo->tsSlotId = ((SColumnNode*)pAnomalyNode->window.pTspk)->slotId; tstrncpy(pInfo->anomalyOpt, pAnomalyNode->anomalyOpt, sizeof(pInfo->anomalyOpt)); @@ -451,7 +466,7 @@ static int32_t anomalyAnalysisWindow(SOperatorInfo* pOperator) { code = taosAnalyBufClose(&analyBuf); QUERY_CHECK_CODE(code, lino, _OVER); - pJson = taosAnalySendReqRetJson(pInfo->algoUrl, ANALYTICS_HTTP_TYPE_POST, &analyBuf); + pJson = taosAnalySendReqRetJson(pInfo->algoUrl, ANALYTICS_HTTP_TYPE_POST, &analyBuf, pInfo->timeout * 1000); if (pJson == NULL) { code = terrno; goto _OVER; diff --git a/source/libs/executor/src/forecastoperator.c b/source/libs/executor/src/forecastoperator.c index e318530352..e9185824a3 100644 --- a/source/libs/executor/src/forecastoperator.c +++ b/source/libs/executor/src/forecastoperator.c @@ -38,6 +38,7 @@ typedef struct { int64_t optRows; int64_t cachedRows; int32_t numOfBlocks; + int64_t timeout; int16_t resTsSlot; int16_t resValSlot; int16_t resLowSlot; @@ -76,10 +77,10 @@ static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock, con int32_t lino = 0; SAnalyticBuf* pBuf = &pSupp->analyBuf; - if (pSupp->cachedRows > ANALY_FORECAST_MAX_HISTORY_ROWS) { + if (pSupp->cachedRows > ANALY_FORECAST_MAX_ROWS) { code = TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; qError("%s rows:%" PRId64 " for forecast cache, error happens, code:%s, upper limit:%d", id, pSupp->cachedRows, - tstrerror(code), ANALY_FORECAST_MAX_HISTORY_ROWS); + tstrerror(code), ANALY_FORECAST_MAX_ROWS); return code; } @@ -157,8 +158,8 @@ static int32_t forecastCloseBuf(SForecastSupp* pSupp, const char* id) { qDebug("%s forecast rows not found from %s, use default:%" PRId64, id, pSupp->algoOpt, pSupp->optRows); } - if (pSupp->optRows > ANALY_MAX_FC_ROWS) { - qError("%s required too many forecast rows, max allowed:%d, required:%" PRId64, id, ANALY_MAX_FC_ROWS, + if (pSupp->optRows > ANALY_FORECAST_MAX_ROWS) { + qError("%s required too many forecast rows, max allowed:%d, required:%" PRId64, id, ANALY_FORECAST_MAX_ROWS, pSupp->optRows); return TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; } @@ -198,12 +199,12 @@ static int32_t forecastCloseBuf(SForecastSupp* pSupp, const char* id) { static int32_t forecastAnalysis(SForecastSupp* pSupp, SSDataBlock* pBlock, const char* pId) { SAnalyticBuf* pBuf = &pSupp->analyBuf; int32_t resCurRow = pBlock->info.rows; - int8_t tmpI8; - int16_t tmpI16; - int32_t tmpI32; - int64_t tmpI64; - float tmpFloat; - double tmpDouble; + int8_t tmpI8 = 0; + int16_t tmpI16 = 0; + int32_t tmpI32 = 0; + int64_t tmpI64 = 0; + float tmpFloat = 0; + double tmpDouble = 0; int32_t code = 0; SColumnInfoData* pResValCol = taosArrayGet(pBlock->pDataBlock, pSupp->resValSlot); @@ -211,12 +212,13 @@ static int32_t forecastAnalysis(SForecastSupp* pSupp, SSDataBlock* pBlock, const return terrno; } - SColumnInfoData* pResTsCol = (pSupp->resTsSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resTsSlot) : NULL); - SColumnInfoData* pResLowCol = (pSupp->resLowSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resLowSlot) : NULL); + SColumnInfoData* pResTsCol = ((pSupp->resTsSlot != -1) ? taosArrayGet(pBlock->pDataBlock, pSupp->resTsSlot) : NULL); + SColumnInfoData* pResLowCol = + ((pSupp->resLowSlot != -1) ? taosArrayGet(pBlock->pDataBlock, pSupp->resLowSlot) : NULL); SColumnInfoData* pResHighCol = (pSupp->resHighSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resHighSlot) : NULL); - SJson* pJson = taosAnalySendReqRetJson(pSupp->algoUrl, ANALYTICS_HTTP_TYPE_POST, pBuf); + SJson* pJson = taosAnalySendReqRetJson(pSupp->algoUrl, ANALYTICS_HTTP_TYPE_POST, pBuf, pSupp->timeout * 1000); if (pJson == NULL) { return terrno; } @@ -527,18 +529,32 @@ static int32_t forecastParseInput(SForecastSupp* pSupp, SNodeList* pFuncs) { return 0; } -static int32_t forecastParseAlgo(SForecastSupp* pSupp) { +static int32_t forecastParseAlgo(SForecastSupp* pSupp, const char* id) { pSupp->maxTs = 0; pSupp->minTs = INT64_MAX; pSupp->numOfRows = 0; if (!taosAnalyGetOptStr(pSupp->algoOpt, "algo", pSupp->algoName, sizeof(pSupp->algoName))) { - qError("failed to get forecast algorithm name from %s", pSupp->algoOpt); + qError("%s failed to get forecast algorithm name from %s", id, pSupp->algoOpt); return TSDB_CODE_ANA_ALGO_NOT_FOUND; } + bool hasTimeout = taosAnalyGetOptInt(pSupp->algoOpt, "timeout", &pSupp->timeout); + if (!hasTimeout) { + qDebug("%s not set the timeout val, set default:%d", id, ANALY_DEFAULT_TIMEOUT); + pSupp->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + if (pSupp->timeout <= 0 || pSupp->timeout > ANALY_MAX_TIMEOUT) { + qDebug("%s timeout val:%" PRId64 "s is invalid (greater than 10min or less than 1s), use default:%dms", + id, pSupp->timeout, ANALY_DEFAULT_TIMEOUT); + pSupp->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + qDebug("%s timeout val is set to: %" PRId64 "s", id, pSupp->timeout); + } + } + if (taosAnalyGetAlgoUrl(pSupp->algoName, ANALY_ALGO_TYPE_FORECAST, pSupp->algoUrl, sizeof(pSupp->algoUrl)) != 0) { - qError("failed to get forecast algorithm url from %s", pSupp->algoName); + qError("%s failed to get forecast algorithm url from %s", id, pSupp->algoName); return TSDB_CODE_ANA_ALGO_NOT_LOAD; } @@ -589,6 +605,7 @@ int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNo goto _error; } + const char* pId = pTaskInfo->id.str; SForecastSupp* pSupp = &pInfo->forecastSupp; SForecastFuncPhysiNode* pForecastPhyNode = (SForecastFuncPhysiNode*)pPhyNode; SExprSupp* pExprSup = &pOperator->exprSupp; @@ -620,7 +637,7 @@ int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNo code = forecastParseOutput(pSupp, pExprSup); QUERY_CHECK_CODE(code, lino, _error); - code = forecastParseAlgo(pSupp); + code = forecastParseAlgo(pSupp, pId); QUERY_CHECK_CODE(code, lino, _error); code = forecastCreateBuf(pSupp); @@ -644,7 +661,7 @@ int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNo *pOptrInfo = pOperator; - qDebug("forecast env is initialized, option:%s", pSupp->algoOpt); + qDebug("%s forecast env is initialized, option:%s", pId, pSupp->algoOpt); return TSDB_CODE_SUCCESS; _error: diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 63ff2fa92b..37466aac8d 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -57,11 +57,12 @@ void destroyStreamCountAggOperatorInfo(void* param) { } destroyStreamBasicInfo(&pInfo->basic); - destroyStreamAggSupporter(&pInfo->streamAggSup); + cleanupExprSupp(&pInfo->scalarSupp); clearGroupResInfo(&pInfo->groupResInfo); taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos); pInfo->pUpdated = NULL; + destroyStreamAggSupporter(&pInfo->streamAggSup); colDataDestroy(&pInfo->twAggSup.timeWindowData); blockDataDestroy(pInfo->pDelRes); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index eacb2fcfc8..f25f711783 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -56,10 +56,11 @@ void destroyStreamEventOperatorInfo(void* param) { } destroyStreamBasicInfo(&pInfo->basic); - destroyStreamAggSupporter(&pInfo->streamAggSup); + clearGroupResInfo(&pInfo->groupResInfo); taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos); pInfo->pUpdated = NULL; + destroyStreamAggSupporter(&pInfo->streamAggSup); cleanupExprSupp(&pInfo->scalarSupp); if (pInfo->pChildren != NULL) { diff --git a/source/libs/executor/src/streamtimesliceoperator.c b/source/libs/executor/src/streamtimesliceoperator.c index 681e07f452..f511c5ab4f 100644 --- a/source/libs/executor/src/streamtimesliceoperator.c +++ b/source/libs/executor/src/streamtimesliceoperator.c @@ -151,7 +151,7 @@ void destroyStreamTimeSliceOperatorInfo(void* param) { pInfo->pOperator = NULL; } colDataDestroy(&pInfo->twAggSup.timeWindowData); - destroyStreamAggSupporter(&pInfo->streamAggSup); + resetPrevAndNextWindow(pInfo->pFillSup); destroyStreamFillSupporter(pInfo->pFillSup); destroyStreamFillInfo(pInfo->pFillInfo); @@ -179,6 +179,7 @@ void destroyStreamTimeSliceOperatorInfo(void* param) { taosArrayDestroy(pInfo->historyWins); taosArrayDestroy(pInfo->pCloseTs); + destroyStreamAggSupporter(&pInfo->streamAggSup); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 33efd0cfb1..c15a9f9224 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -462,7 +462,7 @@ _end: void destroyFlusedPos(void* pRes) { SRowBuffPos* pPos = (SRowBuffPos*)pRes; - if (!pPos->needFree && !pPos->pRowBuff) { + if (pPos->needFree && !pPos->pRowBuff) { taosMemoryFreeClear(pPos->pKey); taosMemoryFree(pPos); } @@ -475,9 +475,11 @@ void destroyFlusedppPos(void* ppRes) { void clearGroupResInfo(SGroupResInfo* pGroupResInfo) { int32_t size = taosArrayGetSize(pGroupResInfo->pRows); - for (int32_t i = pGroupResInfo->index; i < size; i++) { - void* pPos = taosArrayGetP(pGroupResInfo->pRows, i); - destroyFlusedPos(pPos); + if (pGroupResInfo->index >= 0 && pGroupResInfo->index < size) { + for (int32_t i = pGroupResInfo->index; i < size; i++) { + void* pPos = taosArrayGetP(pGroupResInfo->pRows, i); + destroyFlusedPos(pPos); + } } pGroupResInfo->freeItem = false; taosArrayDestroy(pGroupResInfo->pRows); @@ -2204,11 +2206,12 @@ void destroyStreamSessionAggOperatorInfo(void* param) { } destroyStreamBasicInfo(&pInfo->basic); - destroyStreamAggSupporter(&pInfo->streamAggSup); + cleanupExprSupp(&pInfo->scalarSupp); clearGroupResInfo(&pInfo->groupResInfo); taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos); pInfo->pUpdated = NULL; + destroyStreamAggSupporter(&pInfo->streamAggSup); if (pInfo->pChildren != NULL) { int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -4442,10 +4445,11 @@ void destroyStreamStateOperatorInfo(void* param) { } destroyStreamBasicInfo(&pInfo->basic); - destroyStreamAggSupporter(&pInfo->streamAggSup); + clearGroupResInfo(&pInfo->groupResInfo); taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos); pInfo->pUpdated = NULL; + destroyStreamAggSupporter(&pInfo->streamAggSup); cleanupExprSupp(&pInfo->scalarSupp); if (pInfo->pChildren != NULL) { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index be3f0d362b..8233b2d450 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -22,6 +22,9 @@ #include "tanalytics.h" #include "taoserror.h" #include "ttime.h" +#include "functionMgt.h" +#include "ttypes.h" +#include "tglobal.h" static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) { va_list vArgList; @@ -1745,6 +1748,62 @@ static int32_t translateHistogramPartial(SFunctionNode* pFunc, char* pErrBuf, in return TSDB_CODE_SUCCESS; } +#define NUMERIC_TO_STRINGS_LEN 25 +static int32_t translateGreatestleast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + FUNC_ERR_RET(validateParam(pFunc, pErrBuf, len)); + + bool mixTypeToStrings = tsCompareAsStrInGreatest; + + SDataType res = {.type = 0}; + bool resInit = false; + for (int32_t i = 0; i < LIST_LENGTH(pFunc->pParameterList); i++) { + SDataType* para = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, i)); + + if (IS_NULL_TYPE(para->type)) { + res.type = TSDB_DATA_TYPE_NULL; + res.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; + break; + } + + if (!resInit) { + res.type = para->type; + res.bytes = para->bytes; + resInit = true; + continue; + } + + if (IS_MATHABLE_TYPE(para->type)) { + if (res.type == para->type) { + continue; + } else if (IS_MATHABLE_TYPE(res.type) || !mixTypeToStrings) { + int32_t resType = vectorGetConvertType(res.type, para->type); + res.type = resType == 0 ? res.type : resType; + res.bytes = tDataTypes[res.type].bytes; + } else { + // last res is strings, para is numeric and mixTypeToStrings is true + res.bytes = TMAX(res.bytes, NUMERIC_TO_STRINGS_LEN); + } + } else { + if (IS_COMPARE_STR_DATA_TYPE(res.type)) { + int32_t resType = vectorGetConvertType(res.type, para->type); + res.type = resType == 0 ? res.type : resType; + res.bytes = TMAX(res.bytes, para->bytes); + } else if (mixTypeToStrings) { + // last res is numeric, para is string, and mixTypeToStrings is true + res.type = para->type; + res.bytes = TMAX(para->bytes, NUMERIC_TO_STRINGS_LEN); + } else { + // last res is numeric, para is string, and mixTypeToStrings is false + int32_t resType = vectorGetConvertType(res.type, para->type); + res.type = resType == 0 ? res.type : resType; + res.bytes = tDataTypes[resType].bytes; + } + } + } + pFunc->node.resType = res; + return TSDB_CODE_SUCCESS; +} + // clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { @@ -5656,6 +5715,48 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "cols", .translateFunc = invalidColsFunction, }, + { + .name = "greatest", + .type = FUNCTION_TYPE_GREATEST, + .classification = FUNC_MGT_SCALAR_FUNC, + .parameters = {.minParamNum = 2, + .maxParamNum = -1, + .paramInfoPattern = 1, + .inputParaInfo[0][0] = {.isLastParam = true, + .startParam = 1, + .endParam = -1, + .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE | FUNC_PARAM_SUPPORT_BOOL_TYPE | FUNC_PARAM_SUPPORT_TIMESTAMP_TYPE | FUNC_PARAM_SUPPORT_VARCHAR_TYPE | FUNC_PARAM_SUPPORT_NCHAR_TYPE, + .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE, + .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE, + .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,}, + .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_ALL_TYPE}}, + .translateFunc = translateGreatestleast, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = greatestFunction, + .finalizeFunc = NULL + }, + { + .name = "least", + .type = FUNCTION_TYPE_LEAST, + .classification = FUNC_MGT_SCALAR_FUNC, + .parameters = {.minParamNum = 2, + .maxParamNum = -1, + .paramInfoPattern = 1, + .inputParaInfo[0][0] = {.isLastParam = true, + .startParam = 1, + .endParam = -1, + .validDataType = FUNC_PARAM_SUPPORT_NUMERIC_TYPE | FUNC_PARAM_SUPPORT_NULL_TYPE | FUNC_PARAM_SUPPORT_BOOL_TYPE | FUNC_PARAM_SUPPORT_TIMESTAMP_TYPE | FUNC_PARAM_SUPPORT_VARCHAR_TYPE | FUNC_PARAM_SUPPORT_NCHAR_TYPE, + .validNodeType = FUNC_PARAM_SUPPORT_EXPR_NODE, + .paramAttribute = FUNC_PARAM_NO_SPECIFIC_ATTRIBUTE, + .valueRangeFlag = FUNC_PARAM_NO_SPECIFIC_VALUE,}, + .outputParaInfo = {.validDataType = FUNC_PARAM_SUPPORT_ALL_TYPE}}, + .translateFunc = translateGreatestleast, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = leastFunction, + .finalizeFunc = NULL + }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 91be791dff..876b05d55f 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -797,7 +797,7 @@ static bool funcNotSupportStringSma(SFunctionNode* pFunc) { } EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { - if(funcNotSupportStringSma(pFunc)) { + if (funcNotSupportStringSma(pFunc)) { return FUNC_DATA_REQUIRED_DATA_LOAD; } return FUNC_DATA_REQUIRED_SMA_LOAD; @@ -6611,7 +6611,7 @@ int32_t blockDBUsageFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { double compressRadio = 0; if (rawDataSize != 0) { compressRadio = totalDiskSize * 100 / (double)rawDataSize; - len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f]", compressRadio); + len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f%]", compressRadio); } else { len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[NULL]"); } diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 1d1bc66414..257ad3d8ea 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -532,9 +532,17 @@ int32_t sifStr2Num(char *buf, int32_t len, int8_t type, void *val) { static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { int32_t code = 0; int8_t ltype = left->colValType, rtype = right->colValType; - if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) { - return TSDB_CODE_INVALID_PARA; + // if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) { + // return TSDB_CODE_INVALID_PARA; + // } + if (IS_VAR_DATA_TYPE(ltype)) { + if (ltype == TSDB_DATA_TYPE_VARCHAR || ltype == TSDB_DATA_TYPE_BINARY || ltype == TSDB_DATA_TYPE_VARBINARY) { + return 0; + } else { + return TSDB_CODE_INVALID_PARA; + } } + if (ltype == TSDB_DATA_TYPE_FLOAT) { float f = 0; if (IS_NUMERIC_TYPE(rtype)) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index f3b56da372..280a469153 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1,11 +1,14 @@ +#include #include "cJSON.h" #include "function.h" #include "scalar.h" #include "sclInt.h" #include "sclvector.h" #include "tdatablock.h" +#include "tdef.h" #include "tjson.h" #include "ttime.h" +#include "filter.h" typedef float (*_float_fn)(float); typedef float (*_float_fn_2)(float, float); @@ -4403,3 +4406,136 @@ int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam return selectScalarFunction(pInput, inputNum, pOutput); } +typedef struct SCovertScarlarParam { + SScalarParam covertParam; + SScalarParam *param; + bool converted; +} SCovertScarlarParam; + +void freeSCovertScarlarParams(SCovertScarlarParam *pCovertParams, int32_t num) { + if (pCovertParams == NULL) { + return; + } + for (int32_t i = 0; i < num; i++) { + if (pCovertParams[i].converted) { + sclFreeParam(pCovertParams[i].param); + } + } + taosMemoryFree(pCovertParams); +} + +static int32_t vectorCompareAndSelect(SCovertScarlarParam *pParams, int32_t numOfRows, int numOfCols, + int32_t *resultColIndex, EOperatorType optr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t type = GET_PARAM_TYPE(pParams[0].param); + + __compar_fn_t fp = NULL; + code = filterGetCompFunc(&fp, type, optr); + if(code != TSDB_CODE_SUCCESS) { + qError("failed to get compare function, func:%s type:%d, optr:%d", __FUNCTION__, type, optr); + return code; + } + + for (int32_t i = 0; i < numOfRows; i++) { + int selectIndex = 0; + if (colDataIsNull_s(pParams[selectIndex].param->columnData, i)) { + resultColIndex[i] = -1; + continue; + } + for (int32_t j = 1; j < numOfCols; j++) { + if (colDataIsNull_s(pParams[j].param->columnData, i)) { + resultColIndex[i] = -1; + break; + } else { + int32_t leftRowNo = pParams[selectIndex].param->numOfRows == 1 ? 0 : i; + int32_t rightRowNo = pParams[j].param->numOfRows == 1 ? 0 : i; + char *pLeftData = colDataGetData(pParams[selectIndex].param->columnData, leftRowNo); + char *pRightData = colDataGetData(pParams[j].param->columnData, rightRowNo); + bool pRes = filterDoCompare(fp, optr, pLeftData, pRightData); + if (!pRes) { + selectIndex = j; + } + } + resultColIndex[i] = selectIndex; + } + } + + return code; +} + +static int32_t greatestLeastImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, EOperatorType order) { + int32_t code = TSDB_CODE_SUCCESS; + SColumnInfoData *pOutputData = pOutput[0].columnData; + int16_t outputType = GET_PARAM_TYPE(&pOutput[0]); + int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]); + + SCovertScarlarParam *pCovertParams = NULL; + int32_t *resultColIndex = NULL; + + int32_t numOfRows = 0; + bool IsNullType = outputType == TSDB_DATA_TYPE_NULL ? true : false; + // If any column is NULL type, the output is NULL type + for (int32_t i = 0; i < inputNum; i++) { + if (IsNullType) { + break; + } + if (numOfRows != 0 && numOfRows != pInput[i].numOfRows && pInput[i].numOfRows != 1 && numOfRows != 1) { + qError("input rows not match, func:%s, rows:%d, %d", __FUNCTION__, numOfRows, pInput[i].numOfRows); + code = TSDB_CODE_TSC_INTERNAL_ERROR; + goto _return; + } + numOfRows = TMAX(numOfRows, pInput[i].numOfRows); + IsNullType |= IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i])); + } + + if (IsNullType) { + colDataSetNNULL(pOutputData, 0, numOfRows); + pOutput->numOfRows = numOfRows; + return TSDB_CODE_SUCCESS; + } + pCovertParams = taosMemoryMalloc(inputNum * sizeof(SCovertScarlarParam)); + for (int32_t j = 0; j < inputNum; j++) { + SScalarParam *pParam = &pInput[j]; + int16_t oldType = GET_PARAM_TYPE(&pInput[j]); + if (oldType != outputType) { + pCovertParams[j].covertParam = (SScalarParam){0}; + setTzCharset(&pCovertParams[j].covertParam, pParam->tz, pParam->charsetCxt); + SCL_ERR_JRET(vectorConvertSingleCol(pParam, &pCovertParams[j].covertParam, outputType, 0, pParam->numOfRows)); + pCovertParams[j].param = &pCovertParams[j].covertParam; + pCovertParams[j].converted = true; + } else { + pCovertParams[j].param = pParam; + pCovertParams[j].converted = false; + } + } + + resultColIndex = taosMemoryCalloc(numOfRows, sizeof(int32_t)); + SCL_ERR_JRET(vectorCompareAndSelect(pCovertParams, numOfRows, inputNum, resultColIndex, order)); + + for (int32_t i = 0; i < numOfRows; i++) { + int32_t index = resultColIndex[i]; + if (index == -1) { + colDataSetNULL(pOutputData, i); + continue; + } + int32_t rowNo = pCovertParams[index].param->numOfRows == 1 ? 0 : i; + char *data = colDataGetData(pCovertParams[index].param->columnData, rowNo); + SCL_ERR_JRET(colDataSetVal(pOutputData, i, data, false)); + } + + pOutput->numOfRows = numOfRows; + +_return: + freeSCovertScarlarParams(pCovertParams, inputNum); + taosMemoryFree(resultColIndex); + return code; +} + +int32_t greatestFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return greatestLeastImpl(pInput, inputNum, pOutput, OP_TYPE_GREATER_THAN); +} + +int32_t leastFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return greatestLeastImpl(pInput, inputNum, pOutput, OP_TYPE_LOWER_THAN); +} + diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 14dae1226d..ff45baa1e0 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -996,7 +996,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = { - /*NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ + /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 5, 11, 12, 13, 14, 0, -1, 0, 0, 0, -1, /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 5, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1, @@ -1021,7 +1021,7 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = { }; int8_t gDisplyTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = { - /*NULL BOOL TINY SMAL INT BIGI FLOA DOUB VARC TIM NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ + /* NULL BOOL TINY SMAL INT BIGI FLOA DOUB VARC TIM NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ /*NULL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, -1, -1, -1, 20, /*BOOL*/ 0, 1, 2, 3, 4, 5, 6, 7, 8, 5, 10, 11, 12, 13, 14, -1, -1, -1, -1, -1, -1, /*TINY*/ 0, 0, 2, 3, 4, 5, 8, 8, 8, 5, 10, 3, 4, 5, 8, -1, -1, -1, -1, -1, -1, diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index ebb13654b7..29da3fc88e 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -717,7 +717,7 @@ void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList) { // The restart of all tasks requires that all tasks should not have active timer for now. Therefore, the execution // of restart in timer thread will result in a deadlock. int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId) { - return streamTaskSchedTask(pMsgCb, vgId, streamId, taskId, STREAM_EXEC_T_ADD_FAILED_TASK); + return streamTaskSchedTask(pMsgCb, vgId, streamId, taskId, STREAM_EXEC_T_ADD_FAILED_TASK, false); } static void doCleanup(SStreamTask* pTask, SArray* pNotReadyList, SArray* pTimeoutList, void* param) { diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index eb8f2c741a..afda288b90 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -93,7 +93,7 @@ int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, i return TSDB_CODE_OUT_OF_MEMORY; } - return streamTrySchedExec(pTask); + return streamTrySchedExec(pTask, true); } int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) { @@ -814,15 +814,17 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l } int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t dbRefId, ECHECKPOINT_BACKUP_TYPE type) { - int32_t code = 0; - char* path = NULL; - + int32_t code = 0; + char* path = NULL; + int64_t chkptSize = 0; SStreamMeta* pMeta = pTask->pMeta; const char* idStr = pTask->id.idStr; int64_t now = taosGetTimestampMs(); SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES); if (toDelFiles == NULL) { + stError("s-task:%s failed to prepare array list during upload checkpoint, code:%s", pTask->id.idStr, + tstrerror(terrno)); return terrno; } @@ -848,11 +850,11 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d } } - if (code == TSDB_CODE_SUCCESS) { - int32_t size = taosArrayGetSize(toDelFiles); - stDebug("s-task:%s remove redundant %d files", idStr, size); + int32_t num = taosArrayGetSize(toDelFiles); + if (code == TSDB_CODE_SUCCESS && num > 0) { + stDebug("s-task:%s remove redundant %d files", idStr, num); - for (int i = 0; i < size; i++) { + for (int i = 0; i < num; i++) { char* pName = taosArrayGetP(toDelFiles, i); code = deleteCheckpointFile(idStr, pName); if (code != 0) { @@ -868,12 +870,13 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d double el = (taosGetTimestampMs() - now) / 1000.0; if (code == TSDB_CODE_SUCCESS) { - stDebug("s-task:%s complete update checkpointId:%" PRId64 ", elapsed time:%.2fs remove local checkpoint data %s", - idStr, checkpointId, el, path); - taosRemoveDir(path); + code = taosGetDirSize(path, &chkptSize); + stDebug("s-task:%s complete upload checkpointId:%" PRId64 + ", elapsed time:%.2fs, checkpointSize:%.2fKiB local dir:%s", + idStr, checkpointId, el, SIZE_IN_KiB(chkptSize), path); } else { - stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " keep local checkpoint data, elapsed time:%.2fs", idStr, - checkpointId, el); + stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " elapsed time:%.2fs, checkpointSize:%.2fKiB", idStr, + checkpointId, el, SIZE_IN_KiB(chkptSize)); } taosMemoryFree(path); @@ -883,7 +886,7 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t checkpointId) { ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType(); if (type == DATA_UPLOAD_DISABLE) { - stDebug("s-task:%s not allowed to upload checkpoint data", pTask->id.idStr); + stDebug("s-task:%s not config to backup checkpoint data at snode, checkpointId:%"PRId64, pTask->id.idStr, checkpointId); return 0; } @@ -925,6 +928,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s gen checkpoint:%" PRId64 " failed, code:%s", id, ckId, tstrerror(terrno)); } + + int64_t et = taosGetTimestampMs(); + stDebug("s-task:%s gen local checkpoint completed, elapsed time:%.2fs", id, (et - startTs) / 1000.0); } // TODO: monitoring the checkpoint-source msg diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 24ac193937..f08b4a7922 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1834,6 +1834,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S int32_t status = 0; SStreamMeta* pMeta = pTask->pMeta; const char* id = pTask->id.idStr; + bool chkptMsg = false; stDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64 ", msgId:%d", id, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId); @@ -1863,6 +1864,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S // blocked. Note that there is no race condition here. if (pReq->type == STREAM_INPUT__CHECKPOINT_TRIGGER) { streamTaskCloseUpstreamInput(pTask, pReq->upstreamTaskId); + chkptMsg = true; stDebug("s-task:%s close inputQ for upstream:0x%x, msgId:%d", id, pReq->upstreamTaskId, pReq->msgId); } else if (pReq->type == STREAM_INPUT__TRANS_STATE) { stDebug("s-task:%s recv trans-state msgId:%d from upstream:0x%x", id, pReq->msgId, pReq->upstreamTaskId); @@ -1890,5 +1892,5 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S tmsgSendRsp(pRsp); } - return streamTrySchedExec(pTask); + return streamTrySchedExec(pTask, chkptMsg); } diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index ca5b6630fd..a6d0142010 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -243,6 +243,8 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { continue; } + // todo: this lock may blocked by lock in streamMetaStartOneTask function, which may lock a very long time when + // trying to load remote checkpoint data streamMutexLock(&pTask->lock); STaskStatusEntry entry = streamTaskGetStatusEntry(pTask); streamMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 605cf3fe21..350c71204a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1364,6 +1364,10 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) } pMeta->role = (isLeader) ? NODE_ROLE_LEADER : NODE_ROLE_FOLLOWER; + if (!isLeader) { + streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId); + } + streamMetaWUnLock(pMeta); if (isLeader) { diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 7c7ef83d6d..5a507358ee 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -144,11 +144,20 @@ void streamQueueNextItemInSourceQ(SStreamQueue* pQueue, SStreamQueueItem** pItem // let's try the ordinary input q pQueue->qItem = NULL; - int32_t num = taosGetQitem(pQueue->qall, &pQueue->qItem); + int32_t code = taosGetQitem(pQueue->qall, &pQueue->qItem); + if (code) { + stError("s-task:%s failed to extract data from inputQ, code:%s", id, tstrerror(code)); + } if (pQueue->qItem == NULL) { - num = taosReadAllQitems(pQueue->pQueue, pQueue->qall); - num = taosGetQitem(pQueue->qall, &pQueue->qItem); + code = taosReadAllQitems(pQueue->pQueue, pQueue->qall); + if (code) { + stError("s-task:%s failed to read qitem into qall, code:%s", id, tstrerror(code)); + } + code = taosGetQitem(pQueue->qall, &pQueue->qItem); + if (code) { + stError("s-task:%s failed to extract data from inputQ(qall), code:%s", id, tstrerror(code)); + } } *pItem = streamQueueCurItem(pQueue); diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 1f76f349ae..367e54d1a1 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -93,17 +93,21 @@ void streamSetupScheduleTrigger(SStreamTask* pTask) { } } -int32_t streamTrySchedExec(SStreamTask* pTask) { +int32_t streamTrySchedExec(SStreamTask* pTask, bool chkptQueue) { if (streamTaskSetSchedStatusWait(pTask)) { - return streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId, 0); + return streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId, 0, chkptQueue); } else { - stTrace("s-task:%s not launch task since sched status:%d", pTask->id.idStr, pTask->status.schedStatus); + if (chkptQueue) { + stWarn("s-task:%s not launch task in chkpt queue, may delay checkpoint procedure", pTask->id.idStr); + } else { + stTrace("s-task:%s not launch task since sched status:%d", pTask->id.idStr, pTask->status.schedStatus); + } } return 0; } -int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType) { +int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType, bool chkptExec) { int32_t code = 0; int32_t tlen = 0; @@ -142,10 +146,18 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3 stDebug("vgId:%d create msg to exec, type:%d, %s", vgId, execType, streamTaskGetExecType(execType)); } - SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = buf, .contLen = tlen + sizeof(SMsgHead)}; - code = tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); - if (code) { - stError("vgId:%d failed to put msg into stream queue, code:%s, %x", vgId, tstrerror(code), taskId); + if (chkptExec) { + SRpcMsg msg = {.msgType = TDMT_STREAM_CHKPT_EXEC, .pCont = buf, .contLen = tlen + sizeof(SMsgHead)}; + code = tmsgPutToQueue(pMsgCb, STREAM_CHKPT_QUEUE, &msg); + if (code) { + stError("vgId:%d failed to put msg into stream chkpt queue, code:%s, %x", vgId, tstrerror(code), taskId); + } + } else { + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = buf, .contLen = tlen + sizeof(SMsgHead)}; + code = tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); + if (code) { + stError("vgId:%d failed to put msg into stream queue, code:%s, %x", vgId, tstrerror(code), taskId); + } } return code; } @@ -191,12 +203,17 @@ void streamTaskResumeHelper(void* param, void* tmrId) { return; } - code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); + code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK, + (p.state == TASK_STATUS__CK)); if (code) { stError("s-task:%s sched task failed, code:%s", pId->idStr, tstrerror(code)); } else { - stDebug("trigger to resume s-task:%s after idled for %dms", pId->idStr, pTask->status.schedIdleTime); - + if (p.state == TASK_STATUS__CK) { + stDebug("trigger to resume s-task:%s in stream chkpt queue after idled for %dms", pId->idStr, + pTask->status.schedIdleTime); + } else { + stDebug("trigger to resume s-task:%s after idled for %dms", pId->idStr, pTask->status.schedIdleTime); + } // release the task ref count streamTaskClearSchedIdleInfo(pTask); } @@ -339,7 +356,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { } } - code = streamTrySchedExec(pTask); + code = streamTrySchedExec(pTask, false); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s failed to sched to run, wait for next time", pTask->id.idStr); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 378aaa27d0..f7ae4915ec 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -1194,7 +1194,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) { if (code != 0) { return code; } - return streamTrySchedExec(pTask); + return streamTrySchedExec(pTask, false); } void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) { pTask->status.removeBackendFiles = true; } diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index d6dfde1ee6..0255ac01b5 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -394,14 +394,17 @@ int32_t clearFlushedRowBuff(SStreamFileState* pFileState, SStreamSnapshot* pFlus pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey)); pFileState->stateBuffRemoveByPosFn(pFileState, pPos); - SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode); - taosMemoryFreeClear(tmp); + if (pPos->beUsed == false) { + SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode); + taosMemoryFreeClear(tmp); + } if (pPos->pRowBuff) { i++; } } } } + qDebug("clear flushed row buff. %d rows to disk. is all:%d", listNEles(pFlushList), all); _end: if (code != TSDB_CODE_SUCCESS) { @@ -433,7 +436,6 @@ int32_t popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; if (pPos->beUsed == used) { if (used && !pPos->pRowBuff) { - QUERY_CHECK_CONDITION((pPos->needFree == true), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); continue; } code = tdListAppend(pFlushList, &pPos); @@ -441,8 +443,10 @@ int32_t popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey)); pFileState->stateBuffRemoveByPosFn(pFileState, pPos); - SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode); - taosMemoryFreeClear(tmp); + if (pPos->beUsed == false) { + SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode); + taosMemoryFreeClear(tmp); + } if (pPos->pRowBuff) { i++; } @@ -511,9 +515,12 @@ int32_t clearRowBuff(SStreamFileState* pFileState) { if (pFileState->deleteMark != INT64_MAX) { clearExpiredRowBuff(pFileState, pFileState->maxTs - pFileState->deleteMark, false); } - if (isListEmpty(pFileState->freeBuffs)) { - return flushRowBuff(pFileState); - } + do { + int32_t code = flushRowBuff(pFileState); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } while (isListEmpty(pFileState->freeBuffs) && pFileState->curRowCount == pFileState->maxRowCount); return TSDB_CODE_SUCCESS; } @@ -756,10 +763,10 @@ int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** QUERY_CHECK_CODE(code, lino, _end); (*pVal) = pPos->pRowBuff; - if (!pPos->needFree) { - code = tdListPrepend(pFileState->usedBuffs, &pPos); - QUERY_CHECK_CODE(code, lino, _end); - } + // if (!pPos->needFree) { + // code = tdListPrepend(pFileState->usedBuffs, &pPos); + // QUERY_CHECK_CODE(code, lino, _end); + // } _end: if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 8d81a03344..34f48d5960 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -733,9 +733,11 @@ int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTe if (retry) { taosMsleep(10); if (code == TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE) { - sError("vgId:%d, failed to execute fsm since %s. index:%" PRId64, pNode->vgId, terrstr(), pEntry->index); + sError("vgId:%d, will retry to execute fsm after 10ms, last error is %s. index:%" PRId64, pNode->vgId, + tstrerror(code), pEntry->index); } else { - sDebug("vgId:%d, retry on fsm commit since %s. index:%" PRId64, pNode->vgId, terrstr(), pEntry->index); + sDebug("vgId:%d, will retry to execute fsm after 10ms, last error is %s. index:%" PRId64, pNode->vgId, + tstrerror(code), pEntry->index); } } } while (retry); diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index b0bcbd1a4c..23e6e5414e 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -345,7 +345,9 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (!pPage && pTxn->xMalloc != NULL) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { - tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + // when allocating from bufpool failed, it's time to flush cache. + // tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + terrno = ret; return NULL; } @@ -551,5 +553,4 @@ static void tdbPCacheCloseImpl(SPCache *pCache) { tdbOsFree(pCache->pgHash); tdbPCacheDestroyLock(pCache); - return ; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 6dc6aa0940..4c73d6470e 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -104,7 +104,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF } #endif - + /* if (rollback) { ret = tdbPagerRestoreJournals(pPager); if (ret < 0) { @@ -118,6 +118,13 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF return ret; } } + */ + // Always restore journal files with page flushing + ret = tdbPagerRestoreJournals(pPager); + if (ret < 0) { + tdbOsFree(pTb); + return ret; + } // pTb->pBt ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index 4715ccbd41..6319fce86b 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -18,3 +18,6 @@ target_link_libraries(tdbPageDefragmentTest tdb gtest gtest_main) add_executable(tdbPageRecycleTest "tdbPageRecycleTest.cpp") target_link_libraries(tdbPageRecycleTest tdb gtest gtest_main) +# page flush testing +add_executable(tdbPageFlushTest "tdbPageFlushTest.cpp") +target_link_libraries(tdbPageFlushTest tdb gtest gtest_main) diff --git a/source/libs/tdb/test/tdbPageFlushTest.cpp b/source/libs/tdb/test/tdbPageFlushTest.cpp new file mode 100644 index 0000000000..170a28af76 --- /dev/null +++ b/source/libs/tdb/test/tdbPageFlushTest.cpp @@ -0,0 +1,582 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#define ALLOW_FORBID_FUNC +#include "os.h" +#include "tdb.h" + +#include +#include +#include +#include +#include "tlog.h" + +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + clearPool(pPool); + taosMemoryFree(pPool); +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void *poolMallocRestricted(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + if (pPool->size > 1024 * 1024 * 10) { + return NULL; + } + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { + TDB *pEnv = NULL; + + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL); + if (ret) { + pEnv = NULL; + } + + return pEnv; +} + +static void clearDb(char const *db) { taosRemoveDir(db); } + +static void generateBigVal(char *val, int valLen) { + for (int i = 0; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } +} + +static void insertOfp(void) { + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + char const *key = "key123456789"; + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +static void insertMultipleOfp(void) { + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest) { +TEST(TdbPageFlushTest, TbRestoreTest) { + clearDb("tdb"); + + insertMultipleOfp(); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest2) { +TEST(TdbPageFlushTest, TbRestoreTest2) { + clearDb("tdb"); + + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest3) { +TEST(TdbPageFlushTest, TbRestoreTest3) { + clearDb("tdb"); + + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbAbort(pEnv, txn); + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest4) { +TEST(TdbPageFlushTest, TbRestoreTest4) { + clearDb("tdb"); + + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbAbort(pEnv, txn); + closePool(pPool); + tdbTbClose(pDb); + tdbClose(pEnv); + + pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + pPool = openPool(); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest5) { +TEST(TdbPageFlushTest, TbRestoreTest5) { + clearDb("tdb"); + + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + closePool(pPool); + tdbTbClose(pDb); + tdbClose(pEnv); + + pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + pPool = openPool(); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} diff --git a/source/util/src/mpChunk.c b/source/util/src/mpChunk.c index 2c1c415c04..c7bdcd809f 100755 --- a/source/util/src/mpChunk.c +++ b/source/util/src/mpChunk.c @@ -190,6 +190,8 @@ int32_t mpChunkNSAllocMem(SMemPool* pPool, SMPSession* pSession, int64_t size, u void* pRes = NULL; int64_t totalSize = size + sizeof(SMPMemHeader) + sizeof(SMPMemTailer) + alignment; + + MP_ERR_JRET(mpChunkNewNS(pPool, &pChunk, totalSize)); SMPMemHeader* pHeader = (SMPMemHeader*)pChunk->pMemStart; MP_INIT_MEM_HEADER(pHeader, size, false); diff --git a/tests/army/cluster/arbitrator_election.py b/tests/army/cluster/arbitrator_election.py new file mode 100644 index 0000000000..7883e162a7 --- /dev/null +++ b/tests/army/cluster/arbitrator_election.py @@ -0,0 +1,97 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.srvCtl import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +from frame import epath +# from frame.server.dnodes import * +# from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=1): + updatecfgDict = {'dDebugFlag':131} + super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1") + + self.valgrind = 0 + self.db = "test" + self.stb = "meters" + self.childtable_count = 10 + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.execute('CREATE DATABASE db vgroups 1 replica 2;') + + time.sleep(1) + + tdSql.execute("use db;") + + tdSql.execute("CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);") + + tdSql.execute("CREATE TABLE d0 USING meters TAGS (\"California.SanFrancisco\", 2);"); + + count = 0 + + while count < 100: + tdSql.query("show arbgroups;") + + if tdSql.getData(0, 4) == True: + break + + tdLog.info("wait %d seconds for is sync"%count) + time.sleep(1) + + count += 1 + + if count == 100: + tdLog.exit("arbgroup sync failed") + return + + + tdSql.query("show db.vgroups;") + + if(tdSql.getData(0, 4) == "follower") and (tdSql.getData(0, 7) == "leader"): + tdLog.info("stop dnode2") + sc.dnodeStop(2) + + if(tdSql.getData(0, 7) == "follower") and (tdSql.getData(0, 4) == "leader"): + tdLog.info("stop dnode 3") + sc.dnodeStop(3) + + + count = 0 + while count < 11: + tdSql.query("show db.vgroups;") + + if(tdSql.getData(0, 4) == "assigned ") or (tdSql.getData(0, 7) == "assigned "): + break + + tdLog.info("wait %d seconds for set assigned"%count) + time.sleep(1) + + count += 1 + + if count == 11: + tdLog.exit("check assigned failed") + return + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 4427ddcea5..98007c6d8d 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -19,6 +19,7 @@ import random import copy import json +import frame.eos import frame.etool import frame.eutil from frame.log import * @@ -427,6 +428,42 @@ class TBase: return db, stb, child_count, insert_rows + # insert & check + def benchInsert(self, jsonFile, options = "", results = None): + # exe insert + benchmark = frame.etool.benchMarkFile() + cmd = f"{benchmark} {options} -f {jsonFile}" + rlist = frame.eos.runRetList(cmd, True, True, True) + if results != None: + for result in results: + self.checkListString(rlist, result) + + # open json + with open(jsonFile, "r") as file: + data = json.load(file) + + # read json + dbs = data["databases"] + for db in dbs: + dbName = db["dbinfo"]["name"] + stbs = db["super_tables"] + for stb in stbs: + stbName = stb["name"] + child_count = stb["childtable_count"] + insert_rows = stb["insert_rows"] + timestamp_step = stb["timestamp_step"] + + # check result + + # count + sql = f"select count(*) from {dbName}.{stbName}" + tdSql.checkAgg(sql, child_count * insert_rows) + # diff + sql = f"select * from (select diff(ts) as dif from {dbName}.{stbName} partition by tbname) where dif != {timestamp_step};" + tdSql.query(sql) + tdSql.checkRows(0) + # show + tdLog.info(f"insert check passed. db:{dbName} stb:{stbName} child_count:{child_count} insert_rows:{insert_rows}\n") # tmq def tmqBenchJson(self, jsonFile, options="", checkStep=False): diff --git a/tests/army/query/function/ans/ascii.csv b/tests/army/query/function/ans/ascii.csv index 8d186f63e3..0e67acc9d1 100644 --- a/tests/army/query/function/ans/ascii.csv +++ b/tests/army/query/function/ans/ascii.csv @@ -51,7 +51,7 @@ taos> select ASCII('hello') + 1 from ts_4893.meters limit 1 taos> select ASCII('hello') + ASCII('hello') from ts_4893.meters limit 1 ascii('hello') + ascii('hello') | ================================== - 2.080000000000000e+02 | + 208 | taos> select ASCII(nch1) from ts_4893.meters order by ts limit 5 ascii(nch1) | diff --git a/tests/army/query/function/ans/char_length.csv b/tests/army/query/function/ans/char_length.csv index 4b79a42bfa..a8352357cf 100644 --- a/tests/army/query/function/ans/char_length.csv +++ b/tests/army/query/function/ans/char_length.csv @@ -51,7 +51,7 @@ taos> select CHAR_LENGTH('hello') + 1 from ts_4893.meters limit 1 taos> select CHAR_LENGTH('hello') + CHAR_LENGTH('hello') from ts_4893.meters limit 1 char_length('hello') + char_length('hello') | ============================================== - 1.000000000000000e+01 | + 10 | taos> select CHAR_LENGTH(nch1) from ts_4893.meters order by ts limit 5 char_length(nch1) | diff --git a/tests/army/query/function/ans/greatest.csv b/tests/army/query/function/ans/greatest.csv new file mode 100644 index 0000000000..b6854a95d6 --- /dev/null +++ b/tests/army/query/function/ans/greatest.csv @@ -0,0 +1,1409 @@ + +taos> alter local 'compareAsStrInGreatest' '1'; + +taos> select GREATEST(1,2,3,4,5,6,7,8,9,10); + greatest(1,2,3,4,5,6,7,8,9,10) | +================================= + 10 | + +taos> select GREATEST(1,1.1,2.23,3.4444,5.66666666,6.21241241,7.999999999999); + greatest(1,1.1,2.23,3.4444,5.66666666,6.21241241,7.999999999999) | +=================================================================== + 7.999999999999 | + +taos> select GREATEST(1,'2',3.3,4.4,5); + greatest(1,'2',3.3,4.4,5) | +============================ + 5 | + +taos> select GREATEST(121,'18'); + greatest(121,'18') | +============================ + 18 | + +taos> select GREATEST(18888,'18'); + greatest(18888,'18') | +============================ + 18888 | + +taos> select GREATEST(1,2,3,4,5,'5.1'); + greatest(1,2,3,4,5,'5.1') | +============================ + 5.1 | + +taos> select GREATEST('1','2','3','4',5); + greatest('1','2','3','4',5) | +============================== + 5 | + +taos> select GREATEST('1','2','3','4','5'); + greatest('1','2','3','4','5') | +================================ + 5 | + +taos> select GREATEST(1,2,3,4,5,6,7,'a','b','一','二','三'); + greatest(1,2,3,4,5,6,7,'a','b','一','二','三') | +==================================================== + 二 | + +taos> select GREATEST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213'); + greatest(1,2,3,4,5,6,7,'a','b','c','1','2','1231213') | +======================================================== + c | + +taos> select GREATEST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.123123'); + greatest(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.12 | +=================================================================== + c | + +taos> select GREATEST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); + greatest(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint | +=================================================================== + 9 | + +taos> select GREATEST(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); + greatest(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as | +=================================================================== + 9 | + +taos> select GREATEST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20)), cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double)); + greatest(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint | +=================================================================== + 9 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as tinyint)); + greatest(cast(100 as tinyint), cast(101 as tinyint)) | +======================================================= + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as smallint)); + greatest(cast(100 as tinyint), cast(101 as smallint)) | +======================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as int)); + greatest(cast(100 as tinyint), cast(101 as int)) | +=================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as bigint)); + greatest(cast(100 as tinyint), cast(101 as bigint)) | +====================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as float)); + greatest(cast(100 as tinyint), cast(101 as float)) | +===================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as double)); + greatest(cast(100 as tinyint), cast(101 as double)) | +====================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as varchar(20))); + greatest(cast(100 as tinyint), cast(101 as varchar(20))) | +=========================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as nchar(20))); + greatest(cast(100 as tinyint), cast(101 as nchar(20))) | +========================================================= + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as tinyint)); + greatest(cast(101 as tinyint), cast(100 as tinyint)) | +======================================================= + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as smallint)); + greatest(cast(101 as tinyint), cast(100 as smallint)) | +======================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as int)); + greatest(cast(101 as tinyint), cast(100 as int)) | +=================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as bigint)); + greatest(cast(101 as tinyint), cast(100 as bigint)) | +====================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as float)); + greatest(cast(101 as tinyint), cast(100 as float)) | +===================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as double)); + greatest(cast(101 as tinyint), cast(100 as double)) | +====================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as varchar(20))); + greatest(cast(101 as tinyint), cast(100 as varchar(20))) | +=========================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as nchar(20))); + greatest(cast(101 as tinyint), cast(100 as nchar(20))) | +========================================================= + 101 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as smallint)); + greatest(cast(1000 as smallint), cast(1001 as smallint)) | +=========================================================== + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as int)); + greatest(cast(1000 as smallint), cast(1001 as int)) | +====================================================== + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as bigint)); + greatest(cast(1000 as smallint), cast(1001 as bigint)) | +========================================================= + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as float)); + greatest(cast(1000 as smallint), cast(1001 as float)) | +======================================================== + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as double)); + greatest(cast(1000 as smallint), cast(1001 as double)) | +========================================================= + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as varchar(20))); + greatest(cast(1000 as smallint), cast(1001 as varchar(20))) | +============================================================== + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as nchar(20))); + greatest(cast(1000 as smallint), cast(1001 as nchar(20))) | +============================================================ + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as smallint)); + greatest(cast(1001 as smallint), cast(1000 as smallint)) | +=========================================================== + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as int)); + greatest(cast(1001 as smallint), cast(1000 as int)) | +====================================================== + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as bigint)); + greatest(cast(1001 as smallint), cast(1000 as bigint)) | +========================================================= + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as float)); + greatest(cast(1001 as smallint), cast(1000 as float)) | +======================================================== + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as double)); + greatest(cast(1001 as smallint), cast(1000 as double)) | +========================================================= + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as varchar(20))); + greatest(cast(1001 as smallint), cast(1000 as varchar(20))) | +============================================================== + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as nchar(20))); + greatest(cast(1001 as smallint), cast(1000 as nchar(20))) | +============================================================ + 1001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as int)); + greatest(cast(1000000 as int), cast(1000001 as int)) | +======================================================= + 1000001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as bigint)); + greatest(cast(1000000 as int), cast(1000001 as bigint)) | +========================================================== + 1000001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as float)); + greatest(cast(1000000 as int), cast(1000001 as float)) | +========================================================= + 1e+06 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as double)); + greatest(cast(1000000 as int), cast(1000001 as double)) | +========================================================== + 1000001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as varchar(20))); + greatest(cast(1000000 as int), cast(1000001 as varchar(20))) | +=============================================================== + 1000001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as nchar(20))); + greatest(cast(1000000 as int), cast(1000001 as nchar(20))) | +============================================================= + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as int)); + greatest(cast(1000001 as int), cast(1000000 as int)) | +======================================================= + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as bigint)); + greatest(cast(1000001 as int), cast(1000000 as bigint)) | +========================================================== + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as float)); + greatest(cast(1000001 as int), cast(1000000 as float)) | +========================================================= + 1e+06 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as double)); + greatest(cast(1000001 as int), cast(1000000 as double)) | +========================================================== + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as varchar(20))); + greatest(cast(1000001 as int), cast(1000000 as varchar(20))) | +=============================================================== + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as nchar(20))); + greatest(cast(1000001 as int), cast(1000000 as nchar(20))) | +============================================================= + 1000001 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as bigint)); + greatest(cast(1000000000 as bigint), cast(1000000001 as bigint)) | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as float)); + greatest(cast(1000000000 as bigint), cast(1000000001 as float)) | +================================================================== + 1e+09 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as double)); + greatest(cast(1000000000 as bigint), cast(1000000001 as double)) | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as varchar(20))); + greatest(cast(1000000000 as bigint), cast(1000000001 as varchar( | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as nchar(20))); + greatest(cast(1000000000 as bigint), cast(1000000001 as nchar(20 | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as bigint)); + greatest(cast(1000000001 as bigint), cast(1000000000 as bigint)) | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as float)); + greatest(cast(1000000001 as bigint), cast(1000000000 as float)) | +================================================================== + 1e+09 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as double)); + greatest(cast(1000000001 as bigint), cast(1000000000 as double)) | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as varchar(20))); + greatest(cast(1000000001 as bigint), cast(1000000000 as varchar( | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as nchar(20))); + greatest(cast(1000000001 as bigint), cast(1000000000 as nchar(20 | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as float)); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as f | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as double)); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as d | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as timestamp)); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as t | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as varchar(20))); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as v | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as nchar(20))); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as n | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as float)); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as f | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as double)); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as d | +=================================================================== + 100001.109375 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as timestamp)); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as t | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as varchar(20))); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as v | +=================================================================== + 100001.109375 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as nchar(20))); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as n | +=================================================================== + 100001.109375 | + +taos> select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as double)); + greatest(cast(100000.1111111 as double), cast(100001.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as timestamp)); + greatest(cast(100000.1111111 as double), cast(100001.1111111 as | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as varchar(20))); + greatest(cast(100000.1111111 as double), cast(100001.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as nchar(20))); + greatest(cast(100000.1111111 as double), cast(100001.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as double)); + greatest(cast(100001.1111111 as double), cast(100000.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as timestamp)); + greatest(cast(100001.1111111 as double), cast(100000.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as varchar(20))); + greatest(cast(100001.1111111 as double), cast(100000.1111111 as | +=================================================================== + 100001.111111 | + +taos> select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as nchar(20))); + greatest(cast(100001.1111111 as double), cast(100000.1111111 as | +=================================================================== + 100001.111111 | + +taos> select GREATEST(cast('中文测试' as varchar(20)), cast('中文测试一' as varchar(20))); + greatest(cast('中文测试' as varchar(20)), cast('中文测试 | +=================================================================== + 中文测试一 | + +taos> select GREATEST(cast('中文测试' as varchar(20)), cast('中文测试一' as nchar(20))); + greatest(cast('中文测试' as varchar(20)), cast('中文测试 | +=================================================================== + 中文测试一 | + +taos> select GREATEST(cast('中文测试一' as varchar(20)), cast('中文测试' as varchar(20))); + greatest(cast('中文测试一' as varchar(20)), cast('中文测 | +=================================================================== + 中文测试一 | + +taos> select GREATEST(cast('中文测试一' as varchar(20)), cast('中文测试' as nchar(20))); + greatest(cast('中文测试一' as varchar(20)), cast('中文测 | +=================================================================== + 中文测试一 | + +taos> select GREATEST(cast('abc123abc' as varchar(20)), cast('abc124abc' as varchar(20))); + greatest(cast('abc123abc' as varchar(20)), cast('abc124abc' as v | +=================================================================== + abc124abc | + +taos> select GREATEST(cast('abc123abc' as varchar(20)), cast('abc124abc' as nchar(20))); + greatest(cast('abc123abc' as varchar(20)), cast('abc124abc' as n | +=================================================================== + abc124abc | + +taos> select GREATEST(cast('abc124abc' as varchar(20)), cast('abc123abc' as varchar(20))); + greatest(cast('abc124abc' as varchar(20)), cast('abc123abc' as v | +=================================================================== + abc124abc | + +taos> select GREATEST(cast('abc124abc' as varchar(20)), cast('abc123abc' as nchar(20))); + greatest(cast('abc124abc' as varchar(20)), cast('abc123abc' as n | +=================================================================== + abc124abc | + +taos> select GREATEST(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar(20))); + greatest(cast('abc123abc' as nchar(20)), cast('abc124abc' as nch | +=================================================================== + abc124abc | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as float), cast(102 as varchar(20))); + greatest(cast(100 as tinyint), cast(101 as float), cast(102 as v | +=================================================================== + 102 | + +taos> select GREATEST(cast(100 as float), cast(101 as tinyint), cast(102 as varchar(20))); + greatest(cast(100 as float), cast(101 as tinyint), cast(102 as v | +=================================================================== + 102 | + +taos> select GREATEST(cast(100 as float), cast(101 as varchar(20)), cast(102 as tinyint)); + greatest(cast(100 as float), cast(101 as varchar(20)), cast(102 | +=================================================================== + 102 | + +taos> select GREATEST(cast(100 as varchar(20)), cast(101 as float), cast(102 as tinyint)); + greatest(cast(100 as varchar(20)), cast(101 as float), cast(102 | +=================================================================== + 102 | + +taos> select GREATEST('a','b','c','d','e','f','g','h','1231','15155'); + greatest('a','b','c','d','e','f','g','h','1231','15155') | +=========================================================== + h | + +taos> select GREATEST(current, voltage, phase, id, nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; + greatest(current, voltage, phase, id, nch1, nch2, var1, var2) | +================================================================ + 四 | + 三a | + 四 | + 一 | + 一二三四五六七八九十 | + 一二三 | + prision | + 一二三四五六七八九十 | + prision | + 一 | + +taos> select GREATEST(current, voltage, phase, id) from ts_4893.meters order by ts limit 10; + greatest(current, voltage, phase, id) | +======================================== + 221 | + 220 | + 215 | + 216 | + 219 | + 221 | + 215 | + 217 | + 216 | + 223 | + +taos> select GREATEST(nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; + greatest(nch1, nch2, var1, var2) | +=================================== + 四 | + 三a | + 四 | + 一 | + 一二三四五六七八九十 | + 一二三 | + prision | + 一二三四五六七八九十 | + prision | + 一 | + +taos> select GREATEST(221, voltage) from ts_4893.meters order by ts limit 10; + greatest(221, voltage) | +========================= + 221 | + 221 | + 221 | + 221 | + 221 | + 221 | + 221 | + 221 | + 221 | + 223 | + +taos> select GREATEST(5, id) from ts_4893.meters order by ts limit 10; + greatest(5, id) | +======================== + 5 | + 5 | + 5 | + 5 | + 5 | + 5 | + 6 | + 7 | + 8 | + 9 | + +taos> select GREATEST('r', nch1) from ts_4893.meters order by ts limit 10; + greatest('r', nch1) | +================================= + r | + 一二三四五六七八九十 | + update | + r | + r | + r | + r | + 一二三四五六七八九十 | + r | + r | + +taos> select GREATEST('r', nch1, nch2) from ts_4893.meters order by ts limit 10; + greatest('r', nch1, nch2) | +================================= + 四 | + 一二三四五六七八九十 | + update | + 一 | + r | + 一二三 | + r | + 一二三四五六七八九十 | + r | + r | + +taos> select GREATEST('r', var1) from ts_4893.meters order by ts limit 10; + greatest('r', var1) | +================================= + r | + r | + r | + r | + 一二三四五六七八九十 | + update | + r | + r | + r | + r | + +taos> select GREATEST('r', var1, var2) from ts_4893.meters order by ts limit 10; + greatest('r', var1, var2) | +================================= + r | + 三a | + 四 | + r | + 一二三四五六七八九十 | + update | + r | + r | + r | + 一 | + +taos> select GREATEST('二中文测试', nch1) from ts_4893.meters order by ts limit 10; + greatest('二中文测试', nch1) | +==================================== + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + +taos> select GREATEST('二中文测试', nch1, nch2) from ts_4893.meters order by ts limit 10; + greatest('二中文测试', nch1, nch2) | +========================================== + 四 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + +taos> select GREATEST('二中文测试', var1) from ts_4893.meters order by ts limit 10; + greatest('二中文测试', var1) | +==================================== + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + +taos> select GREATEST('二中文测试', var1, var2) from ts_4893.meters order by ts limit 10; + greatest('二中文测试', var1, var2) | +========================================== + 二中文测试 | + 二中文测试 | + 四 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + +taos> select GREATEST('23', 3443434343434343); + greatest('23', 3443434343434343) | +=================================== + 3443434343434343 | + +taos> select GREATEST(co, 3443434343434343) from ts_4893.n1; + greatest(co, 3443434343434343) | +================================= + 3443434343434343 | + 3443434343434343 | + 3443434343434343 | + +taos> select GREATEST('23', 3443434343434343) from ts_4893.n1; + greatest('23', 3443434343434343) | +=================================== + 3443434343434343 | + 3443434343434343 | + 3443434343434343 | + +taos> select GREATEST('23', 1443434343434343) from ts_4893.n1; + greatest('23', 1443434343434343) | +=================================== + 23 | + 23 | + 23 | + +taos> select GREATEST(current, voltage) from ts_4893.n1; + greatest(current, voltage) | +============================= + NULL | + NULL | + 5 | + +taos> select GREATEST(current, voltage, '15') from ts_4893.n1; + greatest(current, voltage, '15') | +=================================== + NULL | + NULL | + 5.000000 | + +taos> alter local 'compareAsStrInGreatest' '0'; + +taos> select GREATEST(1,'2',3.3,4.4,5); + greatest(1,'2',3.3,4.4,5) | +============================ + 5 | + +taos> select GREATEST(1,2,3,4,5,'5.1'); + greatest(1,2,3,4,5,'5.1') | +============================ + 5 | + +taos> select GREATEST(121,'18'); + greatest(121,'18') | +======================== + 121 | + +taos> select GREATEST('1','2','3','4','5'); + greatest('1','2','3','4','5') | +================================ + 5 | + +taos> select GREATEST(1,2,3,4,5,6,7,'a','b','一','二','三'); + greatest(1,2,3,4,5,6,7,'a','b','一','二','三') | +==================================================== + 7 | + +taos> select GREATEST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213'); + greatest(1,2,3,4,5,6,7,'a','b','c','1','2','1231213') | +======================================================== + 1231213 | + +taos> select GREATEST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.123123'); + greatest(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.12 | +=================================================================== + 1231213 | + +taos> select GREATEST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); + greatest(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint | +=================================================================== + 9 | + +taos> select GREATEST(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); + greatest(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as | +=================================================================== + 9 | + +taos> select GREATEST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20)), cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double)); + greatest(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint | +=================================================================== + 9 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as tinyint)); + greatest(cast(100 as tinyint), cast(101 as tinyint)) | +======================================================= + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as smallint)); + greatest(cast(100 as tinyint), cast(101 as smallint)) | +======================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as int)); + greatest(cast(100 as tinyint), cast(101 as int)) | +=================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as bigint)); + greatest(cast(100 as tinyint), cast(101 as bigint)) | +====================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as float)); + greatest(cast(100 as tinyint), cast(101 as float)) | +===================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as double)); + greatest(cast(100 as tinyint), cast(101 as double)) | +====================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as varchar(20))); + greatest(cast(100 as tinyint), cast(101 as varchar(20))) | +=========================================================== + 101 | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as nchar(20))); + greatest(cast(100 as tinyint), cast(101 as nchar(20))) | +========================================================= + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as tinyint)); + greatest(cast(101 as tinyint), cast(100 as tinyint)) | +======================================================= + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as smallint)); + greatest(cast(101 as tinyint), cast(100 as smallint)) | +======================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as int)); + greatest(cast(101 as tinyint), cast(100 as int)) | +=================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as bigint)); + greatest(cast(101 as tinyint), cast(100 as bigint)) | +====================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as float)); + greatest(cast(101 as tinyint), cast(100 as float)) | +===================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as double)); + greatest(cast(101 as tinyint), cast(100 as double)) | +====================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as varchar(20))); + greatest(cast(101 as tinyint), cast(100 as varchar(20))) | +=========================================================== + 101 | + +taos> select GREATEST(cast(101 as tinyint), cast(100 as nchar(20))); + greatest(cast(101 as tinyint), cast(100 as nchar(20))) | +========================================================= + 101 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as smallint)); + greatest(cast(1000 as smallint), cast(1001 as smallint)) | +=========================================================== + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as int)); + greatest(cast(1000 as smallint), cast(1001 as int)) | +====================================================== + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as bigint)); + greatest(cast(1000 as smallint), cast(1001 as bigint)) | +========================================================= + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as float)); + greatest(cast(1000 as smallint), cast(1001 as float)) | +======================================================== + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as double)); + greatest(cast(1000 as smallint), cast(1001 as double)) | +========================================================= + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as varchar(20))); + greatest(cast(1000 as smallint), cast(1001 as varchar(20))) | +============================================================== + 1001 | + +taos> select GREATEST(cast(1000 as smallint), cast(1001 as nchar(20))); + greatest(cast(1000 as smallint), cast(1001 as nchar(20))) | +============================================================ + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as smallint)); + greatest(cast(1001 as smallint), cast(1000 as smallint)) | +=========================================================== + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as int)); + greatest(cast(1001 as smallint), cast(1000 as int)) | +====================================================== + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as bigint)); + greatest(cast(1001 as smallint), cast(1000 as bigint)) | +========================================================= + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as float)); + greatest(cast(1001 as smallint), cast(1000 as float)) | +======================================================== + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as double)); + greatest(cast(1001 as smallint), cast(1000 as double)) | +========================================================= + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as varchar(20))); + greatest(cast(1001 as smallint), cast(1000 as varchar(20))) | +============================================================== + 1001 | + +taos> select GREATEST(cast(1001 as smallint), cast(1000 as nchar(20))); + greatest(cast(1001 as smallint), cast(1000 as nchar(20))) | +============================================================ + 1001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as int)); + greatest(cast(1000000 as int), cast(1000001 as int)) | +======================================================= + 1000001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as bigint)); + greatest(cast(1000000 as int), cast(1000001 as bigint)) | +========================================================== + 1000001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as float)); + greatest(cast(1000000 as int), cast(1000001 as float)) | +========================================================= + 1e+06 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as double)); + greatest(cast(1000000 as int), cast(1000001 as double)) | +========================================================== + 1000001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as varchar(20))); + greatest(cast(1000000 as int), cast(1000001 as varchar(20))) | +=============================================================== + 1000001 | + +taos> select GREATEST(cast(1000000 as int), cast(1000001 as nchar(20))); + greatest(cast(1000000 as int), cast(1000001 as nchar(20))) | +============================================================= + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as int)); + greatest(cast(1000001 as int), cast(1000000 as int)) | +======================================================= + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as bigint)); + greatest(cast(1000001 as int), cast(1000000 as bigint)) | +========================================================== + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as float)); + greatest(cast(1000001 as int), cast(1000000 as float)) | +========================================================= + 1e+06 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as double)); + greatest(cast(1000001 as int), cast(1000000 as double)) | +========================================================== + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as varchar(20))); + greatest(cast(1000001 as int), cast(1000000 as varchar(20))) | +=============================================================== + 1000001 | + +taos> select GREATEST(cast(1000001 as int), cast(1000000 as nchar(20))); + greatest(cast(1000001 as int), cast(1000000 as nchar(20))) | +============================================================= + 1000001 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as bigint)); + greatest(cast(1000000000 as bigint), cast(1000000001 as bigint)) | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as float)); + greatest(cast(1000000000 as bigint), cast(1000000001 as float)) | +================================================================== + 1e+09 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as double)); + greatest(cast(1000000000 as bigint), cast(1000000001 as double)) | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as varchar(20))); + greatest(cast(1000000000 as bigint), cast(1000000001 as varchar( | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000000 as bigint), cast(1000000001 as nchar(20))); + greatest(cast(1000000000 as bigint), cast(1000000001 as nchar(20 | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as bigint)); + greatest(cast(1000000001 as bigint), cast(1000000000 as bigint)) | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as float)); + greatest(cast(1000000001 as bigint), cast(1000000000 as float)) | +================================================================== + 1e+09 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as double)); + greatest(cast(1000000001 as bigint), cast(1000000000 as double)) | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as varchar(20))); + greatest(cast(1000000001 as bigint), cast(1000000000 as varchar( | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(1000000001 as bigint), cast(1000000000 as nchar(20))); + greatest(cast(1000000001 as bigint), cast(1000000000 as nchar(20 | +=================================================================== + 1000000001 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as float)); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as f | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as double)); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as d | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as timestamp)); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as t | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as varchar(20))); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as v | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as nchar(20))); + greatest(cast(100000.1111111 as float), cast(100001.1111111 as n | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as float)); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as f | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as double)); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as d | +=================================================================== + 100001.109375 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as timestamp)); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as t | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as varchar(20))); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as v | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as nchar(20))); + greatest(cast(100001.1111111 as float), cast(100000.1111111 as n | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as double)); + greatest(cast(100000.1111111 as double), cast(100001.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as timestamp)); + greatest(cast(100000.1111111 as double), cast(100001.1111111 as | +=================================================================== + 100001 | + +taos> select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as varchar(20))); + greatest(cast(100000.1111111 as double), cast(100001.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as nchar(20))); + greatest(cast(100000.1111111 as double), cast(100001.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as double)); + greatest(cast(100001.1111111 as double), cast(100000.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as timestamp)); + greatest(cast(100001.1111111 as double), cast(100000.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as varchar(20))); + greatest(cast(100001.1111111 as double), cast(100000.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as nchar(20))); + greatest(cast(100001.1111111 as double), cast(100000.1111111 as | +=================================================================== + 100001.1111111 | + +taos> select GREATEST(cast('中文测试' as varchar(20)), cast('中文测试一' as varchar(20))); + greatest(cast('中文测试' as varchar(20)), cast('中文测试 | +=================================================================== + 中文测试一 | + +taos> select GREATEST(cast('中文测试' as varchar(20)), cast('中文测试一' as nchar(20))); + greatest(cast('中文测试' as varchar(20)), cast('中文测试 | +=================================================================== + 中文测试一 | + +taos> select GREATEST(cast('中文测试一' as varchar(20)), cast('中文测试' as varchar(20))); + greatest(cast('中文测试一' as varchar(20)), cast('中文测 | +=================================================================== + 中文测试一 | + +taos> select GREATEST(cast('中文测试一' as varchar(20)), cast('中文测试' as nchar(20))); + greatest(cast('中文测试一' as varchar(20)), cast('中文测 | +=================================================================== + 中文测试一 | + +taos> select GREATEST(cast('abc123abc' as varchar(20)), cast('abc124abc' as varchar(20))); + greatest(cast('abc123abc' as varchar(20)), cast('abc124abc' as v | +=================================================================== + abc124abc | + +taos> select GREATEST(cast('abc123abc' as varchar(20)), cast('abc124abc' as nchar(20))); + greatest(cast('abc123abc' as varchar(20)), cast('abc124abc' as n | +=================================================================== + abc124abc | + +taos> select GREATEST(cast('abc124abc' as varchar(20)), cast('abc123abc' as varchar(20))); + greatest(cast('abc124abc' as varchar(20)), cast('abc123abc' as v | +=================================================================== + abc124abc | + +taos> select GREATEST(cast('abc124abc' as varchar(20)), cast('abc123abc' as nchar(20))); + greatest(cast('abc124abc' as varchar(20)), cast('abc123abc' as n | +=================================================================== + abc124abc | + +taos> select GREATEST(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar(20))); + greatest(cast('abc123abc' as nchar(20)), cast('abc124abc' as nch | +=================================================================== + abc124abc | + +taos> select GREATEST(cast(100 as tinyint), cast(101 as float), cast(102 as varchar(20))); + greatest(cast(100 as tinyint), cast(101 as float), cast(102 as v | +=================================================================== + 102 | + +taos> select GREATEST(cast(100 as float), cast(101 as tinyint), cast(102 as varchar(20))); + greatest(cast(100 as float), cast(101 as tinyint), cast(102 as v | +=================================================================== + 102 | + +taos> select GREATEST(cast(100 as float), cast(101 as varchar(20)), cast(102 as tinyint)); + greatest(cast(100 as float), cast(101 as varchar(20)), cast(102 | +=================================================================== + 102 | + +taos> select GREATEST(cast(100 as varchar(20)), cast(101 as float), cast(102 as tinyint)); + greatest(cast(100 as varchar(20)), cast(101 as float), cast(102 | +=================================================================== + 102 | + +taos> select GREATEST('a','b','c','d','e','f','g','h','1231','15155'); + greatest('a','b','c','d','e','f','g','h','1231','15155') | +=========================================================== + h | + +taos> select GREATEST(current, voltage, phase, id, nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; + greatest(current, voltage, phase, id, nch1, nch2, var1, var2) | +================================================================ + 221 | + 220 | + 215 | + 216 | + 219 | + 221 | + 215 | + 217 | + 216 | + 223 | + +taos> select GREATEST(current, voltage, phase, id) from ts_4893.meters order by ts limit 10; + greatest(current, voltage, phase, id) | +======================================== + 221 | + 220 | + 215 | + 216 | + 219 | + 221 | + 215 | + 217 | + 216 | + 223 | + +taos> select GREATEST(nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; + greatest(nch1, nch2, var1, var2) | +=================================== + 四 | + 三a | + 四 | + 一 | + 一二三四五六七八九十 | + 一二三 | + prision | + 一二三四五六七八九十 | + prision | + 一 | + +taos> select GREATEST(221, voltage) from ts_4893.meters order by ts limit 10; + greatest(221, voltage) | +========================= + 221 | + 221 | + 221 | + 221 | + 221 | + 221 | + 221 | + 221 | + 221 | + 223 | + +taos> select GREATEST(5, id) from ts_4893.meters order by ts limit 10; + greatest(5, id) | +======================== + 5 | + 5 | + 5 | + 5 | + 5 | + 5 | + 6 | + 7 | + 8 | + 9 | + +taos> select GREATEST('r', nch1) from ts_4893.meters order by ts limit 10; + greatest('r', nch1) | +================================= + r | + 一二三四五六七八九十 | + update | + r | + r | + r | + r | + 一二三四五六七八九十 | + r | + r | + +taos> select GREATEST('r', nch1, nch2) from ts_4893.meters order by ts limit 10; + greatest('r', nch1, nch2) | +================================= + 四 | + 一二三四五六七八九十 | + update | + 一 | + r | + 一二三 | + r | + 一二三四五六七八九十 | + r | + r | + +taos> select GREATEST('r', var1) from ts_4893.meters order by ts limit 10; + greatest('r', var1) | +================================= + r | + r | + r | + r | + 一二三四五六七八九十 | + update | + r | + r | + r | + r | + +taos> select GREATEST('r', var1, var2) from ts_4893.meters order by ts limit 10; + greatest('r', var1, var2) | +================================= + r | + 三a | + 四 | + r | + 一二三四五六七八九十 | + update | + r | + r | + r | + 一 | + +taos> select GREATEST('二中文测试', nch1) from ts_4893.meters order by ts limit 10; + greatest('二中文测试', nch1) | +==================================== + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + +taos> select GREATEST('二中文测试', nch1, nch2) from ts_4893.meters order by ts limit 10; + greatest('二中文测试', nch1, nch2) | +========================================== + 四 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + +taos> select GREATEST('二中文测试', var1) from ts_4893.meters order by ts limit 10; + greatest('二中文测试', var1) | +==================================== + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + +taos> select GREATEST('二中文测试', var1, var2) from ts_4893.meters order by ts limit 10; + greatest('二中文测试', var1, var2) | +========================================== + 二中文测试 | + 二中文测试 | + 四 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + 二中文测试 | + +taos> select GREATEST('23', 3443434343434343); + greatest('23', 3443434343434343) | +=================================== + 3443434343434343 | + +taos> select GREATEST(co, 3443434343434343) from ts_4893.n1; + greatest(co, 3443434343434343) | +================================= + 3443434343434343 | + 3443434343434343 | + 3443434343434343 | + +taos> select GREATEST('23', 1443434343434343) from ts_4893.n1; + greatest('23', 1443434343434343) | +=================================== + 1443434343434343 | + 1443434343434343 | + 1443434343434343 | + +taos> select GREATEST('23', 3443434343434343) from ts_4893.n1 + greatest('23', 3443434343434343) | +=================================== + 3443434343434343 | + 3443434343434343 | + 3443434343434343 | + +taos> select GREATEST(current, voltage) from ts_4893.n1; + greatest(current, voltage) | +============================= + NULL | + NULL | + 5 | + +taos> select GREATEST(current, voltage, '15') from ts_4893.n1; + greatest(current, voltage, '15') | +=================================== + NULL | + NULL | + 15 | + diff --git a/tests/army/query/function/ans/least.csv b/tests/army/query/function/ans/least.csv new file mode 100644 index 0000000000..de66426bd2 --- /dev/null +++ b/tests/army/query/function/ans/least.csv @@ -0,0 +1,1389 @@ + +taos> alter local 'compareAsStrInGreatest' '1'; + +taos> select LEAST(1,2,3,4,5,6,7,8,9,10); + least(1,2,3,4,5,6,7,8,9,10) | +============================== + 1 | + +taos> select LEAST(1,1.1,2.23,3.4444,5.66666666,6.21241241,7.999999999999); + least(1,1.1,2.23,3.4444,5.66666666,6.21241241,7.999999999999) | +================================================================ + 1 | + +taos> select LEAST(1,'2',3.3,4.4,5); + least(1,'2',3.3,4.4,5) | +============================ + 1 | + +taos> select LEAST(1,2,3,4,5,'5.1'); + least(1,2,3,4,5,'5.1') | +============================ + 1 | + +taos> select LEAST('1','2','3','4',5); + least('1','2','3','4',5) | +============================ + 1 | + +taos> select LEAST('1','2','3','4','5'); + least('1','2','3','4','5') | +============================= + 1 | + +taos> select LEAST(1,2,3,4,5,6,7,'a','b','一','二','三'); + least(1,2,3,4,5,6,7,'a','b','一','二','三') | +================================================= + 1 | + +taos> select LEAST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213'); + least(1,2,3,4,5,6,7,'a','b','c','1','2','1231213') | +===================================================== + 1 | + +taos> select LEAST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.123123'); + least(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.12312 | +=================================================================== + 1 | + +taos> select LEAST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); + least(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), | +=================================================================== + 0 | + +taos> select LEAST(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); + least(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as sm | +=================================================================== + 0 | + +taos> select LEAST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20)), cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double)); + least(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), | +=================================================================== + 0 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as tinyint)); + least(cast(100 as tinyint), cast(101 as tinyint)) | +==================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as smallint)); + least(cast(100 as tinyint), cast(101 as smallint)) | +===================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as int)); + least(cast(100 as tinyint), cast(101 as int)) | +================================================ + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as bigint)); + least(cast(100 as tinyint), cast(101 as bigint)) | +=================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as float)); + least(cast(100 as tinyint), cast(101 as float)) | +================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as double)); + least(cast(100 as tinyint), cast(101 as double)) | +=================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as varchar(20))); + least(cast(100 as tinyint), cast(101 as varchar(20))) | +======================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as nchar(20))); + least(cast(100 as tinyint), cast(101 as nchar(20))) | +====================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as tinyint)); + least(cast(101 as tinyint), cast(100 as tinyint)) | +==================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as smallint)); + least(cast(101 as tinyint), cast(100 as smallint)) | +===================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as int)); + least(cast(101 as tinyint), cast(100 as int)) | +================================================ + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as bigint)); + least(cast(101 as tinyint), cast(100 as bigint)) | +=================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as float)); + least(cast(101 as tinyint), cast(100 as float)) | +================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as double)); + least(cast(101 as tinyint), cast(100 as double)) | +=================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as varchar(20))); + least(cast(101 as tinyint), cast(100 as varchar(20))) | +======================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as nchar(20))); + least(cast(101 as tinyint), cast(100 as nchar(20))) | +====================================================== + 100 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as smallint)); + least(cast(1000 as smallint), cast(1001 as smallint)) | +======================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as int)); + least(cast(1000 as smallint), cast(1001 as int)) | +=================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as bigint)); + least(cast(1000 as smallint), cast(1001 as bigint)) | +====================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as float)); + least(cast(1000 as smallint), cast(1001 as float)) | +===================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as double)); + least(cast(1000 as smallint), cast(1001 as double)) | +====================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as varchar(20))); + least(cast(1000 as smallint), cast(1001 as varchar(20))) | +=========================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as nchar(20))); + least(cast(1000 as smallint), cast(1001 as nchar(20))) | +========================================================= + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as smallint)); + least(cast(1001 as smallint), cast(1000 as smallint)) | +======================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as int)); + least(cast(1001 as smallint), cast(1000 as int)) | +=================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as bigint)); + least(cast(1001 as smallint), cast(1000 as bigint)) | +====================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as float)); + least(cast(1001 as smallint), cast(1000 as float)) | +===================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as double)); + least(cast(1001 as smallint), cast(1000 as double)) | +====================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as varchar(20))); + least(cast(1001 as smallint), cast(1000 as varchar(20))) | +=========================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as nchar(20))); + least(cast(1001 as smallint), cast(1000 as nchar(20))) | +========================================================= + 1000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as int)); + least(cast(1000000 as int), cast(1000001 as int)) | +==================================================== + 1000000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as bigint)); + least(cast(1000000 as int), cast(1000001 as bigint)) | +======================================================= + 1000000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as float)); + least(cast(1000000 as int), cast(1000001 as float)) | +====================================================== + 1e+06 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as double)); + least(cast(1000000 as int), cast(1000001 as double)) | +======================================================= + 1000000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as varchar(20))); + least(cast(1000000 as int), cast(1000001 as varchar(20))) | +============================================================ + 1000000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as nchar(20))); + least(cast(1000000 as int), cast(1000001 as nchar(20))) | +========================================================== + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as int)); + least(cast(1000001 as int), cast(1000000 as int)) | +==================================================== + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as bigint)); + least(cast(1000001 as int), cast(1000000 as bigint)) | +======================================================= + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as float)); + least(cast(1000001 as int), cast(1000000 as float)) | +====================================================== + 1e+06 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as double)); + least(cast(1000001 as int), cast(1000000 as double)) | +======================================================= + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as varchar(20))); + least(cast(1000001 as int), cast(1000000 as varchar(20))) | +============================================================ + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as nchar(20))); + least(cast(1000001 as int), cast(1000000 as nchar(20))) | +========================================================== + 1000000 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as bigint)); + least(cast(1000000000 as bigint), cast(1000000001 as bigint)) | +================================================================ + 1000000000 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as float)); + least(cast(1000000000 as bigint), cast(1000000001 as float)) | +=============================================================== + 1e+09 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as double)); + least(cast(1000000000 as bigint), cast(1000000001 as double)) | +================================================================ + 1000000000 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as varchar(20))); + least(cast(1000000000 as bigint), cast(1000000001 as varchar(20) | +=================================================================== + 1000000000 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as nchar(20))); + least(cast(1000000000 as bigint), cast(1000000001 as nchar(20))) | +=================================================================== + 1000000000 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as bigint)); + least(cast(1000000001 as bigint), cast(1000000000 as bigint)) | +================================================================ + 1000000000 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as float)); + least(cast(1000000001 as bigint), cast(1000000000 as float)) | +=============================================================== + 1e+09 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as double)); + least(cast(1000000001 as bigint), cast(1000000000 as double)) | +================================================================ + 1000000000 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as varchar(20))); + least(cast(1000000001 as bigint), cast(1000000000 as varchar(20) | +=================================================================== + 1000000000 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as nchar(20))); + least(cast(1000000001 as bigint), cast(1000000000 as nchar(20))) | +=================================================================== + 1000000000 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as float)); + least(cast(100000.1111111 as float), cast(100001.1111111 as floa | +=================================================================== + 100000 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as double)); + least(cast(100000.1111111 as float), cast(100001.1111111 as doub | +=================================================================== + 100000.109375 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as timestamp)); + least(cast(100000.1111111 as float), cast(100001.1111111 as time | +=================================================================== + 100000 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as varchar(20))); + least(cast(100000.1111111 as float), cast(100001.1111111 as varc | +=================================================================== + 100000.109375 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as nchar(20))); + least(cast(100000.1111111 as float), cast(100001.1111111 as ncha | +=================================================================== + 100000.109375 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as float)); + least(cast(100001.1111111 as float), cast(100000.1111111 as floa | +=================================================================== + 100000 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as double)); + least(cast(100001.1111111 as float), cast(100000.1111111 as doub | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as timestamp)); + least(cast(100001.1111111 as float), cast(100000.1111111 as time | +=================================================================== + 100000 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as varchar(20))); + least(cast(100001.1111111 as float), cast(100000.1111111 as varc | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as nchar(20))); + least(cast(100001.1111111 as float), cast(100000.1111111 as ncha | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as double)); + least(cast(100000.1111111 as double), cast(100001.1111111 as dou | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as timestamp)); + least(cast(100000.1111111 as double), cast(100001.1111111 as tim | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as varchar(20))); + least(cast(100000.1111111 as double), cast(100001.1111111 as var | +=================================================================== + 100000.111111 | + +taos> select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as nchar(20))); + least(cast(100000.1111111 as double), cast(100001.1111111 as nch | +=================================================================== + 100000.111111 | + +taos> select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as double)); + least(cast(100001.1111111 as double), cast(100000.1111111 as dou | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as timestamp)); + least(cast(100001.1111111 as double), cast(100000.1111111 as tim | +=================================================================== + 100000 | + +taos> select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as varchar(20))); + least(cast(100001.1111111 as double), cast(100000.1111111 as var | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as nchar(20))); + least(cast(100001.1111111 as double), cast(100000.1111111 as nch | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast('中文测试' as varchar(20)), cast('中文测试一' as varchar(20))); + least(cast('中文测试' as varchar(20)), cast('中文测试一 | +=================================================================== + 中文测试 | + +taos> select LEAST(cast('中文测试' as varchar(20)), cast('中文测试一' as nchar(20))); + least(cast('中文测试' as varchar(20)), cast('中文测试一 | +=================================================================== + 中文测试 | + +taos> select LEAST(cast('中文测试一' as varchar(20)), cast('中文测试' as varchar(20))); + least(cast('中文测试一' as varchar(20)), cast('中文测试 | +=================================================================== + 中文测试 | + +taos> select LEAST(cast('中文测试一' as varchar(20)), cast('中文测试' as nchar(20))); + least(cast('中文测试一' as varchar(20)), cast('中文测试 | +=================================================================== + 中文测试 | + +taos> select LEAST(cast('abc123abc' as varchar(20)), cast('abc124abc' as varchar(20))); + least(cast('abc123abc' as varchar(20)), cast('abc124abc' as varc | +=================================================================== + abc123abc | + +taos> select LEAST(cast('abc123abc' as varchar(20)), cast('abc124abc' as nchar(20))); + least(cast('abc123abc' as varchar(20)), cast('abc124abc' as ncha | +=================================================================== + abc123abc | + +taos> select LEAST(cast('abc124abc' as varchar(20)), cast('abc123abc' as varchar(20))); + least(cast('abc124abc' as varchar(20)), cast('abc123abc' as varc | +=================================================================== + abc123abc | + +taos> select LEAST(cast('abc124abc' as varchar(20)), cast('abc123abc' as nchar(20))); + least(cast('abc124abc' as varchar(20)), cast('abc123abc' as ncha | +=================================================================== + abc123abc | + +taos> select LEAST(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar(20))); + least(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar( | +=================================================================== + abc123abc | + +taos> select LEAST(cast(100 as tinyint), cast(101 as float), cast(102 as varchar(20))); + least(cast(100 as tinyint), cast(101 as float), cast(102 as varc | +=================================================================== + 100 | + +taos> select LEAST(cast(100 as varchar(20)), cast(101 as float), cast(102 as tinyint)); + least(cast(100 as varchar(20)), cast(101 as float), cast(102 as | +=================================================================== + 100 | + +taos> select LEAST('a','b','c','d','e','f','g','h','1231','15155'); + least('a','b','c','d','e','f','g','h','1231','15155') | +======================================================== + 1231 | + +taos> select LEAST(current, voltage, phase, id, nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; + least(current, voltage, phase, id, nch1, nch2, var1, var2) | +============================================================= + 0 | + 0.138830 | + 0.796942 | + 0.537330 | + 0.313430 | + 0.332767 | + 0.846763 | + 0.637813 | + 0.115989 | + 0.373575 | + +taos> select LEAST(current, voltage, phase, id) from ts_4893.meters order by ts limit 10; + least(current, voltage, phase, id) | +===================================== + 0 | + 0.13883 | + 0.796942 | + 0.53733 | + 0.31343 | + 0.332767 | + 0.846763 | + 0.637813 | + 0.115989 | + 0.373575 | + +taos> select LEAST(nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; + least(nch1, nch2, var1, var2) | +================================= + e | + f | + c | + e | + b | + g | + again | + b | + c | + again | + +taos> select LEAST(221, voltage) from ts_4893.meters order by ts limit 10; + least(221, voltage) | +======================== + 221 | + 220 | + 215 | + 216 | + 219 | + 221 | + 215 | + 217 | + 216 | + 221 | + +taos> select LEAST(5, id) from ts_4893.meters order by ts limit 10; + least(5, id) | +======================== + 0 | + 1 | + 2 | + 3 | + 4 | + 5 | + 5 | + 5 | + 5 | + 5 | + +taos> select LEAST('r', nch1) from ts_4893.meters order by ts limit 10; + least('r', nch1) | +================================= + novel | + r | + r | + prision | + novel | + novel | + again | + r | + novel | + again | + +taos> select LEAST('r', nch1, nch2) from ts_4893.meters order by ts limit 10; + least('r', nch1, nch2) | +================================= + novel | + f | + c | + prision | + e | + novel | + again | + d | + c | + again | + +taos> select LEAST('r', var1) from ts_4893.meters order by ts limit 10; + least('r', var1) | +================================= + novel | + person | + novel | + plate | + r | + r | + prision | + person | + prision | + plate | + +taos> select LEAST('r', var1, var2) from ts_4893.meters order by ts limit 10; + least('r', var1, var2) | +================================= + e | + person | + novel | + e | + b | + g | + b | + b | + e | + plate | + +taos> select LEAST('二中文测试', nch1) from ts_4893.meters order by ts limit 10; + least('二中文测试', nch1) | +================================= + novel | + 一二三四五六七八九十 | + update | + prision | + novel | + novel | + again | + 一二三四五六七八九十 | + novel | + again | + +taos> select LEAST('二中文测试', nch1, nch2) from ts_4893.meters order by ts limit 10; + least('二中文测试', nch1, nch2) | +======================================= + novel | + f | + c | + prision | + e | + novel | + again | + d | + c | + again | + +taos> select LEAST('二中文测试', var1) from ts_4893.meters order by ts limit 10; + least('二中文测试', var1) | +================================= + novel | + person | + novel | + plate | + 一二三四五六七八九十 | + update | + prision | + person | + prision | + plate | + +taos> select LEAST('二中文测试', var1, var2) from ts_4893.meters order by ts limit 10; + least('二中文测试', var1, var2) | +======================================= + e | + person | + novel | + e | + b | + g | + b | + b | + e | + plate | + +taos> select LEAST('23', 3443434343434343); + least('23', 3443434343434343) | +================================ + 23 | + +taos> select LEAST(co, 3443434343434343) from ts_4893.n1; + least(co, 3443434343434343) | +================================= + 23 | + 23 | + 23 | + +taos> select LEAST('23', 3443434343434343) from ts_4893.n1; + least('23', 3443434343434343) | +================================ + 23 | + 23 | + 23 | + +taos> select LEAST('23', 1443434343434343) from ts_4893.n1; + least('23', 1443434343434343) | +================================ + 1443434343434343 | + 1443434343434343 | + 1443434343434343 | + +taos> select LEAST(current, voltage) from ts_4893.n1; + least(current, voltage) | +========================== + NULL | + NULL | + 3 | + +taos> select LEAST(current, voltage, '15') from ts_4893.n1; + least(current, voltage, '15') | +================================ + NULL | + NULL | + 15 | + +taos> alter local 'compareAsStrInGreatest' '0'; + +taos> select LEAST(1,2,3,4,5,6,7,8,9,10); + least(1,2,3,4,5,6,7,8,9,10) | +============================== + 1 | + +taos> select LEAST(1,1.1,2.23,3.4444,5.66666666,6.21241241,7.999999999999); + least(1,1.1,2.23,3.4444,5.66666666,6.21241241,7.999999999999) | +================================================================ + 1 | + +taos> select LEAST(1,'2',3.3,4.4,5); + least(1,'2',3.3,4.4,5) | +============================ + 1 | + +taos> select LEAST(1,2,3,4,5,'5.1'); + least(1,2,3,4,5,'5.1') | +========================= + 1 | + +taos> select LEAST('1','2','3','4',5); + least('1','2','3','4',5) | +=========================== + 1 | + +taos> select LEAST('1','2','3','4','5'); + least('1','2','3','4','5') | +============================= + 1 | + +taos> select LEAST(1,2,3,4,5,6,7,'a','b','一','二','三'); + least(1,2,3,4,5,6,7,'a','b','一','二','三') | +================================================= + 0 | + +taos> select LEAST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213'); + least(1,2,3,4,5,6,7,'a','b','c','1','2','1231213') | +===================================================== + 0 | + +taos> select LEAST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.123123'); + least(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.12312 | +=================================================================== + 0 | + +taos> select LEAST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); + least(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), | +=================================================================== + 0 | + +taos> select LEAST(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); + least(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as sm | +=================================================================== + 0 | + +taos> select LEAST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20)), cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double)); + least(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), | +=================================================================== + 0 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as tinyint)); + least(cast(100 as tinyint), cast(101 as tinyint)) | +==================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as smallint)); + least(cast(100 as tinyint), cast(101 as smallint)) | +===================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as int)); + least(cast(100 as tinyint), cast(101 as int)) | +================================================ + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as bigint)); + least(cast(100 as tinyint), cast(101 as bigint)) | +=================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as float)); + least(cast(100 as tinyint), cast(101 as float)) | +================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as double)); + least(cast(100 as tinyint), cast(101 as double)) | +=================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as varchar(20))); + least(cast(100 as tinyint), cast(101 as varchar(20))) | +======================================================== + 100 | + +taos> select LEAST(cast(100 as tinyint), cast(101 as nchar(20))); + least(cast(100 as tinyint), cast(101 as nchar(20))) | +====================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as tinyint)); + least(cast(101 as tinyint), cast(100 as tinyint)) | +==================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as smallint)); + least(cast(101 as tinyint), cast(100 as smallint)) | +===================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as int)); + least(cast(101 as tinyint), cast(100 as int)) | +================================================ + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as bigint)); + least(cast(101 as tinyint), cast(100 as bigint)) | +=================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as float)); + least(cast(101 as tinyint), cast(100 as float)) | +================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as double)); + least(cast(101 as tinyint), cast(100 as double)) | +=================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as varchar(20))); + least(cast(101 as tinyint), cast(100 as varchar(20))) | +======================================================== + 100 | + +taos> select LEAST(cast(101 as tinyint), cast(100 as nchar(20))); + least(cast(101 as tinyint), cast(100 as nchar(20))) | +====================================================== + 100 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as smallint)); + least(cast(1000 as smallint), cast(1001 as smallint)) | +======================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as int)); + least(cast(1000 as smallint), cast(1001 as int)) | +=================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as bigint)); + least(cast(1000 as smallint), cast(1001 as bigint)) | +====================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as float)); + least(cast(1000 as smallint), cast(1001 as float)) | +===================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as double)); + least(cast(1000 as smallint), cast(1001 as double)) | +====================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as varchar(20))); + least(cast(1000 as smallint), cast(1001 as varchar(20))) | +=========================================================== + 1000 | + +taos> select LEAST(cast(1000 as smallint), cast(1001 as nchar(20))); + least(cast(1000 as smallint), cast(1001 as nchar(20))) | +========================================================= + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as smallint)); + least(cast(1001 as smallint), cast(1000 as smallint)) | +======================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as int)); + least(cast(1001 as smallint), cast(1000 as int)) | +=================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as bigint)); + least(cast(1001 as smallint), cast(1000 as bigint)) | +====================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as float)); + least(cast(1001 as smallint), cast(1000 as float)) | +===================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as double)); + least(cast(1001 as smallint), cast(1000 as double)) | +====================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as varchar(20))); + least(cast(1001 as smallint), cast(1000 as varchar(20))) | +=========================================================== + 1000 | + +taos> select LEAST(cast(1001 as smallint), cast(1000 as nchar(20))); + least(cast(1001 as smallint), cast(1000 as nchar(20))) | +========================================================= + 1000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as int)); + least(cast(1000000 as int), cast(1000001 as int)) | +==================================================== + 1000000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as bigint)); + least(cast(1000000 as int), cast(1000001 as bigint)) | +======================================================= + 1000000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as float)); + least(cast(1000000 as int), cast(1000001 as float)) | +====================================================== + 1e+06 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as double)); + least(cast(1000000 as int), cast(1000001 as double)) | +======================================================= + 1000000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as varchar(20))); + least(cast(1000000 as int), cast(1000001 as varchar(20))) | +============================================================ + 1000000 | + +taos> select LEAST(cast(1000000 as int), cast(1000001 as nchar(20))); + least(cast(1000000 as int), cast(1000001 as nchar(20))) | +========================================================== + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as int)); + least(cast(1000001 as int), cast(1000000 as int)) | +==================================================== + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as bigint)); + least(cast(1000001 as int), cast(1000000 as bigint)) | +======================================================= + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as float)); + least(cast(1000001 as int), cast(1000000 as float)) | +====================================================== + 1e+06 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as double)); + least(cast(1000001 as int), cast(1000000 as double)) | +======================================================= + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as varchar(20))); + least(cast(1000001 as int), cast(1000000 as varchar(20))) | +============================================================ + 1000000 | + +taos> select LEAST(cast(1000001 as int), cast(1000000 as nchar(20))); + least(cast(1000001 as int), cast(1000000 as nchar(20))) | +========================================================== + 1000000 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as bigint)); + least(cast(1000000000 as bigint), cast(1000000001 as bigint)) | +================================================================ + 1000000000 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as float)); + least(cast(1000000000 as bigint), cast(1000000001 as float)) | +=============================================================== + 1e+09 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as double)); + least(cast(1000000000 as bigint), cast(1000000001 as double)) | +================================================================ + 1000000000 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as varchar(20))); + least(cast(1000000000 as bigint), cast(1000000001 as varchar(20) | +=================================================================== + 1000000000 | + +taos> select LEAST(cast(1000000000 as bigint), cast(1000000001 as nchar(20))); + least(cast(1000000000 as bigint), cast(1000000001 as nchar(20))) | +=================================================================== + 1000000000 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as bigint)); + least(cast(1000000001 as bigint), cast(1000000000 as bigint)) | +================================================================ + 1000000000 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as float)); + least(cast(1000000001 as bigint), cast(1000000000 as float)) | +=============================================================== + 1e+09 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as double)); + least(cast(1000000001 as bigint), cast(1000000000 as double)) | +================================================================ + 1000000000 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as varchar(20))); + least(cast(1000000001 as bigint), cast(1000000000 as varchar(20) | +=================================================================== + 1000000000 | + +taos> select LEAST(cast(1000000001 as bigint), cast(1000000000 as nchar(20))); + least(cast(1000000001 as bigint), cast(1000000000 as nchar(20))) | +=================================================================== + 1000000000 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as float)); + least(cast(100000.1111111 as float), cast(100001.1111111 as floa | +=================================================================== + 100000 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as double)); + least(cast(100000.1111111 as float), cast(100001.1111111 as doub | +=================================================================== + 100000.109375 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as timestamp)); + least(cast(100000.1111111 as float), cast(100001.1111111 as time | +=================================================================== + 100000 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as varchar(20))); + least(cast(100000.1111111 as float), cast(100001.1111111 as varc | +=================================================================== + 100000 | + +taos> select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as nchar(20))); + least(cast(100000.1111111 as float), cast(100001.1111111 as ncha | +=================================================================== + 100000 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as float)); + least(cast(100001.1111111 as float), cast(100000.1111111 as floa | +=================================================================== + 100000 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as double)); + least(cast(100001.1111111 as float), cast(100000.1111111 as doub | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as timestamp)); + least(cast(100001.1111111 as float), cast(100000.1111111 as time | +=================================================================== + 100000 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as varchar(20))); + least(cast(100001.1111111 as float), cast(100000.1111111 as varc | +=================================================================== + 100000 | + +taos> select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as nchar(20))); + least(cast(100001.1111111 as float), cast(100000.1111111 as ncha | +=================================================================== + 100000 | + +taos> select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as double)); + least(cast(100000.1111111 as double), cast(100001.1111111 as dou | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as timestamp)); + least(cast(100000.1111111 as double), cast(100001.1111111 as tim | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as varchar(20))); + least(cast(100000.1111111 as double), cast(100001.1111111 as var | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as nchar(20))); + least(cast(100000.1111111 as double), cast(100001.1111111 as nch | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as double)); + least(cast(100001.1111111 as double), cast(100000.1111111 as dou | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as timestamp)); + least(cast(100001.1111111 as double), cast(100000.1111111 as tim | +=================================================================== + 100000 | + +taos> select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as varchar(20))); + least(cast(100001.1111111 as double), cast(100000.1111111 as var | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as nchar(20))); + least(cast(100001.1111111 as double), cast(100000.1111111 as nch | +=================================================================== + 100000.1111111 | + +taos> select LEAST(cast('中文测试' as varchar(20)), cast('中文测试一' as varchar(20))); + least(cast('中文测试' as varchar(20)), cast('中文测试一 | +=================================================================== + 中文测试 | + +taos> select LEAST(cast('中文测试' as varchar(20)), cast('中文测试一' as nchar(20))); + least(cast('中文测试' as varchar(20)), cast('中文测试一 | +=================================================================== + 中文测试 | + +taos> select LEAST(cast('中文测试一' as varchar(20)), cast('中文测试' as varchar(20))); + least(cast('中文测试一' as varchar(20)), cast('中文测试 | +=================================================================== + 中文测试 | + +taos> select LEAST(cast('中文测试一' as varchar(20)), cast('中文测试' as nchar(20))); + least(cast('中文测试一' as varchar(20)), cast('中文测试 | +=================================================================== + 中文测试 | + +taos> select LEAST(cast('abc123abc' as varchar(20)), cast('abc124abc' as varchar(20))); + least(cast('abc123abc' as varchar(20)), cast('abc124abc' as varc | +=================================================================== + abc123abc | + +taos> select LEAST(cast('abc123abc' as varchar(20)), cast('abc124abc' as nchar(20))); + least(cast('abc123abc' as varchar(20)), cast('abc124abc' as ncha | +=================================================================== + abc123abc | + +taos> select LEAST(cast('abc124abc' as varchar(20)), cast('abc123abc' as varchar(20))); + least(cast('abc124abc' as varchar(20)), cast('abc123abc' as varc | +=================================================================== + abc123abc | + +taos> select LEAST(cast('abc124abc' as varchar(20)), cast('abc123abc' as nchar(20))); + least(cast('abc124abc' as varchar(20)), cast('abc123abc' as ncha | +=================================================================== + abc123abc | + +taos> select LEAST(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar(20))); + least(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar( | +=================================================================== + abc123abc | + +taos> select LEAST(cast(100 as tinyint), cast(101 as float), cast(102 as varchar(20))); + least(cast(100 as tinyint), cast(101 as float), cast(102 as varc | +=================================================================== + 100 | + +taos> select LEAST(cast(100 as varchar(20)), cast(101 as float), cast(102 as tinyint)); + least(cast(100 as varchar(20)), cast(101 as float), cast(102 as | +=================================================================== + 100 | + +taos> select LEAST('a','b','c','d','e','f','g','h','1231','15155'); + least('a','b','c','d','e','f','g','h','1231','15155') | +======================================================== + 1231 | + +taos> select LEAST(current, voltage, phase, id, nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; + least(current, voltage, phase, id, nch1, nch2, var1, var2) | +============================================================= + 0 | + 0 | + 0 | + 0 | + 0 | + 0 | + 0 | + 0 | + 0 | + 0 | + +taos> select LEAST(current, voltage, phase, id) from ts_4893.meters order by ts limit 10; + least(current, voltage, phase, id) | +===================================== + 0 | + 0.13883 | + 0.796942 | + 0.53733 | + 0.31343 | + 0.332767 | + 0.846763 | + 0.637813 | + 0.115989 | + 0.373575 | + +taos> select LEAST(nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; + least(nch1, nch2, var1, var2) | +================================= + e | + f | + c | + e | + b | + g | + again | + b | + c | + again | + +taos> select LEAST(221, voltage) from ts_4893.meters order by ts limit 10; + least(221, voltage) | +======================== + 221 | + 220 | + 215 | + 216 | + 219 | + 221 | + 215 | + 217 | + 216 | + 221 | + +taos> select LEAST(5, id) from ts_4893.meters order by ts limit 10; + least(5, id) | +======================== + 0 | + 1 | + 2 | + 3 | + 4 | + 5 | + 5 | + 5 | + 5 | + 5 | + +taos> select LEAST('r', nch1) from ts_4893.meters order by ts limit 10; + least('r', nch1) | +================================= + novel | + r | + r | + prision | + novel | + novel | + again | + r | + novel | + again | + +taos> select LEAST('r', nch1, nch2) from ts_4893.meters order by ts limit 10; + least('r', nch1, nch2) | +================================= + novel | + f | + c | + prision | + e | + novel | + again | + d | + c | + again | + +taos> select LEAST('r', var1) from ts_4893.meters order by ts limit 10; + least('r', var1) | +================================= + novel | + person | + novel | + plate | + r | + r | + prision | + person | + prision | + plate | + +taos> select LEAST('r', var1, var2) from ts_4893.meters order by ts limit 10; + least('r', var1, var2) | +================================= + e | + person | + novel | + e | + b | + g | + b | + b | + e | + plate | + +taos> select LEAST('二中文测试', nch1) from ts_4893.meters order by ts limit 10; + least('二中文测试', nch1) | +================================= + novel | + 一二三四五六七八九十 | + update | + prision | + novel | + novel | + again | + 一二三四五六七八九十 | + novel | + again | + +taos> select LEAST('二中文测试', nch1, nch2) from ts_4893.meters order by ts limit 10; + least('二中文测试', nch1, nch2) | +======================================= + novel | + f | + c | + prision | + e | + novel | + again | + d | + c | + again | + +taos> select LEAST('二中文测试', var1) from ts_4893.meters order by ts limit 10; + least('二中文测试', var1) | +================================= + novel | + person | + novel | + plate | + 一二三四五六七八九十 | + update | + prision | + person | + prision | + plate | + +taos> select LEAST('二中文测试', var1, var2) from ts_4893.meters order by ts limit 10; + least('二中文测试', var1, var2) | +======================================= + e | + person | + novel | + e | + b | + g | + b | + b | + e | + plate | + +taos> select LEAST('23', 3443434343434343); + least('23', 3443434343434343) | +================================ + 23 | + +taos> select LEAST(co, 3443434343434343) from ts_4893.n1; + least(co, 3443434343434343) | +============================== + 23 | + 23 | + 23 | + +taos> select LEAST('23', 3443434343434343) from ts_4893.n1; + least('23', 3443434343434343) | +================================ + 23 | + 23 | + 23 | + +taos> select LEAST('23', 1443434343434343) from ts_4893.n1; + least('23', 1443434343434343) | +================================ + 23 | + 23 | + 23 | + +taos> select LEAST(current, voltage) from ts_4893.n1; + least(current, voltage) | +========================== + NULL | + NULL | + 3 | + +taos> select LEAST(current, voltage, '15') from ts_4893.n1; + least(current, voltage, '15') | +================================ + NULL | + NULL | + 3 | + diff --git a/tests/army/query/function/ans/max.csv b/tests/army/query/function/ans/max.csv index 40f8b2ac23..e13a26847b 100644 --- a/tests/army/query/function/ans/max.csv +++ b/tests/army/query/function/ans/max.csv @@ -576,7 +576,7 @@ taos> select max(total_voltage) from (select sum(voltage) as total_voltage from taos> select round(max(current), 2) from ts_4893.meters round(max(current), 2) | ========================= - 1.2000000e+01 | + 12 | taos> select pow(max(current), 2) from ts_4893.meters pow(max(current), 2) | @@ -651,7 +651,7 @@ taos> select max(cast(10000000000 as bigint unsigned)) taos> select max(cast(1.1 as float)) max(cast(1.1 as float)) | ========================== - 1.1000000e+00 | + 1.1 | taos> select max(cast(1.1 as double)) max(cast(1.1 as double)) | diff --git a/tests/army/query/function/ans/min.csv b/tests/army/query/function/ans/min.csv index d77744a4f9..6426276446 100644 --- a/tests/army/query/function/ans/min.csv +++ b/tests/army/query/function/ans/min.csv @@ -576,7 +576,7 @@ taos> select min(total_voltage) from (select sum(voltage) as total_voltage from taos> select round(min(current), 2) from ts_4893.meters round(min(current), 2) | ========================= - 8.0000000e+00 | + 8 | taos> select pow(min(current), 2) from ts_4893.meters pow(min(current), 2) | @@ -651,7 +651,7 @@ taos> select min(cast(10000000000 as bigint unsigned)) taos> select min(cast(1.1 as float)) min(cast(1.1 as float)) | ========================== - 1.1000000e+00 | + 1.1 | taos> select min(cast(1.1 as double)) min(cast(1.1 as double)) | diff --git a/tests/army/query/function/ans/mod.csv b/tests/army/query/function/ans/mod.csv index 9d2232f49d..c222257308 100644 --- a/tests/army/query/function/ans/mod.csv +++ b/tests/army/query/function/ans/mod.csv @@ -42,26 +42,26 @@ taos> select MOD(10.55, 1) + 1 taos> select MOD(MOD(MOD(MOD(MOD(MOD(MOD(123.123456789, 9), 8), 7), 6), 5), 4), 3) mod(mod(mod(mod(mod(mod(mod(123.123456789, 9), 8), 7), 6), 5), 4 | =================================================================== - 1.234567890000022e-01 | + 0.123456789000002 | taos> select MOD(MOD(MOD(MOD(MOD(MOD(MOD(123456789.123456789, -1), -2), -3), -4), -5), -6), -7) mod(mod(mod(mod(mod(mod(mod(123456789.123456789, -1), -2), -3), | =================================================================== - 1.234567910432816e-01 | + 0.123456791043282 | taos> select MOD(87654321.123456789, id + 1) from ts_4893.meters order by ts limit 10 mod(87654321.123456789, id + 1) | ================================== - 1.234567910432816e-01 | - 1.123456791043282e+00 | - 1.234567910432816e-01 | - 1.123456791043282e+00 | - 1.123456791043282e+00 | - 3.123456791043282e+00 | - 6.123456791043282e+00 | - 1.123456791043282e+00 | - 1.234567910432816e-01 | - 1.123456791043282e+00 | + 0.123456791043282 | + 1.12345679104328 | + 0.123456791043282 | + 1.12345679104328 | + 1.12345679104328 | + 3.12345679104328 | + 6.12345679104328 | + 1.12345679104328 | + 0.123456791043282 | + 1.12345679104328 | taos> select MOD(current, id + 1) from ts_4893.meters order by ts limit 10 mod(current, id + 1) | @@ -94,16 +94,16 @@ taos> select MOD(current, 1) from ts_4893.meters order by ts limit 10 taos> select MOD(sqrt(current), abs(id + 1)) from ts_4893.meters order by ts limit 10 mod(sqrt(current), abs(id + 1)) | ================================== - 2.634337159700784e-01 | - 9.281394021770111e-01 | - 1.296964830944782e-01 | - 3.351566768190027e+00 | - 3.272002495118848e+00 | - 2.916847677517688e+00 | - 3.097741066924800e+00 | - 3.310891102586806e+00 | - 3.350522322288470e+00 | - 3.215120509901375e+00 | + 0.263433715970078 | + 0.928139402177011 | + 0.129696483094478 | + 3.35156676819003 | + 3.27200249511885 | + 2.91684767751769 | + 3.0977410669248 | + 3.31089110258681 | + 3.35052232228847 | + 3.21512050990137 | taos> select mod(10, -3) mod(10, -3) | diff --git a/tests/army/query/function/ans/pi.csv b/tests/army/query/function/ans/pi.csv index fb7d662ca3..a364f472c6 100644 --- a/tests/army/query/function/ans/pi.csv +++ b/tests/army/query/function/ans/pi.csv @@ -190,5 +190,5 @@ taos> select voltage / pi() from ts_4893.meters limit 1 taos> select id, case when voltage > 100 then pi() else pi() / 2 end from ts_4893.meters limit 1 id | case when voltage > 100 then pi() else pi() / 2 end | ==================================================================== - 0 | 3.141592653589793e+00 | + 0 | 3.14159265358979 | diff --git a/tests/army/query/function/ans/position.csv b/tests/army/query/function/ans/position.csv index 1547564322..8e082ef574 100644 --- a/tests/army/query/function/ans/position.csv +++ b/tests/army/query/function/ans/position.csv @@ -201,7 +201,7 @@ taos> select ABS(POSITION('aaa' IN 'aaaaaaaaa')) taos> select POW(POSITION('aaa' IN 'aaaaaaaaa'), 2) pow(position('aaa' in 'aaaaaaaaa'), 2) | ========================================= - 1.000000000000000e+00 | + 1 | taos> select position('t' in 'taos') position('t' in 'taos') | diff --git a/tests/army/query/function/ans/rand.csv b/tests/army/query/function/ans/rand.csv index 2e87f3404f..5685bd901e 100644 --- a/tests/army/query/function/ans/rand.csv +++ b/tests/army/query/function/ans/rand.csv @@ -50,12 +50,12 @@ taos> select rand(-1) taos> select rand(12345678901234567890) rand(12345678901234567890) | ============================= - 2.008294813338805e-01 | + 0.20082948133388 | taos> select rand(-12345678901234567890) rand(-12345678901234567890) | ============================== - 8.401877171547095e-01 | + 0.84018771715471 | taos> select rand(12345), rand(12345) rand(12345) | rand(12345) | diff --git a/tests/army/query/function/ans/round.csv b/tests/army/query/function/ans/round.csv index 2c185abd73..4ae3a85376 100644 --- a/tests/army/query/function/ans/round.csv +++ b/tests/army/query/function/ans/round.csv @@ -67,12 +67,12 @@ taos> select ROUND(10.55, 1) + 1 taos> select ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(123.123456789, 9), 8), 7), 6), 5), 4)) round(round(round(round(round(round(round(123.123456789, 9), 8), | =================================================================== - 1.230000000000000e+02 | + 123 | taos> select ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(123456789.123456789, -1), -2), -3), -4), -5), -6)) round(round(round(round(round(round(round(123456789.123456789, - | =================================================================== - 1.230000000000000e+08 | + 123000000 | taos> select ROUND(current) from ts_4893.meters order by ts limit 20 round(current) | @@ -101,16 +101,16 @@ taos> select ROUND(current) from ts_4893.meters order by ts limit 20 taos> select ROUND(87654321.123456789, id) from ts_4893.meters order by ts limit 10 round(87654321.123456789, id) | ================================ - 8.765432100000000e+07 | - 8.765432109999999e+07 | - 8.765432112000000e+07 | - 8.765432112300000e+07 | - 8.765432112350000e+07 | - 8.765432112345999e+07 | - 8.765432112345700e+07 | - 8.765432112345681e+07 | - 8.765432112345679e+07 | - 8.765432112345679e+07 | + 87654321 | + 87654321.1 | + 87654321.12 | + 87654321.123 | + 87654321.1235 | + 87654321.12346 | + 87654321.123457 | + 87654321.1234568 | + 87654321.1234568 | + 87654321.1234568 | taos> select ROUND(current, id) from ts_4893.meters order by ts limit 10 round(current, id) | @@ -286,7 +286,7 @@ taos> select round(voltage, -1) from ts_4893.meters limit 1 taos> select round(current * voltage, 2) from ts_4893.meters limit 1 round(current * voltage, 2) | ============================== - 2.353650000000000e+03 | + 2353.65 | taos> select round(abs(voltage), 2) from ts_4893.meters limit 1 round(abs(voltage), 2) | diff --git a/tests/army/query/function/ans/sign.csv b/tests/army/query/function/ans/sign.csv index f8fc961c2b..700e88fde6 100644 --- a/tests/army/query/function/ans/sign.csv +++ b/tests/army/query/function/ans/sign.csv @@ -164,7 +164,7 @@ taos> select sign(cast(1 as bigint unsigned)) taos> select sign(cast(1 as float)) sign(cast(1 as float)) | ========================= - 1.0000000e+00 | + 1 | taos> select sign(cast(1 as double)) sign(cast(1 as double)) | @@ -316,30 +316,30 @@ taos> select sign(current) from ts_4893.meters order by ts limit 10 taos> select sign(cast(current as float)) from ts_4893.d0 order by ts limit 10 sign(cast(current as float)) | =============================== - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | taos> select sign(cast(current as float)) from ts_4893.meters order by ts limit 10 sign(cast(current as float)) | =============================== - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | - 1.0000000e+00 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | + 1 | taos> select sign(null) sign(null) | diff --git a/tests/army/query/function/ans/stddev.csv b/tests/army/query/function/ans/stddev.csv index c0f93a9bcd..02f83a7742 100644 --- a/tests/army/query/function/ans/stddev.csv +++ b/tests/army/query/function/ans/stddev.csv @@ -109,17 +109,17 @@ taos> select stddev_pop(total_voltage) from (select sum(voltage) as total_voltag taos> select round(stddev_pop(current), 2) from ts_4893.meters round(stddev_pop(current), 2) | ================================ - 1.150000000000000e+00 | + 1.15 | taos> select pow(stddev_pop(current), 2) from ts_4893.meters pow(stddev_pop(current), 2) | ============================== - 1.332500071133751e+00 | + 1.33250007113375 | taos> select log(stddev_pop(voltage) + 1) from ts_4893.meters log(stddev_pop(voltage) + 1) | =============================== - 1.354922290183882e+00 | + 1.35492229018388 | taos> select groupid, stddev_pop(voltage) from ts_4893.meters group by groupid order by groupid groupid | stddev_pop(voltage) | diff --git a/tests/army/query/function/ans/trunc.csv b/tests/army/query/function/ans/trunc.csv index 15411afbf3..8d1316b760 100644 --- a/tests/army/query/function/ans/trunc.csv +++ b/tests/army/query/function/ans/trunc.csv @@ -47,40 +47,40 @@ taos> select TRUNCATE(10.55, 1) + 1 taos> select TRUNCATE(TRUNCATE(TRUNCATE(TRUNCATE(TRUNCATE(TRUNCATE(TRUNCATE(123.123456789, 9), 8), 7), 6), 5), 4), 3) truncate(truncate(truncate(truncate(truncate(truncate(truncate(1 | =================================================================== - 1.231230000000000e+02 | + 123.123 | taos> select TRUNCATE(TRUNCATE(TRUNCATE(TRUNCATE(TRUNCATE(TRUNCATE(TRUNCATE(123456789.123456789, -1), -2), -3), -4), -5), -6), -7) truncate(truncate(truncate(truncate(truncate(truncate(truncate(1 | =================================================================== - 1.200000000000000e+08 | + 120000000 | taos> select TRUNCATE(87654321.123456789, id) from ts_4893.meters order by ts limit 10 truncate(87654321.123456789, id) | =================================== - 8.765432100000000e+07 | - 8.765432109999999e+07 | - 8.765432112000000e+07 | - 8.765432112300000e+07 | - 8.765432112340000e+07 | - 8.765432112345000e+07 | - 8.765432112345600e+07 | - 8.765432112345670e+07 | - 8.765432112345679e+07 | - 8.765432112345679e+07 | + 87654321 | + 87654321.1 | + 87654321.12 | + 87654321.123 | + 87654321.1234 | + 87654321.12345 | + 87654321.123456 | + 87654321.1234567 | + 87654321.1234568 | + 87654321.1234568 | taos> select TRUNCATE(current, id) from ts_4893.meters order by ts limit 10 truncate(current, id) | ======================== - 1.0000000e+01 | - 8.5000000e+00 | - 9.7900000e+00 | - 1.1233000e+01 | - 1.0706000e+01 | - 8.5080004e+00 | - 9.5959997e+00 | - 1.0962000e+01 | - 1.1226000e+01 | - 1.0337000e+01 | + 10 | + 8.5 | + 9.79 | + 11.233 | + 10.706 | + 8.508 | + 9.596 | + 10.962 | + 11.226 | + 10.337 | taos> select TRUNCATE(current, 1) from ts_4893.meters order by ts limit 10 truncate(current, 1) | @@ -144,26 +144,26 @@ taos> select TRUNC(10.55, 1) + 1 taos> select TRUNC(TRUNC(TRUNC(TRUNC(TRUNC(TRUNC(TRUNC(123.123456789, 9), 8), 7), 6), 5), 4), 3) trunc(trunc(trunc(trunc(trunc(trunc(trunc(123.123456789, 9), 8), | =================================================================== - 1.231230000000000e+02 | + 123.123 | taos> select TRUNC(TRUNC(TRUNC(TRUNC(TRUNC(TRUNC(TRUNC(123456789.123456789, -1), -2), -3), -4), -5), -6), -7) trunc(trunc(trunc(trunc(trunc(trunc(trunc(123456789.123456789, - | =================================================================== - 1.200000000000000e+08 | + 120000000 | taos> select TRUNC(87654321.123456789, id) from ts_4893.meters order by ts limit 10 trunc(87654321.123456789, id) | ================================ - 8.765432100000000e+07 | - 8.765432109999999e+07 | - 8.765432112000000e+07 | - 8.765432112300000e+07 | - 8.765432112340000e+07 | - 8.765432112345000e+07 | - 8.765432112345600e+07 | - 8.765432112345670e+07 | - 8.765432112345679e+07 | - 8.765432112345679e+07 | + 87654321 | + 87654321.1 | + 87654321.12 | + 87654321.123 | + 87654321.1234 | + 87654321.12345 | + 87654321.123456 | + 87654321.1234567 | + 87654321.1234568 | + 87654321.1234568 | taos> select TRUNC(current, id) from ts_4893.meters order by ts limit 10 trunc(current, id) | @@ -289,7 +289,7 @@ taos> select truncate(100.9876, 2) taos> select truncate(99999999999999.9999, 2) truncate(99999999999999.9999, 2) | =================================== - 1.000000000000000e+14 | + 100000000000000 | taos> select truncate(-5.678, 2) truncate(-5.678, 2) | @@ -314,7 +314,7 @@ taos> select truncate(phase, 3) from ts_4893.meters limit 1 taos> select truncate(voltage + current, 2) from ts_4893.meters limit 1 truncate(voltage + current, 2) | ================================= - 2.316400000000000e+02 | + 231.64 | taos> select truncate(voltage, -1) from ts_4893.meters limit 1 truncate(voltage, -1) | @@ -329,7 +329,7 @@ taos> select round(truncate(voltage, 1), 2) from ts_4893.meters limit 1 taos> select truncate(abs(current), 1) from ts_4893.meters limit 1 truncate(abs(current), 1) | ============================ - 1.0600000e+01 | + 10.6 | taos> select truncate(exp(phase), 2) from ts_4893.meters limit 1 truncate(exp(phase), 2) | diff --git a/tests/army/query/function/ans/varpop.csv b/tests/army/query/function/ans/varpop.csv index 829996d978..ec5d479f3a 100644 --- a/tests/army/query/function/ans/varpop.csv +++ b/tests/army/query/function/ans/varpop.csv @@ -99,7 +99,7 @@ taos> select var_pop(total_voltage) from (select sum(voltage) as total_voltage f taos> select round(var_pop(current), 2) from ts_4893.meters round(var_pop(current), 2) | ============================= - 1.330000000000000e+00 | + 1.33 | taos> select pow(var_pop(current), 2) from ts_4893.meters pow(var_pop(current), 2) | diff --git a/tests/army/query/function/in/greatest.in b/tests/army/query/function/in/greatest.in new file mode 100644 index 0000000000..098ef626e9 --- /dev/null +++ b/tests/army/query/function/in/greatest.in @@ -0,0 +1,232 @@ +alter local 'compareAsStrInGreatest' '1'; +select GREATEST(1,2,3,4,5,6,7,8,9,10); +select GREATEST(1,1.1,2.23,3.4444,5.66666666,6.21241241,7.999999999999); +select GREATEST(1,'2',3.3,4.4,5); +select GREATEST(121,'18'); +select GREATEST(18888,'18'); +select GREATEST(1,2,3,4,5,'5.1'); +select GREATEST('1','2','3','4',5); +select GREATEST('1','2','3','4','5'); +select GREATEST(1,2,3,4,5,6,7,'a','b','一','二','三'); +select GREATEST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213'); +select GREATEST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.123123'); +select GREATEST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); +select GREATEST(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); +select GREATEST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20)), cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double)); +select GREATEST(cast(100 as tinyint), cast(101 as tinyint)); +select GREATEST(cast(100 as tinyint), cast(101 as smallint)); +select GREATEST(cast(100 as tinyint), cast(101 as int)); +select GREATEST(cast(100 as tinyint), cast(101 as bigint)); +select GREATEST(cast(100 as tinyint), cast(101 as float)); +select GREATEST(cast(100 as tinyint), cast(101 as double)); +select GREATEST(cast(100 as tinyint), cast(101 as varchar(20))); +select GREATEST(cast(100 as tinyint), cast(101 as nchar(20))); +select GREATEST(cast(101 as tinyint), cast(100 as tinyint)); +select GREATEST(cast(101 as tinyint), cast(100 as smallint)); +select GREATEST(cast(101 as tinyint), cast(100 as int)); +select GREATEST(cast(101 as tinyint), cast(100 as bigint)); +select GREATEST(cast(101 as tinyint), cast(100 as float)); +select GREATEST(cast(101 as tinyint), cast(100 as double)); +select GREATEST(cast(101 as tinyint), cast(100 as varchar(20))); +select GREATEST(cast(101 as tinyint), cast(100 as nchar(20))); +select GREATEST(cast(1000 as smallint), cast(1001 as smallint)); +select GREATEST(cast(1000 as smallint), cast(1001 as int)); +select GREATEST(cast(1000 as smallint), cast(1001 as bigint)); +select GREATEST(cast(1000 as smallint), cast(1001 as float)); +select GREATEST(cast(1000 as smallint), cast(1001 as double)); +select GREATEST(cast(1000 as smallint), cast(1001 as varchar(20))); +select GREATEST(cast(1000 as smallint), cast(1001 as nchar(20))); +select GREATEST(cast(1001 as smallint), cast(1000 as smallint)); +select GREATEST(cast(1001 as smallint), cast(1000 as int)); +select GREATEST(cast(1001 as smallint), cast(1000 as bigint)); +select GREATEST(cast(1001 as smallint), cast(1000 as float)); +select GREATEST(cast(1001 as smallint), cast(1000 as double)); +select GREATEST(cast(1001 as smallint), cast(1000 as varchar(20))); +select GREATEST(cast(1001 as smallint), cast(1000 as nchar(20))); +select GREATEST(cast(1000000 as int), cast(1000001 as int)); +select GREATEST(cast(1000000 as int), cast(1000001 as bigint)); +select GREATEST(cast(1000000 as int), cast(1000001 as float)); +select GREATEST(cast(1000000 as int), cast(1000001 as double)); +select GREATEST(cast(1000000 as int), cast(1000001 as varchar(20))); +select GREATEST(cast(1000000 as int), cast(1000001 as nchar(20))); +select GREATEST(cast(1000001 as int), cast(1000000 as int)); +select GREATEST(cast(1000001 as int), cast(1000000 as bigint)); +select GREATEST(cast(1000001 as int), cast(1000000 as float)); +select GREATEST(cast(1000001 as int), cast(1000000 as double)); +select GREATEST(cast(1000001 as int), cast(1000000 as varchar(20))); +select GREATEST(cast(1000001 as int), cast(1000000 as nchar(20))); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as bigint)); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as float)); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as double)); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as varchar(20))); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as nchar(20))); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as bigint)); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as float)); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as double)); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as varchar(20))); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as nchar(20))); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as float)); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as double)); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as timestamp)); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as varchar(20))); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as nchar(20))); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as float)); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as double)); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as timestamp)); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as varchar(20))); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as nchar(20))); +select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as double)); +select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as timestamp)); +select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as varchar(20))); +select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as nchar(20))); +select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as double)); +select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as timestamp)); +select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as varchar(20))); +select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as nchar(20))); +select GREATEST(cast('中文测试' as varchar(20)), cast('中文测试一' as varchar(20))); +select GREATEST(cast('中文测试' as varchar(20)), cast('中文测试一' as nchar(20))); +select GREATEST(cast('中文测试一' as varchar(20)), cast('中文测试' as varchar(20))); +select GREATEST(cast('中文测试一' as varchar(20)), cast('中文测试' as nchar(20))); +select GREATEST(cast('abc123abc' as varchar(20)), cast('abc124abc' as varchar(20))); +select GREATEST(cast('abc123abc' as varchar(20)), cast('abc124abc' as nchar(20))); +select GREATEST(cast('abc124abc' as varchar(20)), cast('abc123abc' as varchar(20))); +select GREATEST(cast('abc124abc' as varchar(20)), cast('abc123abc' as nchar(20))); +select GREATEST(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar(20))); +select GREATEST(cast(100 as tinyint), cast(101 as float), cast(102 as varchar(20))); +select GREATEST(cast(100 as float), cast(101 as tinyint), cast(102 as varchar(20))); +select GREATEST(cast(100 as float), cast(101 as varchar(20)), cast(102 as tinyint)); +select GREATEST(cast(100 as varchar(20)), cast(101 as float), cast(102 as tinyint)); +select GREATEST('a','b','c','d','e','f','g','h','1231','15155'); +select GREATEST(current, voltage, phase, id, nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; +select GREATEST(current, voltage, phase, id) from ts_4893.meters order by ts limit 10; +select GREATEST(nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; +select GREATEST(221, voltage) from ts_4893.meters order by ts limit 10; +select GREATEST(5, id) from ts_4893.meters order by ts limit 10; +select GREATEST('r', nch1) from ts_4893.meters order by ts limit 10; +select GREATEST('r', nch1, nch2) from ts_4893.meters order by ts limit 10; +select GREATEST('r', var1) from ts_4893.meters order by ts limit 10; +select GREATEST('r', var1, var2) from ts_4893.meters order by ts limit 10; +select GREATEST('二中文测试', nch1) from ts_4893.meters order by ts limit 10; +select GREATEST('二中文测试', nch1, nch2) from ts_4893.meters order by ts limit 10; +select GREATEST('二中文测试', var1) from ts_4893.meters order by ts limit 10; +select GREATEST('二中文测试', var1, var2) from ts_4893.meters order by ts limit 10; +select GREATEST('23', 3443434343434343); +select GREATEST(co, 3443434343434343) from ts_4893.n1; +select GREATEST('23', 3443434343434343) from ts_4893.n1; +select GREATEST('23', 1443434343434343) from ts_4893.n1; +select GREATEST(current, voltage) from ts_4893.n1; +select GREATEST(current, voltage, '15') from ts_4893.n1; +alter local 'compareAsStrInGreatest' '0'; +select GREATEST(1,'2',3.3,4.4,5); +select GREATEST(1,2,3,4,5,'5.1'); +select GREATEST(121,'18'); +select GREATEST('1','2','3','4','5'); +select GREATEST(1,2,3,4,5,6,7,'a','b','一','二','三'); +select GREATEST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213'); +select GREATEST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.123123'); +select GREATEST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); +select GREATEST(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); +select GREATEST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20)), cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double)); +select GREATEST(cast(100 as tinyint), cast(101 as tinyint)); +select GREATEST(cast(100 as tinyint), cast(101 as smallint)); +select GREATEST(cast(100 as tinyint), cast(101 as int)); +select GREATEST(cast(100 as tinyint), cast(101 as bigint)); +select GREATEST(cast(100 as tinyint), cast(101 as float)); +select GREATEST(cast(100 as tinyint), cast(101 as double)); +select GREATEST(cast(100 as tinyint), cast(101 as varchar(20))); +select GREATEST(cast(100 as tinyint), cast(101 as nchar(20))); +select GREATEST(cast(101 as tinyint), cast(100 as tinyint)); +select GREATEST(cast(101 as tinyint), cast(100 as smallint)); +select GREATEST(cast(101 as tinyint), cast(100 as int)); +select GREATEST(cast(101 as tinyint), cast(100 as bigint)); +select GREATEST(cast(101 as tinyint), cast(100 as float)); +select GREATEST(cast(101 as tinyint), cast(100 as double)); +select GREATEST(cast(101 as tinyint), cast(100 as varchar(20))); +select GREATEST(cast(101 as tinyint), cast(100 as nchar(20))); +select GREATEST(cast(1000 as smallint), cast(1001 as smallint)); +select GREATEST(cast(1000 as smallint), cast(1001 as int)); +select GREATEST(cast(1000 as smallint), cast(1001 as bigint)); +select GREATEST(cast(1000 as smallint), cast(1001 as float)); +select GREATEST(cast(1000 as smallint), cast(1001 as double)); +select GREATEST(cast(1000 as smallint), cast(1001 as varchar(20))); +select GREATEST(cast(1000 as smallint), cast(1001 as nchar(20))); +select GREATEST(cast(1001 as smallint), cast(1000 as smallint)); +select GREATEST(cast(1001 as smallint), cast(1000 as int)); +select GREATEST(cast(1001 as smallint), cast(1000 as bigint)); +select GREATEST(cast(1001 as smallint), cast(1000 as float)); +select GREATEST(cast(1001 as smallint), cast(1000 as double)); +select GREATEST(cast(1001 as smallint), cast(1000 as varchar(20))); +select GREATEST(cast(1001 as smallint), cast(1000 as nchar(20))); +select GREATEST(cast(1000000 as int), cast(1000001 as int)); +select GREATEST(cast(1000000 as int), cast(1000001 as bigint)); +select GREATEST(cast(1000000 as int), cast(1000001 as float)); +select GREATEST(cast(1000000 as int), cast(1000001 as double)); +select GREATEST(cast(1000000 as int), cast(1000001 as varchar(20))); +select GREATEST(cast(1000000 as int), cast(1000001 as nchar(20))); +select GREATEST(cast(1000001 as int), cast(1000000 as int)); +select GREATEST(cast(1000001 as int), cast(1000000 as bigint)); +select GREATEST(cast(1000001 as int), cast(1000000 as float)); +select GREATEST(cast(1000001 as int), cast(1000000 as double)); +select GREATEST(cast(1000001 as int), cast(1000000 as varchar(20))); +select GREATEST(cast(1000001 as int), cast(1000000 as nchar(20))); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as bigint)); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as float)); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as double)); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as varchar(20))); +select GREATEST(cast(1000000000 as bigint), cast(1000000001 as nchar(20))); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as bigint)); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as float)); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as double)); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as varchar(20))); +select GREATEST(cast(1000000001 as bigint), cast(1000000000 as nchar(20))); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as float)); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as double)); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as timestamp)); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as varchar(20))); +select GREATEST(cast(100000.1111111 as float), cast(100001.1111111 as nchar(20))); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as float)); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as double)); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as timestamp)); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as varchar(20))); +select GREATEST(cast(100001.1111111 as float), cast(100000.1111111 as nchar(20))); +select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as double)); +select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as timestamp)); +select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as varchar(20))); +select GREATEST(cast(100000.1111111 as double), cast(100001.1111111 as nchar(20))); +select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as double)); +select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as timestamp)); +select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as varchar(20))); +select GREATEST(cast(100001.1111111 as double), cast(100000.1111111 as nchar(20))); +select GREATEST(cast('中文测试' as varchar(20)), cast('中文测试一' as varchar(20))); +select GREATEST(cast('中文测试' as varchar(20)), cast('中文测试一' as nchar(20))); +select GREATEST(cast('中文测试一' as varchar(20)), cast('中文测试' as varchar(20))); +select GREATEST(cast('中文测试一' as varchar(20)), cast('中文测试' as nchar(20))); +select GREATEST(cast('abc123abc' as varchar(20)), cast('abc124abc' as varchar(20))); +select GREATEST(cast('abc123abc' as varchar(20)), cast('abc124abc' as nchar(20))); +select GREATEST(cast('abc124abc' as varchar(20)), cast('abc123abc' as varchar(20))); +select GREATEST(cast('abc124abc' as varchar(20)), cast('abc123abc' as nchar(20))); +select GREATEST(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar(20))); +select GREATEST(cast(100 as tinyint), cast(101 as float), cast(102 as varchar(20))); +select GREATEST(cast(100 as float), cast(101 as tinyint), cast(102 as varchar(20))); +select GREATEST(cast(100 as float), cast(101 as varchar(20)), cast(102 as tinyint)); +select GREATEST(cast(100 as varchar(20)), cast(101 as float), cast(102 as tinyint)); +select GREATEST('a','b','c','d','e','f','g','h','1231','15155'); +select GREATEST(current, voltage, phase, id, nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; +select GREATEST(current, voltage, phase, id) from ts_4893.meters order by ts limit 10; +select GREATEST(nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; +select GREATEST(221, voltage) from ts_4893.meters order by ts limit 10; +select GREATEST(5, id) from ts_4893.meters order by ts limit 10; +select GREATEST('r', nch1) from ts_4893.meters order by ts limit 10; +select GREATEST('r', nch1, nch2) from ts_4893.meters order by ts limit 10; +select GREATEST('r', var1) from ts_4893.meters order by ts limit 10; +select GREATEST('r', var1, var2) from ts_4893.meters order by ts limit 10; +select GREATEST('二中文测试', nch1) from ts_4893.meters order by ts limit 10; +select GREATEST('二中文测试', nch1, nch2) from ts_4893.meters order by ts limit 10; +select GREATEST('二中文测试', var1) from ts_4893.meters order by ts limit 10; +select GREATEST('二中文测试', var1, var2) from ts_4893.meters order by ts limit 10; +select GREATEST('23', 3443434343434343); +select GREATEST(co, 3443434343434343) from ts_4893.n1; +select GREATEST('23', 1443434343434343) from ts_4893.n1; +select GREATEST('23', 3443434343434343) from ts_4893.n1 +select GREATEST(current, voltage) from ts_4893.n1; +select GREATEST(current, voltage, '15') from ts_4893.n1; \ No newline at end of file diff --git a/tests/army/query/function/in/least.in b/tests/army/query/function/in/least.in new file mode 100644 index 0000000000..946d00b59b --- /dev/null +++ b/tests/army/query/function/in/least.in @@ -0,0 +1,228 @@ +alter local 'compareAsStrInGreatest' '1'; +select LEAST(1,2,3,4,5,6,7,8,9,10); +select LEAST(1,1.1,2.23,3.4444,5.66666666,6.21241241,7.999999999999); +select LEAST(1,'2',3.3,4.4,5); +select LEAST(1,2,3,4,5,'5.1'); +select LEAST('1','2','3','4',5); +select LEAST('1','2','3','4','5'); +select LEAST(1,2,3,4,5,6,7,'a','b','一','二','三'); +select LEAST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213'); +select LEAST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.123123'); +select LEAST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); +select LEAST(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); +select LEAST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20)), cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double)); +select LEAST(cast(100 as tinyint), cast(101 as tinyint)); +select LEAST(cast(100 as tinyint), cast(101 as smallint)); +select LEAST(cast(100 as tinyint), cast(101 as int)); +select LEAST(cast(100 as tinyint), cast(101 as bigint)); +select LEAST(cast(100 as tinyint), cast(101 as float)); +select LEAST(cast(100 as tinyint), cast(101 as double)); +select LEAST(cast(100 as tinyint), cast(101 as varchar(20))); +select LEAST(cast(100 as tinyint), cast(101 as nchar(20))); +select LEAST(cast(101 as tinyint), cast(100 as tinyint)); +select LEAST(cast(101 as tinyint), cast(100 as smallint)); +select LEAST(cast(101 as tinyint), cast(100 as int)); +select LEAST(cast(101 as tinyint), cast(100 as bigint)); +select LEAST(cast(101 as tinyint), cast(100 as float)); +select LEAST(cast(101 as tinyint), cast(100 as double)); +select LEAST(cast(101 as tinyint), cast(100 as varchar(20))); +select LEAST(cast(101 as tinyint), cast(100 as nchar(20))); +select LEAST(cast(1000 as smallint), cast(1001 as smallint)); +select LEAST(cast(1000 as smallint), cast(1001 as int)); +select LEAST(cast(1000 as smallint), cast(1001 as bigint)); +select LEAST(cast(1000 as smallint), cast(1001 as float)); +select LEAST(cast(1000 as smallint), cast(1001 as double)); +select LEAST(cast(1000 as smallint), cast(1001 as varchar(20))); +select LEAST(cast(1000 as smallint), cast(1001 as nchar(20))); +select LEAST(cast(1001 as smallint), cast(1000 as smallint)); +select LEAST(cast(1001 as smallint), cast(1000 as int)); +select LEAST(cast(1001 as smallint), cast(1000 as bigint)); +select LEAST(cast(1001 as smallint), cast(1000 as float)); +select LEAST(cast(1001 as smallint), cast(1000 as double)); +select LEAST(cast(1001 as smallint), cast(1000 as varchar(20))); +select LEAST(cast(1001 as smallint), cast(1000 as nchar(20))); +select LEAST(cast(1000000 as int), cast(1000001 as int)); +select LEAST(cast(1000000 as int), cast(1000001 as bigint)); +select LEAST(cast(1000000 as int), cast(1000001 as float)); +select LEAST(cast(1000000 as int), cast(1000001 as double)); +select LEAST(cast(1000000 as int), cast(1000001 as varchar(20))); +select LEAST(cast(1000000 as int), cast(1000001 as nchar(20))); +select LEAST(cast(1000001 as int), cast(1000000 as int)); +select LEAST(cast(1000001 as int), cast(1000000 as bigint)); +select LEAST(cast(1000001 as int), cast(1000000 as float)); +select LEAST(cast(1000001 as int), cast(1000000 as double)); +select LEAST(cast(1000001 as int), cast(1000000 as varchar(20))); +select LEAST(cast(1000001 as int), cast(1000000 as nchar(20))); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as bigint)); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as float)); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as double)); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as varchar(20))); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as nchar(20))); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as bigint)); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as float)); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as double)); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as varchar(20))); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as nchar(20))); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as float)); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as double)); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as timestamp)); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as varchar(20))); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as nchar(20))); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as float)); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as double)); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as timestamp)); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as varchar(20))); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as nchar(20))); +select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as double)); +select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as timestamp)); +select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as varchar(20))); +select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as nchar(20))); +select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as double)); +select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as timestamp)); +select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as varchar(20))); +select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as nchar(20))); +select LEAST(cast('中文测试' as varchar(20)), cast('中文测试一' as varchar(20))); +select LEAST(cast('中文测试' as varchar(20)), cast('中文测试一' as nchar(20))); +select LEAST(cast('中文测试一' as varchar(20)), cast('中文测试' as varchar(20))); +select LEAST(cast('中文测试一' as varchar(20)), cast('中文测试' as nchar(20))); +select LEAST(cast('abc123abc' as varchar(20)), cast('abc124abc' as varchar(20))); +select LEAST(cast('abc123abc' as varchar(20)), cast('abc124abc' as nchar(20))); +select LEAST(cast('abc124abc' as varchar(20)), cast('abc123abc' as varchar(20))); +select LEAST(cast('abc124abc' as varchar(20)), cast('abc123abc' as nchar(20))); +select LEAST(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar(20))); +select LEAST(cast(100 as tinyint), cast(101 as float), cast(102 as varchar(20))); +select LEAST(cast(100 as varchar(20)), cast(101 as float), cast(102 as tinyint)); +select LEAST('a','b','c','d','e','f','g','h','1231','15155'); +select LEAST(current, voltage, phase, id, nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; +select LEAST(current, voltage, phase, id) from ts_4893.meters order by ts limit 10; +select LEAST(nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; +select LEAST(221, voltage) from ts_4893.meters order by ts limit 10; +select LEAST(5, id) from ts_4893.meters order by ts limit 10; +select LEAST('r', nch1) from ts_4893.meters order by ts limit 10; +select LEAST('r', nch1, nch2) from ts_4893.meters order by ts limit 10; +select LEAST('r', var1) from ts_4893.meters order by ts limit 10; +select LEAST('r', var1, var2) from ts_4893.meters order by ts limit 10; +select LEAST('二中文测试', nch1) from ts_4893.meters order by ts limit 10; +select LEAST('二中文测试', nch1, nch2) from ts_4893.meters order by ts limit 10; +select LEAST('二中文测试', var1) from ts_4893.meters order by ts limit 10; +select LEAST('二中文测试', var1, var2) from ts_4893.meters order by ts limit 10; +select LEAST('23', 3443434343434343); +select LEAST(co, 3443434343434343) from ts_4893.n1; +select LEAST('23', 3443434343434343) from ts_4893.n1; +select LEAST('23', 1443434343434343) from ts_4893.n1; +select LEAST(current, voltage) from ts_4893.n1; +select LEAST(current, voltage, '15') from ts_4893.n1; +alter local 'compareAsStrInGreatest' '0'; +select LEAST(1,2,3,4,5,6,7,8,9,10); +select LEAST(1,1.1,2.23,3.4444,5.66666666,6.21241241,7.999999999999); +select LEAST(1,'2',3.3,4.4,5); +select LEAST(1,2,3,4,5,'5.1'); +select LEAST('1','2','3','4',5); +select LEAST('1','2','3','4','5'); +select LEAST(1,2,3,4,5,6,7,'a','b','一','二','三'); +select LEAST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213'); +select LEAST(1,2,3,4,5,6,7,'a','b','c','1','2','1231213','1231213.123123'); +select LEAST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); +select LEAST(cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20))); +select LEAST(cast(0 as bool), cast(1 as tinyint), cast(2 as smallint), cast(3 as int), cast(4 as bigint), cast(5 as float), cast(6 as double), cast(8 as varchar(20)), cast(9 as nchar(20)), cast(0 as bool), cast(1 as tinyint unsigned), cast(2 as smallint unsigned), cast(3 as int unsigned), cast(4 as bigint unsigned), cast(5 as float), cast(6 as double)); +select LEAST(cast(100 as tinyint), cast(101 as tinyint)); +select LEAST(cast(100 as tinyint), cast(101 as smallint)); +select LEAST(cast(100 as tinyint), cast(101 as int)); +select LEAST(cast(100 as tinyint), cast(101 as bigint)); +select LEAST(cast(100 as tinyint), cast(101 as float)); +select LEAST(cast(100 as tinyint), cast(101 as double)); +select LEAST(cast(100 as tinyint), cast(101 as varchar(20))); +select LEAST(cast(100 as tinyint), cast(101 as nchar(20))); +select LEAST(cast(101 as tinyint), cast(100 as tinyint)); +select LEAST(cast(101 as tinyint), cast(100 as smallint)); +select LEAST(cast(101 as tinyint), cast(100 as int)); +select LEAST(cast(101 as tinyint), cast(100 as bigint)); +select LEAST(cast(101 as tinyint), cast(100 as float)); +select LEAST(cast(101 as tinyint), cast(100 as double)); +select LEAST(cast(101 as tinyint), cast(100 as varchar(20))); +select LEAST(cast(101 as tinyint), cast(100 as nchar(20))); +select LEAST(cast(1000 as smallint), cast(1001 as smallint)); +select LEAST(cast(1000 as smallint), cast(1001 as int)); +select LEAST(cast(1000 as smallint), cast(1001 as bigint)); +select LEAST(cast(1000 as smallint), cast(1001 as float)); +select LEAST(cast(1000 as smallint), cast(1001 as double)); +select LEAST(cast(1000 as smallint), cast(1001 as varchar(20))); +select LEAST(cast(1000 as smallint), cast(1001 as nchar(20))); +select LEAST(cast(1001 as smallint), cast(1000 as smallint)); +select LEAST(cast(1001 as smallint), cast(1000 as int)); +select LEAST(cast(1001 as smallint), cast(1000 as bigint)); +select LEAST(cast(1001 as smallint), cast(1000 as float)); +select LEAST(cast(1001 as smallint), cast(1000 as double)); +select LEAST(cast(1001 as smallint), cast(1000 as varchar(20))); +select LEAST(cast(1001 as smallint), cast(1000 as nchar(20))); +select LEAST(cast(1000000 as int), cast(1000001 as int)); +select LEAST(cast(1000000 as int), cast(1000001 as bigint)); +select LEAST(cast(1000000 as int), cast(1000001 as float)); +select LEAST(cast(1000000 as int), cast(1000001 as double)); +select LEAST(cast(1000000 as int), cast(1000001 as varchar(20))); +select LEAST(cast(1000000 as int), cast(1000001 as nchar(20))); +select LEAST(cast(1000001 as int), cast(1000000 as int)); +select LEAST(cast(1000001 as int), cast(1000000 as bigint)); +select LEAST(cast(1000001 as int), cast(1000000 as float)); +select LEAST(cast(1000001 as int), cast(1000000 as double)); +select LEAST(cast(1000001 as int), cast(1000000 as varchar(20))); +select LEAST(cast(1000001 as int), cast(1000000 as nchar(20))); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as bigint)); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as float)); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as double)); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as varchar(20))); +select LEAST(cast(1000000000 as bigint), cast(1000000001 as nchar(20))); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as bigint)); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as float)); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as double)); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as varchar(20))); +select LEAST(cast(1000000001 as bigint), cast(1000000000 as nchar(20))); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as float)); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as double)); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as timestamp)); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as varchar(20))); +select LEAST(cast(100000.1111111 as float), cast(100001.1111111 as nchar(20))); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as float)); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as double)); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as timestamp)); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as varchar(20))); +select LEAST(cast(100001.1111111 as float), cast(100000.1111111 as nchar(20))); +select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as double)); +select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as timestamp)); +select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as varchar(20))); +select LEAST(cast(100000.1111111 as double), cast(100001.1111111 as nchar(20))); +select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as double)); +select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as timestamp)); +select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as varchar(20))); +select LEAST(cast(100001.1111111 as double), cast(100000.1111111 as nchar(20))); +select LEAST(cast('中文测试' as varchar(20)), cast('中文测试一' as varchar(20))); +select LEAST(cast('中文测试' as varchar(20)), cast('中文测试一' as nchar(20))); +select LEAST(cast('中文测试一' as varchar(20)), cast('中文测试' as varchar(20))); +select LEAST(cast('中文测试一' as varchar(20)), cast('中文测试' as nchar(20))); +select LEAST(cast('abc123abc' as varchar(20)), cast('abc124abc' as varchar(20))); +select LEAST(cast('abc123abc' as varchar(20)), cast('abc124abc' as nchar(20))); +select LEAST(cast('abc124abc' as varchar(20)), cast('abc123abc' as varchar(20))); +select LEAST(cast('abc124abc' as varchar(20)), cast('abc123abc' as nchar(20))); +select LEAST(cast('abc123abc' as nchar(20)), cast('abc124abc' as nchar(20))); +select LEAST(cast(100 as tinyint), cast(101 as float), cast(102 as varchar(20))); +select LEAST(cast(100 as varchar(20)), cast(101 as float), cast(102 as tinyint)); +select LEAST('a','b','c','d','e','f','g','h','1231','15155'); +select LEAST(current, voltage, phase, id, nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; +select LEAST(current, voltage, phase, id) from ts_4893.meters order by ts limit 10; +select LEAST(nch1, nch2, var1, var2) from ts_4893.meters order by ts limit 10; +select LEAST(221, voltage) from ts_4893.meters order by ts limit 10; +select LEAST(5, id) from ts_4893.meters order by ts limit 10; +select LEAST('r', nch1) from ts_4893.meters order by ts limit 10; +select LEAST('r', nch1, nch2) from ts_4893.meters order by ts limit 10; +select LEAST('r', var1) from ts_4893.meters order by ts limit 10; +select LEAST('r', var1, var2) from ts_4893.meters order by ts limit 10; +select LEAST('二中文测试', nch1) from ts_4893.meters order by ts limit 10; +select LEAST('二中文测试', nch1, nch2) from ts_4893.meters order by ts limit 10; +select LEAST('二中文测试', var1) from ts_4893.meters order by ts limit 10; +select LEAST('二中文测试', var1, var2) from ts_4893.meters order by ts limit 10; +select LEAST('23', 3443434343434343); +select LEAST(co, 3443434343434343) from ts_4893.n1; +select LEAST('23', 3443434343434343) from ts_4893.n1; +select LEAST('23', 1443434343434343) from ts_4893.n1; +select LEAST(current, voltage) from ts_4893.n1; +select LEAST(current, voltage, '15') from ts_4893.n1; \ No newline at end of file diff --git a/tests/army/query/function/test_function.py b/tests/army/query/function/test_function.py index c583d08cec..27e54407cc 100644 --- a/tests/army/query/function/test_function.py +++ b/tests/army/query/function/test_function.py @@ -40,37 +40,10 @@ class TDTestCase(TBase): "`var2` VARCHAR(50)) TAGS (`groupid` TINYINT, `location` VARCHAR(16));") tdSql.execute("CREATE table d0 using meters tags(1, 'beijing')") tdSql.execute("insert into d0 file '%s'" % datafile) - - def test_normal_query(self, testCase): - # read sql from .sql file and execute - tdLog.info(f"test normal query.") - sqlFile = etool.curFile(__file__, f"in/{testCase}.in") - ansFile = etool.curFile(__file__, f"ans/{testCase}.csv") - with open(sqlFile, 'r') as sql_file: - sql_statement = '' - tdSql.csvLine = 0 - for line in sql_file: - if not line.strip() or line.strip().startswith('--'): - continue - - sql_statement += line.strip() - if sql_statement.endswith(';'): - sql_statement = sql_statement.rstrip(';') - tdSql.checkDataCsvByLine(sql_statement, ansFile) - sql_statement = '' - err_file_path = etool.curFile(__file__, f"in/{testCase}.err") - if not os.path.isfile(err_file_path): - return None - with open(err_file_path, 'r') as err_file: - err_statement = '' - for line in err_file: - if not line.strip() or line.strip().startswith('--'): - continue - - err_statement += line.strip() - if err_statement.endswith(';'): - tdSql.error(err_statement) - err_statement = '' + tdSql.execute("CREATE TABLE `n1` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, co NCHAR(10))") + tdSql.execute("insert into n1 values(now, 1, null, '23')") + tdSql.execute("insert into n1 values(now, null, 3, '23')") + tdSql.execute("insert into n1 values(now, 5, 3, '23')") def test_normal_query_new(self, testCase): # read sql from .sql file and execute @@ -310,6 +283,223 @@ class TDTestCase(TBase): tdSql.error("select * from (select to_iso8601(ts, timezone()), timezone() from ts_4893.meters \ order by ts desc) limit 1000;", expectErrInfo="Invalid parameter data type : to_iso8601") # TS-5340 + def test_greatest(self): + self.test_normal_query_new("greatest") + + tdSql.execute("alter local 'compareAsStrInGreatest' '1';") + + tdSql.query("select GREATEST(NULL, NULL, NULL, NULL);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select GREATEST(1, NULL, NULL, NULL);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select GREATEST(id, NULL, 1) from ts_4893.meters order by ts limit 10;") + tdSql.checkRows(10) + tdSql.checkData(0, 0, None) + + tdSql.query("select GREATEST(cast(100 as tinyint), cast(101 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:00:00.101") + + tdSql.query("select GREATEST(cast(101 as tinyint), cast(100 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:00:00.101") + + tdSql.query("select GREATEST(cast(1000 as smallint), cast(1001 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:00:01.001") + + tdSql.query("select GREATEST(cast(1001 as smallint), cast(1000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:00:01.001") + + tdSql.query("select GREATEST(cast(1000000 as int), cast(1000001 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:16:40.001") + + tdSql.query("select GREATEST(cast(1000001 as int), cast(1000000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:16:40.001") + + tdSql.query("select GREATEST(cast(1000000000 as bigint), cast(1000000001 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-12 21:46:40.001") + + tdSql.query("select GREATEST(cast(1000000001 as bigint), cast(1000000000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-12 21:46:40.001") + + tdSql.query("select GREATEST(cast(1725506504000 as timestamp), cast(1725506510000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "2024-09-05 11:21:50") + + tdSql.query("select GREATEST(cast(1725506510000 as timestamp), cast(1725506504000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "2024-09-05 11:21:50") + + tdSql.query("select GREATEST(cast(100 as tinyint), cast(101 as varchar(20)), cast(102 as float));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "102.000000") + + tdSql.query("select GREATEST(cast(100 as varchar(20)), cast(101 as tinyint), cast(102 as float));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "102.000000") + + tdSql.query("select GREATEST(now, 1);") + tdSql.query("select GREATEST(now, 1.0);") + tdSql.query("select GREATEST(now, '1');") + + tdSql.error("select GREATEST(1)") + tdSql.error("select GREATEST(cast('a' as varbinary), cast('b' as varbinary), 'c', 'd');") + tdSql.error("select GREATEST(6, cast('f' as varbinary), cast('b' as varbinary), 'c', 'd');") + + def test_least(self): + self.test_normal_query_new("least") + + tdSql.execute("alter local 'compareAsStrInGreatest' '1';") + + tdSql.query("select LEAST(NULL, NULL, NULL, NULL);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select LEAST(1, NULL, NULL, NULL);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select LEAST(id, NULL, 1) from ts_4893.meters order by ts limit 10;") + tdSql.checkRows(10) + tdSql.checkData(0, 0, None) + + tdSql.query("select LEAST(cast(100 as tinyint), cast(101 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:00:00.100") + + tdSql.query("select LEAST(cast(101 as tinyint), cast(100 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:00:00.100") + + tdSql.query("select LEAST(cast(1000 as smallint), cast(1001 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:00:01.000") + + tdSql.query("select LEAST(cast(1001 as smallint), cast(1000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:00:01.000") + + tdSql.query("select LEAST(cast(1000000 as int), cast(1000001 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:16:40.000") + + tdSql.query("select LEAST(cast(1000001 as int), cast(1000000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-01 08:16:40.000") + + tdSql.query("select LEAST(cast(1000000000 as bigint), cast(1000000001 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-12 21:46:40.000") + + tdSql.query("select LEAST(cast(1000000001 as bigint), cast(1000000000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "1970-01-12 21:46:40.000") + + tdSql.query("select LEAST(cast(1725506504000 as timestamp), cast(1725506510000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "2024-09-05 11:21:44") + + tdSql.query("select LEAST(cast(1725506510000 as timestamp), cast(1725506504000 as timestamp));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "2024-09-05 11:21:44") + + tdSql.query("select LEAST(cast(100 as tinyint), cast(101 as varchar(20)), cast(102 as float));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "100") + + tdSql.query("select LEAST(cast(100 as varchar(20)), cast(101 as tinyint), cast(102 as float));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "100") + + tdSql.query("select LEAST(cast(100 as float), cast(101 as tinyint), cast(102 as varchar(20)));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "100.000000") + + tdSql.query("select LEAST(cast(100 as float), cast(101 as varchar(20)), cast(102 as tinyint));") + tdSql.checkRows(1) + tdSql.checkData(0, 0, "100.000000") + + tdSql.query("select LEAST(now, 1);") + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.checkData(0, 0, "1970-01-01 08:00:00.001") + + tdSql.query("select LEAST(now, 1.0);") + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select LEAST(now, '1');") + tdSql.checkRows(1) + tdSql.checkCols(1) + tdSql.checkData(0, 0, "1") + + tdSql.error("select LEAST(cast('a' as varbinary), cast('b' as varbinary), 'c', 'd');") + tdSql.error("select LEAST(cast('f' as varbinary), cast('b' as varbinary), 'c', 'd');") + + def test_greatest_large_table(self): + tdLog.info("test greatest large table.") + + ts = 1741341251000 + create_table_sql = "CREATE TABLE `large_table` (`ts` TIMESTAMP" + for i in range(1, 1001): + if i % 5 == 1: + create_table_sql += f", `col{i}` INT" + elif i % 5 == 2: + create_table_sql += f", `col{i}` FLOAT" + elif i % 5 == 3: + create_table_sql += f", `col{i}` DOUBLE" + elif i % 5 == 4: + create_table_sql += f", `col{i}` VARCHAR(64)" + else: + create_table_sql += f", `col{i}` NCHAR(50)" + create_table_sql += ");" + tdSql.execute(create_table_sql) + + for j in range(1000): + insert_sql = f"INSERT INTO `large_table` VALUES ({ts +j}" + for i in range(1, 1001): + if i % 5 == 1: + insert_sql += f", {j + i}" + elif i % 5 == 2: + insert_sql += f", {j + i}.1" + elif i % 5 == 3: + insert_sql += f", {j + i}.2" + elif i % 5 == 4: + insert_sql += f", '{j + i}'" + else: + insert_sql += f", '{j + i}'" + insert_sql += ");" + tdSql.execute(insert_sql) + + greatest_query = "SELECT GREATEST(" + for i in range(1, 1001): + greatest_query += f"`col{i}`" + if i < 1000: + greatest_query += ", " + greatest_query += ") FROM `large_table` LIMIT 1;" + tdLog.info(f"greatest_query: {greatest_query}") + tdSql.execute(greatest_query) + + greatest_query = "SELECT " + for i in range(1, 1001): + greatest_query += f"`col{i}` > `col5`" + if i < 1000: + greatest_query += ", " + greatest_query += " FROM `large_table` LIMIT 1;" + tdLog.info(f"greatest_query: {greatest_query}") + tdSql.execute(greatest_query) + def run(self): tdLog.debug(f"start to excute {__file__}") @@ -326,7 +516,10 @@ class TDTestCase(TBase): self.test_degrees() self.test_radians() self.test_rand() - + self.test_greatest() + self.test_least() + self.test_greatest_large_table() + # char function self.test_char_length() self.test_char() diff --git a/tests/army/tools/benchmark/basic/csv-export.py b/tests/army/tools/benchmark/basic/csv-export.py new file mode 100644 index 0000000000..65ffb3e541 --- /dev/null +++ b/tests/army/tools/benchmark/basic/csv-export.py @@ -0,0 +1,237 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +import os +import json +import csv +import datetime + +import frame +import frame.eos +import frame.etool +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + def caseDescription(self): + """ + [TS-5089] taosBenchmark support exporting csv + """ + + + def clear_directory(self, target_dir: str = 'csv'): + try: + if not os.path.exists(target_dir): + return + for entry in os.listdir(target_dir): + entry_path = os.path.join(target_dir, entry) + if os.path.isfile(entry_path) or os.path.islink(entry_path): + os.unlink(entry_path) + else: + shutil.rmtree(entry_path) + + tdLog.debug("clear succ, dir: %s " % (target_dir)) + except OSError as e: + tdLog.exit("clear fail, dir: %s " % (target_dir)) + + + def convert_timestamp(self, ts, ts_format): + dt_object = datetime.datetime.fromtimestamp(ts / 1000) + formatted_time = dt_object.strftime(ts_format) + return formatted_time + + + def calc_time_slice_partitions(self, total_start_ts, total_end_ts, ts_step, ts_format, ts_interval): + interval_days = int(ts_interval[:-1]) + n_days_millis = interval_days * 24 * 60 * 60 * 1000 + + dt_start = datetime.datetime.fromtimestamp(total_start_ts / 1000.0) + formatted_str = dt_start.strftime(ts_format) + s0_dt = datetime.datetime.strptime(formatted_str, ts_format) + s0 = int(s0_dt.timestamp() * 1000) + + partitions = [] + current_s = s0 + + while current_s <= total_end_ts: + current_end = current_s + n_days_millis + start_actual = max(current_s, total_start_ts) + end_actual = min(current_end, total_end_ts) + + if start_actual >= end_actual: + count = 0 + else: + delta = end_actual - start_actual + delta + delta_start = start_actual - total_start_ts + delta_end = end_actual - total_start_ts + if delta % ts_step: + count = delta // ts_step + 1 + else: + count = delta // ts_step + + partitions.append({ + "start_ts": current_s, + "end_ts": current_end, + "start_time": self.convert_timestamp(current_s, ts_format), + "end_time": self.convert_timestamp(current_end, ts_format), + "count": count + }) + + current_s += n_days_millis + + # partitions = [p for p in partitions if p['count'] > 0] + return partitions + + + def check_stb_csv_correct(self, csv_file_name, all_rows, interlace_rows): + # open as csv + tbname_idx = 14 + count = 0 + batch = 0 + name = "" + header = True + with open(csv_file_name) as file: + rows = csv.reader(file) + for row in rows: + if header: + header = False + continue + + # interlace_rows + if name == "": + name = row[tbname_idx] + batch = 1 + else: + if name == row[tbname_idx]: + batch += 1 + else: + # switch to another child table + if batch != interlace_rows: + tdLog.exit(f"interlace_rows invalid. tbName={name} actual={batch} expected={interlace_rows} i={count} csv_file_name={csv_file_name}") + batch = 1 + name = row[tbname_idx] + # count ++ + count += 1 + # batch + if batch != interlace_rows: + tdLog.exit(f"interlace_rows invalid. tbName={name} actual={batch} expected={interlace_rows} i={count} csv_file_name={csv_file_name}") + + # check all rows + if count != all_rows: + tdLog.exit(f"all_rows invalid. actual={count} expected={all_rows} csv_file_name={csv_file_name}") + + tdLog.info(f"Check generate csv file successfully. csv_file_name={csv_file_name} count={count} interlace_rows={batch}") + + + # check correct + def check_stb_correct(self, data, db, stb): + filepath = data["output_dir"] + stbName = stb["name"] + child_count = stb["childtable_to"] - stb["childtable_from"] + insert_rows = stb["insert_rows"] + interlace_rows = stb["interlace_rows"] + csv_file_prefix = stb["csv_file_prefix"] + csv_ts_format = stb.get("csv_ts_format", None) + csv_ts_interval = stb.get("csv_ts_interval", None) + + ts_step = stb["timestamp_step"] + total_start_ts = stb["start_timestamp"] + total_end_ts = total_start_ts + ts_step * insert_rows + + + all_rows = child_count * insert_rows + if interlace_rows > 0: + # interlace + + if not csv_ts_format: + # normal + csv_file_name = f"{filepath}{csv_file_prefix}.csv" + self.check_stb_csv_correct(csv_file_name, all_rows, interlace_rows) + else: + # time slice + partitions = self.calc_time_slice_partitions(total_start_ts, total_end_ts, ts_step, csv_ts_format, csv_ts_interval) + for part in partitions: + csv_file_name = f"{filepath}{csv_file_prefix}_{part['start_time']}_{part['end_time']}.csv" + self.check_stb_csv_correct(csv_file_name, part['count'] * child_count, interlace_rows) + else: + # batch + thread_count = stb["thread_count"] + interlace_rows = insert_rows + if not csv_ts_format: + # normal + for i in range(thread_count): + csv_file_name = f"{filepath}{csv_file_prefix}_{i + 1}.csv" + if i < child_count % thread_count: + self.check_stb_csv_correct(csv_file_name, insert_rows * (child_count // thread_count + 1), interlace_rows) + else: + self.check_stb_csv_correct(csv_file_name, insert_rows * (child_count // thread_count), interlace_rows) + else: + # time slice + for i in range(thread_count): + partitions = self.calc_time_slice_partitions(total_start_ts, total_end_ts, ts_step, csv_ts_format, csv_ts_interval) + for part in partitions: + csv_file_name = f"{filepath}{csv_file_prefix}_{i + 1}_{part['start_time']}_{part['end_time']}.csv" + if i < child_count % thread_count: + slice_rows = part['count'] * (child_count // thread_count + 1) + else: + slice_rows = part['count'] * (child_count // thread_count) + + self.check_stb_csv_correct(csv_file_name, slice_rows, part['count']) + + + # check result + def check_result(self, jsonFile): + # csv + with open(jsonFile) as file: + data = json.load(file) + + # read json + database = data["databases"][0] + stables = database["super_tables"] + + for stable in stables: + # check csv context correct + self.check_stb_correct(data, database, stable) + + + def check_export_csv(self, benchmark, jsonFile, options=""): + # clear + self.clear_directory() + + # exec + cmd = f"{benchmark} {options} -f {jsonFile}" + eos.exe(cmd) + + # check result + self.check_result(jsonFile) + + + def run(self): + # path + benchmark = etool.benchMarkFile() + + # do check interlace normal + json = "tools/benchmark/basic/json/csv-export.json" + self.check_export_csv(benchmark, json) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/army/tools/benchmark/basic/exportCsv.py b/tests/army/tools/benchmark/basic/exportCsv.py deleted file mode 100644 index b8b3828ea6..0000000000 --- a/tests/army/tools/benchmark/basic/exportCsv.py +++ /dev/null @@ -1,110 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- -import os -import json -import csv - -import frame -import frame.etool -from frame.log import * -from frame.cases import * -from frame.sql import * -from frame.caseBase import * -from frame import * - - -class TDTestCase(TBase): - def caseDescription(self): - """ - [TD-11510] taosBenchmark test cases - """ - # check correct - def checkCorrect(self, csvFile, allRows, interlaceRows): - # open as csv - count = 0 - batch = 0 - name = "" - with open(csvFile) as file: - rows = csv.reader(file) - for row in rows: - # interlaceRows - if name == "": - name = row[0] - batch = 1 - else: - if name == row[0]: - batch += 1 - else: - # switch to another child table - if batch != interlaceRows: - tdLog.exit(f"interlaceRows invalid. tbName={name} real={batch} expect={interlaceRows} i={count} csvFile={csvFile}") - batch = 1 - name = row[0] - # count ++ - count += 1 - # batch - if batch != interlaceRows: - tdLog.exit(f"interlaceRows invalid. tbName={name} real={batch} expect={interlaceRows} i={count} csvFile={csvFile}") - - # check all rows - if count != allRows: - tdLog.exit(f"allRows invalid. real={count} expect={allRows} csvFile={csvFile}") - - tdLog.info(f"Check generate csv file successfully. csvFile={csvFile} count={count} interlaceRows={batch}") - - # check result - def checResult(self, jsonFile): - # csv - with open(jsonFile) as file: - data = json.load(file) - - # read json - database = data["databases"][0] - out = data["csvPath"] - dbName = database["dbinfo"]["name"] - stables = database["super_tables"] - for stable in stables: - stbName = stable["name"] - childs = stable["childtable_count"] - insertRows = stable["insert_rows"] - interlaceRows = stable["interlace_rows"] - csvFile = f"{out}{dbName}-{stbName}.csv" - rows = childs * insertRows - if interlaceRows == 0: - interlaceRows = insertRows - # check csv context correct - self.checkCorrect(csvFile, rows, interlaceRows) - - def checkExportCsv(self, benchmark, jsonFile, options=""): - # exec - cmd = f"{benchmark} {options} -f {jsonFile}" - os.system(cmd) - - # check result - self.checResult(jsonFile) - - def run(self): - # path - benchmark = etool.benchMarkFile() - - # do check - json = "tools/benchmark/basic/json/exportCsv.json" - self.checkExportCsv(benchmark, json) - - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/army/tools/benchmark/basic/insertBasic.py b/tests/army/tools/benchmark/basic/insertBasic.py index f1d3b81732..dfc5e3aff1 100644 --- a/tests/army/tools/benchmark/basic/insertBasic.py +++ b/tests/army/tools/benchmark/basic/insertBasic.py @@ -13,6 +13,7 @@ import os import json import frame +import frame.eos import frame.etool from frame.log import * from frame.cases import * @@ -109,7 +110,6 @@ class TDTestCase(TBase): tdLog.info(f" vgroups real={tdSql.getData(0,0)} expect={vgroups}") tdSql.checkData(0, 0, vgroups, True) - # bugs ts def checkVGroups(self, benchmark): # vgroups with command line set @@ -117,12 +117,19 @@ class TDTestCase(TBase): # vgroups with json file self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/insertBasic.json", "", True) + + def checkInsertManyStb(self): + # many stb + self.benchInsert("./tools/benchmark/basic/json/insertManyStb.json") + def run(self): - benchmark = etool.benchMarkFile() + benchmark = frame.etool.benchMarkFile() # vgroups self.checkVGroups(benchmark) + # check many stable + self.checkInsertManyStb() def stop(self): tdSql.close() diff --git a/tests/army/tools/benchmark/basic/json/csv-export.json b/tests/army/tools/benchmark/basic/json/csv-export.json new file mode 100644 index 0000000000..88beab0de1 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/csv-export.json @@ -0,0 +1,172 @@ +{ + "filetype": "csvfile", + "output_dir": "./csv/", + "databases": [ + { + "dbinfo": { + "name": "csvdb", + "precision": "ms" + }, + "super_tables": [ + { + "name": "interlace-normal", + "childtable_count": 1010, + "insert_rows": 1000, + "interlace_rows": 1, + "childtable_prefix": "d", + "timestamp_step": 1000000, + "start_timestamp":1700000000000, + "childtable_from": 1000, + "childtable_to": 1010, + "csv_file_prefix": "data", + "csv_output_header": "yes", + "csv_tbname_alias": "device_id", + "csv_compress_level": "none", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min": 1}, + { "type": "double", "name": "dc", "min":10, "max":10}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic", "fillNull":"false"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi", "min":100, "max":120}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 16} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + }, + { + "name": "interlace-timeslice", + "childtable_count": 1010, + "insert_rows": 1000, + "interlace_rows": 1, + "childtable_prefix": "d", + "timestamp_step": 1000000, + "start_timestamp":1700000000000, + "childtable_from": 1000, + "childtable_to": 1010, + "csv_file_prefix": "data", + "csv_ts_format": "%Y%m%d", + "csv_ts_interval": "1d", + "csv_output_header": "yes", + "csv_tbname_alias": "device_id", + "csv_compress_level": "none", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min": 1}, + { "type": "double", "name": "dc", "min":10, "max":10}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic", "fillNull":"false"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi", "min":100, "max":120}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 16} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + }, + { + "name": "batch-normal", + "childtable_count": 1010, + "insert_rows": 1000, + "interlace_rows": 0, + "thread_count": 8, + "childtable_prefix": "d", + "timestamp_step": 1000000, + "start_timestamp":1700000000000, + "childtable_from": 1000, + "childtable_to": 1010, + "csv_file_prefix": "data", + "csv_output_header": "yes", + "csv_tbname_alias": "device_id", + "csv_compress_level": "none", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min": 1}, + { "type": "double", "name": "dc", "min":10, "max":10}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic", "fillNull":"false"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi", "min":100, "max":120}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 16} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + }, + { + "name": "batch-timeslice", + "childtable_count": 1010, + "insert_rows": 1000, + "interlace_rows": 0, + "thread_count": 8, + "childtable_prefix": "d", + "timestamp_step": 1000000, + "start_timestamp":1700000000000, + "childtable_from": 1000, + "childtable_to": 1010, + "csv_file_prefix": "data", + "csv_ts_format": "%Y%m%d", + "csv_ts_interval": "1d", + "csv_output_header": "yes", + "csv_tbname_alias": "device_id", + "csv_compress_level": "none", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min": 1}, + { "type": "double", "name": "dc", "min":10, "max":10}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic", "fillNull":"false"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi", "min":100, "max":120}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 16} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/tools/benchmark/basic/json/exportCsv.json b/tests/army/tools/benchmark/basic/json/exportCsv.json deleted file mode 100644 index 05a7341eb6..0000000000 --- a/tests/army/tools/benchmark/basic/json/exportCsv.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "filetype": "csvfile", - "csvPath": "./csv/", - "num_of_records_per_req": 10000, - "databases": [ - { - "dbinfo": { - "name": "csvdb" - }, - "super_tables": [ - { - "name": "batchTable", - "childtable_count": 5, - "insert_rows": 100, - "interlace_rows": 0, - "childtable_prefix": "d", - "timestamp_step": 10, - "start_timestamp":1600000000000, - "columns": [ - { "type": "bool", "name": "bc"}, - { "type": "float", "name": "fc", "min": 1}, - { "type": "double", "name": "dc", "min":10, "max":10}, - { "type": "tinyint", "name": "ti"}, - { "type": "smallint", "name": "si"}, - { "type": "int", "name": "ic", "fillNull":"false"}, - { "type": "bigint", "name": "bi"}, - { "type": "utinyint", "name": "uti"}, - { "type": "usmallint", "name": "usi", "min":100, "max":120}, - { "type": "uint", "name": "ui"}, - { "type": "ubigint", "name": "ubi"}, - { "type": "binary", "name": "bin", "len": 16}, - { "type": "nchar", "name": "nch", "len": 16} - ], - "tags": [ - {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, - {"type": "binary", "name": "location", "len": 16, - "values": ["San Francisco", "Los Angles", "San Diego", - "San Jose", "Palo Alto", "Campbell", "Mountain View", - "Sunnyvale", "Santa Clara", "Cupertino"] - } - ] - }, - { - "name": "interlaceTable", - "childtable_count": 5, - "insert_rows": 100, - "interlace_rows": 10, - "childtable_prefix": "d", - "timestamp_step": 1000, - "start_timestamp":1700000000000, - "columns": [ - { "type": "bool", "name": "bc"}, - { "type": "float", "name": "fc", "min":16}, - { "type": "double", "name": "dc", "min":16}, - { "type": "tinyint", "name": "ti"}, - { "type": "smallint", "name": "si"}, - { "type": "int", "name": "ic", "fillNull":"false"}, - { "type": "bigint", "name": "bi"}, - { "type": "utinyint", "name": "uti"}, - { "type": "usmallint", "name": "usi"}, - { "type": "uint", "name": "ui"}, - { "type": "ubigint", "name": "ubi"}, - { "type": "binary", "name": "bin", "len": 32}, - { "type": "nchar", "name": "nch", "len": 64} - ], - "tags": [ - {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, - {"type": "binary", "name": "location", "len": 16, - "values": ["San Francisco", "Los Angles", "San Diego", - "San Jose", "Palo Alto", "Campbell", "Mountain View", - "Sunnyvale", "Santa Clara", "Cupertino"] - } - ] - } - ] - } - ] -} diff --git a/tests/army/tools/benchmark/basic/json/insertManyStb.json b/tests/army/tools/benchmark/basic/json/insertManyStb.json new file mode 100644 index 0000000000..caa418c858 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/insertManyStb.json @@ -0,0 +1,151 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 3000, + "thread_count": 2, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "test1", + "drop": "yes", + "precision": "us", + "vgroups": 1 + }, + "super_tables": [ + { + "name": "meters1", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 1000, + "childtable_prefix": "da", + "insert_mode": "stmt", + "timestamp_step": 15, + "start_timestamp":1700000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters2", + "child_table_exists": "no", + "childtable_count": 3, + "insert_rows": 100, + "childtable_prefix": "db", + "insert_mode": "stmt", + "timestamp_step": 20, + "interlace_rows": 1, + "start_timestamp":1700000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters3", + "child_table_exists": "no", + "childtable_count": 5, + "insert_rows": 100, + "childtable_prefix": "dc", + "insert_mode": "stmt2", + "timestamp_step": 20, + "interlace_rows": 1, + "start_timestamp":1700000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters4", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 70, + "childtable_prefix": "dd", + "insert_mode": "stmt2", + "timestamp_step": 50, + "interlace_rows": 0, + "start_timestamp":1700000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + } + ] + }, + { + "dbinfo": { + "name": "test2", + "drop": "yes", + "precision": "ns", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters1", + "child_table_exists": "no", + "childtable_count": 3, + "insert_rows": 120, + "childtable_prefix": "de", + "insert_mode": "taosc", + "timestamp_step": 15, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters2", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 200, + "childtable_prefix": "df", + "insert_mode": "taosc", + "timestamp_step": 2, + "interlace_rows": 4, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1bfd86df4d..f79c4c291b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -93,7 +93,8 @@ ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_json.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/demo.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/exportCsv.py +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/csv-export.py +# ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/csv-import.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/from-to.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/from-to-continue.py diff --git a/tests/script/tsim/analytics/basic0.sim b/tests/script/tsim/analytics/basic0.sim index 999b2fff37..0d9a29a19b 100644 --- a/tests/script/tsim/analytics/basic0.sim +++ b/tests/script/tsim/analytics/basic0.sim @@ -91,6 +91,14 @@ sql_error select count(*) from ct1 anomaly_window(c6, 'algo=ksigma,k=2'); sql_error select forecast(c6, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 +print ==================== invalid timeout parameter, will reset the parameters. +sql select forecast(c1, 'algo=holtwinters, timeout=6000') from ct1; +sql select forecast(c1, 'algo=holtwinters, timeout=0') from ct1; + +print =========================== valid timeout +sql select forecast(c1, 'algo=holtwinters, timeout=120') from ct1; + + sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1 diff --git a/tests/system-test/0-others/test_show_disk_usage.py b/tests/system-test/0-others/test_show_disk_usage.py index eb5bdf1aa7..1c863d7cdf 100644 --- a/tests/system-test/0-others/test_show_disk_usage.py +++ b/tests/system-test/0-others/test_show_disk_usage.py @@ -102,7 +102,8 @@ class TDTestCase: elif "Compress_radio=" in item[0]: value = item[0].split("=")[1].split(" ")[0].replace("[", "").replace("]", "") if value != 'NULL': - compress_radio = float(value) + tValue = value[0:len(value) - 1] + compress_radio = float(tValue) #tdLog.debug("compress_occupied: %s" % compress_radio) return disk_occupied, compress_radio diff --git a/tests/system-test/8-stream/checkpoint_info2.py b/tests/system-test/8-stream/checkpoint_info2.py index 3dc57477f7..f4c8da8c9d 100644 --- a/tests/system-test/8-stream/checkpoint_info2.py +++ b/tests/system-test/8-stream/checkpoint_info2.py @@ -22,6 +22,8 @@ from util.cluster import * # should be used by -N option class TDTestCase: updatecfgDict = {'checkpointInterval': 60 , + 'vdebugflag':143, + 'ddebugflag':143 } def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 110a644e90..1ee2bc4ce6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -19,7 +19,8 @@ IF(TD_WEBSOCKET) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND RUSTFLAGS=-Ctarget-feature=-crt-static cargo build --release --locked -p taos-ws-sys --features rustls + COMMAND cargo update + COMMAND RUSTFLAGS=-Ctarget-feature=-crt-static cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include @@ -37,7 +38,8 @@ IF(TD_WEBSOCKET) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND cargo build --release --locked -p taos-ws-sys --features rustls + COMMAND cargo update + COMMAND cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/taosws.dll ${CMAKE_BINARY_DIR}/build/lib COMMAND cp target/release/taosws.dll.lib ${CMAKE_BINARY_DIR}/build/lib/taosws.lib @@ -56,7 +58,8 @@ IF(TD_WEBSOCKET) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND cargo build --release --locked -p taos-ws-sys --features rustls + COMMAND cargo update + COMMAND cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index a3e542a768..99b1c204a2 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -668,28 +668,22 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t printf("%*" PRIu64, width, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: + width = width >= LENGTH ? LENGTH - 1 : width; if (tsEnableScience) { printf("%*.7e", width, GET_FLOAT_VAL(val)); } else { - n = snprintf(buf, LENGTH, "%*.*g", width, FLT_DIG, GET_FLOAT_VAL(val)); - if (n > SHELL_FLOAT_WIDTH) { - printf("%*.7e", width, GET_FLOAT_VAL(val)); - } else { - printf("%s", buf); - } + snprintf(buf, LENGTH, "%*.*g", width, FLT_DIG, GET_FLOAT_VAL(val)); + printf("%s", buf); } break; case TSDB_DATA_TYPE_DOUBLE: + width = width >= LENGTH ? LENGTH - 1 : width; if (tsEnableScience) { snprintf(buf, LENGTH, "%*.15e", width, GET_DOUBLE_VAL(val)); printf("%s", buf); } else { - n = snprintf(buf, LENGTH, "%*.*g", width, DBL_DIG, GET_DOUBLE_VAL(val)); - if (n > SHELL_DOUBLE_WIDTH) { - printf("%*.15e", width, GET_DOUBLE_VAL(val)); - } else { - printf("%*s", width, buf); - } + snprintf(buf, LENGTH, "%*.*g", width, DBL_DIG, GET_DOUBLE_VAL(val)); + printf("%*s", width, buf); } break; case TSDB_DATA_TYPE_VARBINARY: { diff --git a/tools/taos-tools/README-CN.md b/tools/taos-tools/README-CN.md index 3def035f68..da14e81cd1 100644 --- a/tools/taos-tools/README-CN.md +++ b/tools/taos-tools/README-CN.md @@ -18,7 +18,7 @@ taosdump 是用于备份 TDengine 数据到本地目录和从本地目录恢复 #### 对于 Ubuntu/Debian 系统 ```shell -sudo apt install libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libssl-dev +sudo apt install libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g zlib1g-dev pkg-config libssl-dev ``` #### 对于 CentOS 7/RHEL 系统 diff --git a/tools/taos-tools/example/csv-export.json b/tools/taos-tools/example/csv-export.json new file mode 100644 index 0000000000..7fa3e96f2f --- /dev/null +++ b/tools/taos-tools/example/csv-export.json @@ -0,0 +1,54 @@ +{ + "filetype": "csvfile", + "output_path": "./csv/", + "databases": [ + { + "dbinfo": { + "name": "csvdb", + "precision": "ms" + }, + "super_tables": [ + { + "name": "table", + "childtable_count": 1010, + "insert_rows": 1000, + "interlace_rows": 1, + "childtable_prefix": "d", + "timestamp_step": 1000000, + "start_timestamp": "2020-10-01 00:00:00.000", + "childtable_from": 1000, + "childtable_to": 1010, + "csv_file_prefix": "data", + "csv_ts_format": "%Y%m%d", + "csv_ts_interval": "1d", + "csv_output_header": "true", + "csv_tbname_alias": "device_id", + "csv_compress_level": "none", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "min": 1}, + { "type": "double", "name": "dc", "min":10, "max":10}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si"}, + { "type": "int", "name": "ic", "fillNull":"false"}, + { "type": "bigint", "name": "bi"}, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi", "min":100, "max":120}, + { "type": "uint", "name": "ui"}, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 16} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"type": "binary", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tools/taos-tools/inc/bench.h b/tools/taos-tools/inc/bench.h index 97c82ac2fa..c413d953b7 100644 --- a/tools/taos-tools/inc/bench.h +++ b/tools/taos-tools/inc/bench.h @@ -20,6 +20,9 @@ #define CURL_STATICLIB #define ALLOW_FORBID_FUNC +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + #ifdef LINUX #ifndef _ALPINE @@ -476,6 +479,13 @@ typedef struct SChildTable_S { int32_t pkCnt; } SChildTable; +typedef enum { + CSV_COMPRESS_NONE = 0, + CSV_COMPRESS_FAST = 1, + CSV_COMPRESS_BALANCE = 6, + CSV_COMPRESS_BEST = 9 +} CsvCompressionLevel; + #define PRIMARY_KEY "PRIMARY KEY" typedef struct SSuperTable_S { char *stbName; @@ -578,6 +588,15 @@ typedef struct SSuperTable_S { // execute sqls after create super table char **sqls; + + char* csv_file_prefix; + char* csv_ts_format; + char* csv_ts_interval; + char* csv_tbname_alias; + long csv_ts_intv_secs; + bool csv_output_header; + CsvCompressionLevel csv_compress_level; + } SSuperTable; typedef struct SDbCfg_S { @@ -776,9 +795,11 @@ typedef struct SArguments_S { bool mistMode; bool escape_character; bool pre_load_tb_meta; - char csvPath[MAX_FILE_NAME_LEN]; - bool bind_vgroup; + + char* output_path; + char output_path_buf[MAX_PATH_LEN]; + } SArguments; typedef struct SBenchConn { diff --git a/tools/taos-tools/inc/benchCsv.h b/tools/taos-tools/inc/benchCsv.h index 25d0c55eba..6bf531cf14 100644 --- a/tools/taos-tools/inc/benchCsv.h +++ b/tools/taos-tools/inc/benchCsv.h @@ -16,21 +16,82 @@ #ifndef INC_BENCHCSV_H_ #define INC_BENCHCSV_H_ -#include +#include + +#include "bench.h" + + +typedef enum { + CSV_NAMING_I_SINGLE, + CSV_NAMING_I_TIME_SLICE, + CSV_NAMING_B_THREAD, + CSV_NAMING_B_THREAD_TIME_SLICE +} CsvNamingType; + +typedef enum { + CSV_ERR_OK = 0, + CSV_ERR_OPEN_FAILED, + CSV_ERR_WRITE_FAILED +} CsvIoError; + +typedef struct { + const char* filename; + CsvCompressionLevel compress_level; + CsvIoError result; + union { + gzFile gf; + FILE* fp; + } handle; +} CsvFileHandle; + +typedef struct { + char* buf; + int length; +} CsvRowTagsBuf; + +typedef struct { + char* buf; + int buf_size; + int length; +} CsvRowColsBuf; + +typedef struct { + CsvNamingType naming_type; + size_t total_threads; + char mode[MIDDLE_BUFF_LEN]; + char thread_formatter[SMALL_BUFF_LEN]; + char csv_header[LARGE_BUFF_LEN]; + int csv_header_length; + SDataBase* db; + SSuperTable* stb; + int64_t start_ts; + int64_t end_ts; + int64_t ts_step; + int64_t interlace_step; +} CsvWriteMeta; + +typedef struct { + uint64_t ctb_start_idx; + uint64_t ctb_end_idx; + uint64_t ctb_count; + uint64_t total_rows; + time_t start_secs; + time_t end_secs; + int64_t start_ts; + int64_t end_ts; + size_t thread_id; + bool output_header; + int tags_buf_size; + CsvRowTagsBuf* tags_buf_array; + CsvRowColsBuf* cols_buf; +} CsvThreadMeta; + +typedef struct { + CsvWriteMeta* write_meta; + CsvThreadMeta thread_meta; +} CsvThreadArgs; + int csvTestProcess(); -int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir); - -char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k); - -char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k); - -int32_t genRowByField(char* buf, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k); - -void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir); - -int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain); -int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain); - #endif // INC_BENCHCSV_H_ diff --git a/tools/taos-tools/inc/benchLog.h b/tools/taos-tools/inc/benchLog.h index 426112bcd8..ab74aaff75 100644 --- a/tools/taos-tools/inc/benchLog.h +++ b/tools/taos-tools/inc/benchLog.h @@ -16,6 +16,9 @@ #ifndef INC_BENCHLOG_H_ #define INC_BENCHLOG_H_ +#include +#include + // // suport thread safe log module // @@ -53,7 +56,7 @@ void exitLog(); (int32_t)timeSecs.tv_usec); \ fprintf(stdout, "DEBG: "); \ fprintf(stdout, "%s(%d) ", __FILE__, __LINE__); \ - fprintf(stdout, "" fmt, __VA_ARGS__); \ + fprintf(stdout, "" fmt, ##__VA_ARGS__); \ unlockLog(LOG_STDOUT); \ } \ } while (0) @@ -74,7 +77,7 @@ void exitLog(); (int32_t)timeSecs.tv_usec); \ fprintf(stdout, "DEBG: "); \ fprintf(stdout, "%s(%d) ", __FILE__, __LINE__); \ - fprintf(stdout, "" fmt, __VA_ARGS__); \ + fprintf(stdout, "" fmt, ##__VA_ARGS__); \ unlockLog(LOG_STDOUT); \ } \ } while (0) @@ -94,7 +97,7 @@ void exitLog(); do { \ if (g_arguments->debug_print) { \ lockLog(LOG_STDOUT); \ - fprintf(stdout, "" fmt, __VA_ARGS__); \ + fprintf(stdout, "" fmt, ##__VA_ARGS__); \ unlockLog(LOG_STDOUT); \ } \ } while (0) @@ -102,14 +105,14 @@ void exitLog(); #define infoPrintNoTimestamp(fmt, ...) \ do { \ lockLog(LOG_STDOUT); \ - fprintf(stdout, "" fmt, __VA_ARGS__); \ + fprintf(stdout, "" fmt, ##__VA_ARGS__); \ unlockLog(LOG_STDOUT); \ } while (0) #define infoPrintNoTimestampToFile(fmt, ...) \ do { \ lockLog(LOG_RESULT); \ - fprintf(g_arguments->fpOfInsertResult, "" fmt, __VA_ARGS__); \ + fprintf(g_arguments->fpOfInsertResult, "" fmt, ##__VA_ARGS__); \ unlockLog(LOG_RESULT); \ } while (0) @@ -126,7 +129,7 @@ void exitLog(); ptm->tm_mon + 1, \ ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, \ (int32_t)timeSecs.tv_usec); \ - fprintf(stdout, "INFO: " fmt, __VA_ARGS__); \ + fprintf(stdout, "INFO: " fmt, ##__VA_ARGS__); \ unlockLog(LOG_STDOUT); \ } while (0) @@ -142,7 +145,7 @@ void exitLog(); fprintf(g_arguments->fpOfInsertResult,"[%02d/%02d %02d:%02d:%02d.%06d] ", ptm->tm_mon + 1, \ ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, \ (int32_t)timeSecs.tv_usec); \ - fprintf(g_arguments->fpOfInsertResult, "INFO: " fmt, __VA_ARGS__);\ + fprintf(g_arguments->fpOfInsertResult, "INFO: " fmt, ##__VA_ARGS__);\ unlockLog(LOG_RESULT); \ } while (0) @@ -160,7 +163,7 @@ void exitLog(); ptm->tm_mon + 1, \ ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, \ (int32_t)timeSecs.tv_usec); \ - fprintf(stderr, "PERF: " fmt, __VA_ARGS__); \ + fprintf(stderr, "PERF: " fmt, ##__VA_ARGS__); \ unlockLog(LOG_STDERR); \ if (g_arguments->fpOfInsertResult && !g_arguments->terminate) { \ lockLog(LOG_RESULT); \ @@ -172,7 +175,7 @@ void exitLog(); (int32_t)timeSecs.tv_usec); \ fprintf(g_arguments->fpOfInsertResult, "PERF: "); \ fprintf(g_arguments->fpOfInsertResult, \ - "" fmt, __VA_ARGS__); \ + "" fmt, ##__VA_ARGS__); \ unlockLog(LOG_RESULT); \ } \ } \ @@ -196,7 +199,7 @@ void exitLog(); if (g_arguments->debug_print) { \ fprintf(stderr, "%s(%d) ", __FILE__, __LINE__); \ } \ - fprintf(stderr, "" fmt, __VA_ARGS__); \ + fprintf(stderr, "" fmt, ##__VA_ARGS__); \ fprintf(stderr, "\033[0m"); \ unlockLog(LOG_STDERR); \ if (g_arguments->fpOfInsertResult && !g_arguments->terminate) { \ @@ -206,7 +209,7 @@ void exitLog(); ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, \ (int32_t)timeSecs.tv_usec); \ fprintf(g_arguments->fpOfInsertResult, "ERROR: "); \ - fprintf(g_arguments->fpOfInsertResult, "" fmt, __VA_ARGS__); \ + fprintf(g_arguments->fpOfInsertResult, "" fmt, ##__VA_ARGS__); \ unlockLog(LOG_RESULT); \ } \ } while (0) @@ -229,7 +232,7 @@ void exitLog(); if (g_arguments->debug_print) { \ fprintf(stderr, "%s(%d) ", __FILE__, __LINE__); \ } \ - fprintf(stderr, "" fmt, __VA_ARGS__); \ + fprintf(stderr, "" fmt, ##__VA_ARGS__); \ fprintf(stderr, "\033[0m"); \ unlockLog(LOG_STDERR); \ if (g_arguments->fpOfInsertResult && !g_arguments->terminate) { \ @@ -239,7 +242,7 @@ void exitLog(); ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, \ (int32_t)timeSecs.tv_usec); \ fprintf(g_arguments->fpOfInsertResult, "WARN: "); \ - fprintf(g_arguments->fpOfInsertResult, "" fmt, __VA_ARGS__); \ + fprintf(g_arguments->fpOfInsertResult, "" fmt, ##__VA_ARGS__); \ unlockLog(LOG_RESULT); \ } \ } while (0) @@ -262,7 +265,7 @@ void exitLog(); if (g_arguments->debug_print) { \ fprintf(stderr, "%s(%d) ", __FILE__, __LINE__); \ } \ - fprintf(stderr, "" fmt, __VA_ARGS__); \ + fprintf(stderr, "" fmt, ##__VA_ARGS__); \ fprintf(stderr, "\033[0m"); \ unlockLog(LOG_STDERR); \ if (g_arguments->fpOfInsertResult && !g_arguments->terminate) { \ @@ -272,7 +275,7 @@ void exitLog(); ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, \ (int32_t)timeSecs.tv_usec); \ fprintf(g_arguments->fpOfInsertResult, "SUCC: "); \ - fprintf(g_arguments->fpOfInsertResult, "" fmt, __VA_ARGS__); \ + fprintf(g_arguments->fpOfInsertResult, "" fmt, ##__VA_ARGS__); \ unlockLog(LOG_RESULT); \ } \ } while (0) diff --git a/tools/taos-tools/src/CMakeLists.txt b/tools/taos-tools/src/CMakeLists.txt index 1f0899db5c..320fb1f413 100644 --- a/tools/taos-tools/src/CMakeLists.txt +++ b/tools/taos-tools/src/CMakeLists.txt @@ -316,6 +316,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin ENDIF () ENDIF () + + TARGET_LINK_LIBRARIES(taosBenchmark z) + ELSE () ADD_DEFINITIONS(-DWINDOWS) SET(CMAKE_C_STANDARD 11) @@ -331,6 +334,7 @@ ELSE () ADD_DEPENDENCIES(taosdump deps-snappy) ADD_DEPENDENCIES(taosdump deps-libargp) ADD_DEPENDENCIES(taosdump apache-avro) + ADD_DEPENDENCIES(taosBenchmark tools-zlib) IF (${WEBSOCKET}) INCLUDE_DIRECTORIES(/usr/local/include/) @@ -362,4 +366,8 @@ ELSE () ENDIF () TARGET_LINK_LIBRARIES(taosBenchmark taos msvcregex pthread toolscJson ${WEBSOCKET_LINK_FLAGS}) + + TARGET_LINK_LIBRARIES(taosBenchmark zlibstatic) + ENDIF () + diff --git a/tools/taos-tools/src/benchCsv.c b/tools/taos-tools/src/benchCsv.c index 8186438643..f8c43dbb97 100644 --- a/tools/taos-tools/src/benchCsv.c +++ b/tools/taos-tools/src/benchCsv.c @@ -10,293 +10,1373 @@ * FITNESS FOR A PARTICULAR PURPOSE. */ -#include -#include "benchLog.h" -#include -#include +#include +#include +#include +#include +#include +#include "benchLog.h" +#include "benchData.h" +#include "benchDataMix.h" +#include "benchCsv.h" // // main etry // #define SHOW_CNT 100000 +#define GEN_ROW_FIELDS_TAG 0 +#define GEN_ROW_FIELDS_COL 1 -static void *csvWriteThread(void *param) { - // write thread - for (int i = 0; i < g_arguments->databases->size; i++) { - // database - SDataBase * db = benchArrayGet(g_arguments->databases, i); - for (int j=0; j < db->superTbls->size; j++) { - // stb - SSuperTable* stb = benchArrayGet(db->superTbls, j); - // gen csv - int ret = genWithSTable(db, stb, g_arguments->csvPath); - if(ret != 0) { - errorPrint("failed generate to csv. db=%s stb=%s error code=%d \n", db->dbName, stb->stbName, ret); - return NULL; - } + + +static int csvValidateParamTsFormat(const char* csv_ts_format) { + if (!csv_ts_format) return 0; + + struct tm test_tm = { + .tm_year = 70, + .tm_mon = 0, + .tm_mday = 1, + .tm_hour = 0, + .tm_min = 0, + .tm_sec = 0, + .tm_isdst = -1 + }; + mktime(&test_tm); + + char buffer[1024]; + size_t len = strftime(buffer, sizeof(buffer), csv_ts_format, &test_tm); + if (len == 0) { + return -1; + } + +#ifdef _WIN32 + const char* invalid_chars = "/\\:*?\"<>|"; +#else + const char* invalid_chars = "/\\?\"<>|"; +#endif + if (strpbrk(buffer, invalid_chars) != NULL) { + return -1; + } + + int has_Y = 0, has_m = 0, has_d = 0; + const char* p = csv_ts_format; + while (*p) { + if (*p == '%') { + p++; + switch (*p) { + case 'Y': has_Y = 1; break; + case 'm': has_m = 1; break; + case 'd': has_d = 1; break; + } + } + p++; + } + + if (has_Y == 0 || has_m == 0 || has_d == 0) { + return -1; + } + + return 0; +} + + +static long csvValidateParamTsInterval(const char* csv_ts_interval) { + if (!csv_ts_interval || *csv_ts_interval == '\0') return -1; + + char* endptr; + errno = 0; + const long num = strtol(csv_ts_interval, &endptr, 10); + + if (errno == ERANGE || + endptr == csv_ts_interval || + num <= 0) { + return -1; + } + + if (*endptr == '\0' || + *(endptr + 1) != '\0') { + return -1; + } + + switch (tolower(*endptr)) { + case 's': return num; + case 'm': return num * 60; + case 'h': return num * 60 * 60; + case 'd': return num * 60 * 60 * 24; + default : return -1; + } +} + + +static int csvParseParameter() { + // csv_output_path + size_t len = strlen(g_arguments->output_path); + if (len == 0) { + errorPrint("Failed to generate csv files, the specified output path is empty. Please provide a valid path.\n"); + return -1; + } + if (g_arguments->output_path[len - 1] != '/') { + int n = snprintf(g_arguments->output_path_buf, sizeof(g_arguments->output_path_buf), "%s/", g_arguments->output_path); + if (n < 0 || n >= sizeof(g_arguments->output_path_buf)) { + errorPrint("Failed to generate csv files, path buffer overflow risk when appending '/'. path: %s.\n", + g_arguments->output_path); + return -1; + } + g_arguments->output_path = g_arguments->output_path_buf; + } + + return 0; +} + + +static int csvParseStbParameter(SSuperTable* stb) { + // csv_ts_format + if (stb->csv_ts_format) { + if (csvValidateParamTsFormat(stb->csv_ts_format) != 0) { + errorPrint("Failed to generate csv files, the parameter `csv_ts_format` is invalid. csv_ts_format: %s.\n", + stb->csv_ts_format); + return -1; } } + + // csv_ts_interval + long csv_ts_intv_secs = csvValidateParamTsInterval(stb->csv_ts_interval); + if (csv_ts_intv_secs <= 0) { + errorPrint("Failed to generate csv files, the parameter `csv_ts_interval` is invalid. csv_ts_interval: %s.\n", + stb->csv_ts_interval); + return -1; + } + stb->csv_ts_intv_secs = csv_ts_intv_secs; + + return 0; +} + + +static time_t csvAlignTimestamp(time_t seconds, const char* ts_format) { + struct tm aligned_tm; +#ifdef _WIN32 + localtime_s(&aligned_tm, &seconds); +#else + localtime_r(&seconds, &aligned_tm); +#endif + + int has_Y = 0, has_m = 0, has_d = 0, has_H = 0, has_M = 0, has_S = 0; + const char* p = ts_format; + while (*p) { + if (*p == '%') { + p++; + switch (*p) { + case 'Y': has_Y = 1; break; + case 'm': has_m = 1; break; + case 'd': has_d = 1; break; + case 'H': has_H = 1; break; + case 'M': has_M = 1; break; + case 'S': has_S = 1; break; + } + } + p++; + } + + if (!has_S) aligned_tm.tm_sec = 0; + if (!has_M) aligned_tm.tm_min = 0; + if (!has_H) aligned_tm.tm_hour = 0; + if (!has_d) aligned_tm.tm_mday = 1; + if (!has_m) aligned_tm.tm_mon = 0; + if (!has_Y) aligned_tm.tm_year = 0; + + return mktime(&aligned_tm); +} + + +static time_t csvGetStartSeconds(int precision, int64_t start_ts, const char* csv_ts_format) { + time_t start_seconds = 0; + + if (precision == TSDB_TIME_PRECISION_MICRO) { + start_seconds = start_ts / 1000000L; + } else if (precision == TSDB_TIME_PRECISION_NANO) { + start_seconds = start_ts / 1000000000L; + } else { + start_seconds = start_ts / 1000L; + } + return csvAlignTimestamp(start_seconds, csv_ts_format); +} + + +static void csvConvertTime2String(time_t time_value, char* ts_format, char* time_buf, size_t buf_size) { + struct tm tm_result; + char* old_locale = setlocale(LC_TIME, "C"); +#ifdef _WIN32 + localtime_s(&tm_result, &time_value); +#else + localtime_r(&time_value, &tm_result); +#endif + strftime(time_buf, buf_size, ts_format, &tm_result); + if (old_locale) { + setlocale(LC_TIME, old_locale); + } + return; +} + + +static CsvNamingType csvGetFileNamingType(SSuperTable* stb) { + if (stb->interlaceRows > 0) { + if (stb->csv_ts_format) { + return CSV_NAMING_I_TIME_SLICE; + } else { + return CSV_NAMING_I_SINGLE; + } + } else { + if (stb->csv_ts_format) { + return CSV_NAMING_B_THREAD_TIME_SLICE; + } else { + return CSV_NAMING_B_THREAD; + } + } +} + + +static time_t csvCalcTimestampFromSeconds(int precision, time_t secs) { + time_t ts = 0; + + if (precision == TSDB_TIME_PRECISION_MICRO) { + ts = secs * 1000000L; + } else if (precision == TSDB_TIME_PRECISION_NANO) { + ts = secs * 1000000000L; + } else { + ts = secs * 1000L; + } + return ts; +} + + +static void csvCalcTimestampStep(CsvWriteMeta* write_meta) { + write_meta->ts_step = csvCalcTimestampFromSeconds(write_meta->db->precision, write_meta->stb->csv_ts_intv_secs); + return; +} + + +static void csvCalcSliceTimestamp(CsvWriteMeta* write_meta, CsvThreadMeta* thread_meta) { + thread_meta->start_ts = csvCalcTimestampFromSeconds(write_meta->db->precision, thread_meta->start_secs); + thread_meta->end_ts = csvCalcTimestampFromSeconds(write_meta->db->precision, thread_meta->end_secs); + return; +} + + +static void csvCalcCtbRange(CsvThreadMeta* thread_meta, size_t total_threads, int64_t ctb_offset, int64_t ctb_count) { + uint64_t ctb_start_idx = 0; + uint64_t ctb_end_idx = 0; + size_t tid_idx = thread_meta->thread_id - 1; + size_t base = ctb_count / total_threads; + size_t remainder = ctb_count % total_threads; + + if (tid_idx < remainder) { + ctb_start_idx = ctb_offset + tid_idx * (base + 1); + ctb_end_idx = ctb_start_idx + (base + 1); + } else { + ctb_start_idx = ctb_offset + remainder * (base + 1) + (tid_idx - remainder) * base; + ctb_end_idx = ctb_start_idx + base; + } + + if (ctb_end_idx > ctb_offset + ctb_count) { + ctb_end_idx = ctb_offset + ctb_count; + } + + thread_meta->ctb_start_idx = ctb_start_idx; + thread_meta->ctb_end_idx = ctb_end_idx; + thread_meta->ctb_count = ctb_end_idx - ctb_start_idx; + return; +} + + +static void csvGenThreadFormatter(CsvWriteMeta* write_meta) { + int digits = 0; + + if (write_meta->total_threads == 0) { + digits = 1; + } else { + for (int n = write_meta->total_threads; n > 0; n /= 10) { + digits++; + } + } + + if (digits <= 1) { + (void)snprintf(write_meta->thread_formatter, sizeof(write_meta->thread_formatter), "%%d"); + } else { + (void)snprintf(write_meta->thread_formatter, sizeof(write_meta->thread_formatter), "%%0%dd", digits); + } + return; +} + + +static int csvGenCsvHeader(CsvWriteMeta* write_meta) { + SSuperTable* stb = write_meta->stb; + char* buf = write_meta->csv_header; + int pos = 0; + int size = sizeof(write_meta->csv_header); + + if (!write_meta->stb->csv_output_header) { + return 0; + } + + // ts + pos += snprintf(buf + pos, size - pos, "ts"); + + // columns + for (size_t i = 0; i < stb->cols->size; ++i) { + Field* col = benchArrayGet(stb->cols, i); + pos += snprintf(buf + pos, size - pos, ",%s", col->name); + } + + // tbname + pos += snprintf(buf + pos, size - pos, ",%s", write_meta->stb->csv_tbname_alias); + + // tags + for (size_t i = 0; i < stb->tags->size; ++i) { + Field* tag = benchArrayGet(stb->tags, i); + pos += snprintf(buf + pos, size - pos, ",%s", tag->name); + } + + // line break + pos += snprintf(buf + pos, size - pos, "\n"); + + write_meta->csv_header_length = (pos > 0 && pos < size) ? pos : 0; + return (pos > 0 && pos < size) ? 0 : -1; +} + + +int csvGenCreateDbSql(SDataBase* db, char* buf, int size) { + int pos = 0; + + pos += snprintf(buf + pos, size - pos, "CREATE DATABASE IF NOT EXISTS "); + if (pos <= 0 || pos >= size) return -1; + + pos += snprintf(buf + pos, size - pos, g_arguments->escape_character ? "`%s`" : "%s", db->dbName); + if (pos <= 0 || pos >= size) return -1; + + if (-1 != g_arguments->inputted_vgroups) { + pos += snprintf(buf + pos, size - pos, " VGROUPS %d", g_arguments->inputted_vgroups); + if (pos <= 0 || pos >= size) return -1; + } + + if (db->cfgs) { + for (size_t i = 0; i < db->cfgs->size; ++i) { + SDbCfg* cfg = benchArrayGet(db->cfgs, i); + if (cfg->valuestring) { + pos += snprintf(buf + pos, size - pos, " %s %s", cfg->name, cfg->valuestring); + } else { + pos += snprintf(buf + pos, size - pos, " %s %d", cfg->name, cfg->valueint); + } + if (pos <= 0 || pos >= size) return -1; + } + } + + switch (db->precision) { + case TSDB_TIME_PRECISION_MILLI: + pos += snprintf(buf + pos, size - pos, " PRECISION 'ms';\n"); + break; + case TSDB_TIME_PRECISION_MICRO: + pos += snprintf(buf + pos, size - pos, " PRECISION 'us';\n"); + break; + case TSDB_TIME_PRECISION_NANO: + pos += snprintf(buf + pos, size - pos, " PRECISION 'ns';\n"); + break; + } + + return (pos > 0 && pos < size) ? pos : -1; +} + + +static int csvExportCreateDbSql(CsvWriteMeta* write_meta, FILE* fp) { + char buf[LARGE_BUFF_LEN] = {0}; + int ret = 0; + int length = 0; + + length = csvGenCreateDbSql(write_meta->db, buf, sizeof(buf)); + if (length < 0) { + errorPrint("Failed to generate create db sql, maybe buffer[%zu] not enough.\n", sizeof(buf)); + return -1; + } + + ret = fwrite(buf, 1, length, fp); + if (ret != length) { + errorPrint("Failed to write create db sql: %s. expected written %d but %d.\n", + buf, length, ret); + if (ferror(fp)) { + perror("error"); + } + return -1; + } + + return 0; +} + + +int csvGenCreateStbSql(SDataBase* db, SSuperTable* stb, char* buf, int size) { + int pos = 0; + + pos += snprintf(buf + pos, size - pos, "CREATE TABLE IF NOT EXISTS "); + if (pos <= 0 || pos >= size) return -1; + + pos += snprintf(buf + pos, size - pos, g_arguments->escape_character ? "`%s`.`%s`" : "%s.%s", db->dbName, stb->stbName); + if (pos <= 0 || pos >= size) return -1; + + pos += snprintf(buf + pos, size - pos, " (ts TIMESTAMP"); + if (pos <= 0 || pos >= size) return -1; + + + // columns + for (size_t i = 0; i < stb->cols->size; ++i) { + Field* col = benchArrayGet(stb->cols, i); + + if (col->type == TSDB_DATA_TYPE_BINARY + || col->type == TSDB_DATA_TYPE_NCHAR + || col->type == TSDB_DATA_TYPE_VARBINARY + || col->type == TSDB_DATA_TYPE_GEOMETRY) { + + if (col->type == TSDB_DATA_TYPE_GEOMETRY && col->length < 21) { + errorPrint("%s() LN%d, geometry filed len must be greater than 21 on %zu.\n", __func__, __LINE__, i); + return -1; + } + + pos += snprintf(buf + pos, size - pos, ",%s %s(%d)", col->name, convertDatatypeToString(col->type), col->length); + } else { + pos += snprintf(buf + pos, size - pos, ",%s %s", col->name, convertDatatypeToString(col->type)); + } + if (pos <= 0 || pos >= size) return -1; + + // primary key + if (stb->primary_key && i == 0) { + pos += snprintf(buf + pos, size - pos, " %s", PRIMARY_KEY); + if (pos <= 0 || pos >= size) return -1; + } + + // compress key + if (strlen(col->encode) > 0) { + pos += snprintf(buf + pos, size - pos, " encode '%s'", col->encode); + if (pos <= 0 || pos >= size) return -1; + } + if (strlen(col->compress) > 0) { + pos += snprintf(buf + pos, size - pos, " compress '%s'", col->compress); + if (pos <= 0 || pos >= size) return -1; + } + if (strlen(col->level) > 0) { + pos += snprintf(buf + pos, size - pos, " level '%s'", col->level); + if (pos <= 0 || pos >= size) return -1; + } + } + + pos += snprintf(buf + pos, size - pos, ") TAGS ("); + if (pos <= 0 || pos >= size) return -1; + + + // tags + for (size_t i = 0; i < stb->tags->size; ++i) { + Field* tag = benchArrayGet(stb->tags, i); + + if (i > 0) { + pos += snprintf(buf + pos, size - pos, ","); + if (pos <= 0 || pos >= size) return -1; + } + + if (tag->type == TSDB_DATA_TYPE_BINARY + || tag->type == TSDB_DATA_TYPE_NCHAR + || tag->type == TSDB_DATA_TYPE_VARBINARY + || tag->type == TSDB_DATA_TYPE_GEOMETRY) { + + if (tag->type == TSDB_DATA_TYPE_GEOMETRY && tag->length < 21) { + errorPrint("%s() LN%d, geometry filed len must be greater than 21 on %zu.\n", __func__, __LINE__, i); + return -1; + } + + pos += snprintf(buf + pos, size - pos, "%s %s(%d)", tag->name, convertDatatypeToString(tag->type), tag->length); + + } else { + pos += snprintf(buf + pos, size - pos, "%s %s", tag->name, convertDatatypeToString(tag->type)); + } + if (pos <= 0 || pos >= size) return -1; + } + + pos += snprintf(buf + pos, size - pos, ")"); + if (pos <= 0 || pos >= size) return -1; + + + // comment + if (stb->comment != NULL) { + pos += snprintf(buf + pos, size - pos," COMMENT '%s'", stb->comment); + if (pos <= 0 || pos >= size) return -1; + } + + // delay + if (stb->delay >= 0) { + pos += snprintf(buf + pos, size - pos, " DELAY %d", stb->delay); + if (pos <= 0 || pos >= size) return -1; + } + + // file factor + if (stb->file_factor >= 0) { + pos += snprintf(buf + pos, size - pos, " FILE_FACTOR %f", stb->file_factor / 100.0); + if (pos <= 0 || pos >= size) return -1; + } + + // rollup + if (stb->rollup != NULL) { + pos += snprintf(buf + pos, size - pos, " ROLLUP(%s)", stb->rollup); + if (pos <= 0 || pos >= size) return -1; + } + + // max delay + if (stb->max_delay != NULL) { + pos += snprintf(buf + pos, size - pos, " MAX_DELAY %s", stb->max_delay); + if (pos <= 0 || pos >= size) return -1; + } + + // watermark + if (stb->watermark != NULL) { + pos += snprintf(buf + pos, size - pos, " WATERMARK %s", stb->watermark); + if (pos <= 0 || pos >= size) return -1; + } + + bool first_sma = true; + for (size_t i = 0; i < stb->cols->size; ++i) { + Field* col = benchArrayGet(stb->cols, i); + if (col->sma) { + if (first_sma) { + pos += snprintf(buf + pos, size - pos, " SMA(%s", col->name); + first_sma = false; + } else { + pos += snprintf(buf + pos, size - pos, ",%s", col->name); + } + if (pos <= 0 || pos >= size) return -1; + } + } + if (!first_sma) { + pos += snprintf(buf + pos, size - pos, ")"); + if (pos <= 0 || pos >= size) return -1; + } + + pos += snprintf(buf + pos, size - pos, ";\n"); + if (pos <= 0 || pos >= size) return -1; + + // infoPrint("create stable: <%s>.\n", buf); + return (pos > 0 && pos < size) ? pos : -1; +} + + +static int csvExportCreateStbSql(CsvWriteMeta* write_meta, FILE* fp) { + char buf[4096] = {0}; + int ret = 0; + int length = 0; + + length = csvGenCreateStbSql(write_meta->db, write_meta->stb, buf, sizeof(buf)); + if (length < 0) { + errorPrint("Failed to generate create stb sql, maybe buffer[%zu] not enough.\n", sizeof(buf)); + return -1; + } + + ret = fwrite(buf, 1, length, fp); + if (ret != length) { + errorPrint("Failed to write create stb sql: %s. expected written %d but %d.\n", + buf, length, ret); + if (ferror(fp)) { + perror("error"); + } + return -1; + } + + return 0; +} + + +static int csvExportCreateSql(CsvWriteMeta* write_meta) { + char fullname[MAX_PATH_LEN] = {0}; + int ret = 0; + int length = 0; + FILE* fp = NULL; + + + length = snprintf(fullname, sizeof(fullname), "%s%s.txt", g_arguments->output_path, "create_stmt"); + if (length <= 0 || length >= sizeof(fullname)) { + return -1; + } + + fp = fopen(fullname, "w"); + if (!fp) { + return -1; + } + + + // export db + ret = csvExportCreateDbSql(write_meta, fp); + if (ret < 0) { + goto end; + } + + // export stb + ret = csvExportCreateStbSql(write_meta, fp); + if (ret < 0) { + goto end; + } + + succPrint("Export create sql to file: %s successfully.\n", fullname); + +end: + if (fp) { + fclose(fp); + } + + return ret; +} + + +static int csvInitWriteMeta(SDataBase* db, SSuperTable* stb, CsvWriteMeta* write_meta) { + write_meta->naming_type = csvGetFileNamingType(stb); + write_meta->total_threads = 1; + write_meta->csv_header_length = 0; + write_meta->db = db; + write_meta->stb = stb; + write_meta->start_ts = stb->startTimestamp; + write_meta->end_ts = stb->startTimestamp + stb->timestamp_step * stb->insertRows; + write_meta->ts_step = stb->timestamp_step * stb->insertRows; + write_meta->interlace_step = stb->timestamp_step * stb->interlaceRows; + + int ret = csvGenCsvHeader(write_meta); + if (ret < 0) { + errorPrint("Failed to generate csv header data. database: %s, super table: %s, naming type: %d.\n", + db->dbName, stb->stbName, write_meta->naming_type); + return -1; + } + + switch (write_meta->naming_type) { + case CSV_NAMING_I_SINGLE: { + (void)snprintf(write_meta->mode, sizeof(write_meta->mode), "interlace::normal"); + break; + } + case CSV_NAMING_I_TIME_SLICE: { + (void)snprintf(write_meta->mode, sizeof(write_meta->mode), "interlace::time-slice"); + csvCalcTimestampStep(write_meta); + break; + } + case CSV_NAMING_B_THREAD: { + write_meta->total_threads = MIN(g_arguments->nthreads, stb->childTblCount); + (void)snprintf(write_meta->mode, sizeof(write_meta->mode), "batch[%zu]::normal", write_meta->total_threads); + csvGenThreadFormatter(write_meta); + break; + } + case CSV_NAMING_B_THREAD_TIME_SLICE: { + write_meta->total_threads = MIN(g_arguments->nthreads, stb->childTblCount); + (void)snprintf(write_meta->mode, sizeof(write_meta->mode), "batch[%zu]::time-slice", write_meta->total_threads); + csvGenThreadFormatter(write_meta); + csvCalcTimestampStep(write_meta); + break; + } + default: { + write_meta->naming_type = CSV_NAMING_I_SINGLE; + break; + } + } + + return 0; +} + + +static void csvInitThreadMeta(CsvWriteMeta* write_meta, uint32_t thread_id, CsvThreadMeta* thread_meta) { + SDataBase* db = write_meta->db; + SSuperTable* stb = write_meta->stb; + + thread_meta->ctb_start_idx = 0; + thread_meta->ctb_end_idx = 0; + thread_meta->ctb_count = 0; + thread_meta->start_secs = 0; + thread_meta->end_secs = 0; + thread_meta->start_ts = write_meta->start_ts; + thread_meta->end_ts = write_meta->end_ts; + thread_meta->thread_id = thread_id; + thread_meta->output_header = false; + thread_meta->tags_buf_size = 0; + thread_meta->tags_buf_array = NULL; + thread_meta->cols_buf = NULL; + + csvCalcCtbRange(thread_meta, write_meta->total_threads, stb->childTblFrom, stb->childTblCount); + + switch (write_meta->naming_type) { + case CSV_NAMING_I_SINGLE: + case CSV_NAMING_B_THREAD: { + break; + } + case CSV_NAMING_I_TIME_SLICE: + case CSV_NAMING_B_THREAD_TIME_SLICE: { + thread_meta->start_secs = csvGetStartSeconds(db->precision, stb->startTimestamp, stb->csv_ts_format); + thread_meta->end_secs = thread_meta->start_secs + write_meta->stb->csv_ts_intv_secs; + csvCalcSliceTimestamp(write_meta, thread_meta); + break; + } + default: { + break; + } + } + + return; +} + + +static void csvUpdateSliceRange(CsvWriteMeta* write_meta, CsvThreadMeta* thread_meta, int64_t last_end_ts) { + SDataBase* db = write_meta->db; + SSuperTable* stb = write_meta->stb; + + switch (write_meta->naming_type) { + case CSV_NAMING_I_SINGLE: + case CSV_NAMING_B_THREAD: { + break; + } + case CSV_NAMING_I_TIME_SLICE: + case CSV_NAMING_B_THREAD_TIME_SLICE: { + thread_meta->start_secs = csvGetStartSeconds(db->precision, last_end_ts, stb->csv_ts_format); + thread_meta->end_secs = thread_meta->start_secs + write_meta->stb->csv_ts_intv_secs; + csvCalcSliceTimestamp(write_meta, thread_meta); + break; + } + default: { + break; + } + } + + return; +} + + +static const char* csvGetGzipFilePrefix(CsvCompressionLevel csv_compress_level) { + if (csv_compress_level == CSV_COMPRESS_NONE) { + return ""; + } else { + return ".gz"; + } +} + + +static int csvGetFileFullname(CsvWriteMeta* write_meta, CsvThreadMeta* thread_meta, char* fullname, size_t size) { + char thread_buf[SMALL_BUFF_LEN]; + char start_time_buf[MIDDLE_BUFF_LEN]; + char end_time_buf[MIDDLE_BUFF_LEN]; + int ret = -1; + const char* base_path = g_arguments->output_path; + const char* file_prefix = write_meta->stb->csv_file_prefix; + const char* gzip_suffix = csvGetGzipFilePrefix(write_meta->stb->csv_compress_level); + + switch (write_meta->naming_type) { + case CSV_NAMING_I_SINGLE: { + ret = snprintf(fullname, size, "%s%s.csv%s", base_path, file_prefix, gzip_suffix); + break; + } + case CSV_NAMING_I_TIME_SLICE: { + csvConvertTime2String(thread_meta->start_secs, write_meta->stb->csv_ts_format, start_time_buf, sizeof(start_time_buf)); + csvConvertTime2String(thread_meta->end_secs, write_meta->stb->csv_ts_format, end_time_buf, sizeof(end_time_buf)); + ret = snprintf(fullname, size, "%s%s_%s_%s.csv%s", base_path, file_prefix, start_time_buf, end_time_buf, gzip_suffix); + break; + } + case CSV_NAMING_B_THREAD: { + (void)snprintf(thread_buf, sizeof(thread_buf), write_meta->thread_formatter, thread_meta->thread_id); + ret = snprintf(fullname, size, "%s%s_%s.csv%s", base_path, file_prefix, thread_buf, gzip_suffix); + break; + } + case CSV_NAMING_B_THREAD_TIME_SLICE: { + (void)snprintf(thread_buf, sizeof(thread_buf), write_meta->thread_formatter, thread_meta->thread_id); + csvConvertTime2String(thread_meta->start_secs, write_meta->stb->csv_ts_format, start_time_buf, sizeof(start_time_buf)); + csvConvertTime2String(thread_meta->end_secs, write_meta->stb->csv_ts_format, end_time_buf, sizeof(end_time_buf)); + ret = snprintf(fullname, size, "%s%s_%s_%s_%s.csv%s", base_path, file_prefix, thread_buf, start_time_buf, end_time_buf, gzip_suffix); + break; + } + default: { + ret = -1; + break; + } + } + + return (ret > 0 && (size_t)ret < size) ? 0 : -1; +} + + +static int64_t csvCalcSliceBatchTimestamp(CsvWriteMeta* write_meta, int64_t slice_cur_ts, int64_t slice_end_ts) { + int64_t slice_batch_ts = 0; + + switch (write_meta->naming_type) { + case CSV_NAMING_I_SINGLE: + case CSV_NAMING_I_TIME_SLICE: { + slice_batch_ts = MIN(slice_cur_ts + write_meta->interlace_step, slice_end_ts); + break; + } + case CSV_NAMING_B_THREAD: + case CSV_NAMING_B_THREAD_TIME_SLICE: { + slice_batch_ts = slice_end_ts; + break; + } + default: { + break; + } + } + + return slice_batch_ts; +} + + +static int csvGenRowFields(char* buf, int size, SSuperTable* stb, int fields_cate, int64_t* k) { + int pos = 0; + BArray* fields = NULL; + int16_t field_count = 0; + char* binanry_prefix = stb->binaryPrefex ? stb->binaryPrefex : ""; + char* nchar_prefix = stb->ncharPrefex ? stb->ncharPrefex : ""; + + if (!buf || !stb || !k || size <= 0) { + return -1; + } + + if (fields_cate == GEN_ROW_FIELDS_TAG) { + fields = stb->tags; + field_count = stb->tags->size; + } else { + fields = stb->cols; + field_count = stb->cols->size; + } + + for (uint16_t i = 0; i < field_count; ++i) { + Field* field = benchArrayGet(fields, i); + char* prefix = ""; + if(field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY) { + prefix = binanry_prefix; + } else if(field->type == TSDB_DATA_TYPE_NCHAR) { + prefix = nchar_prefix; + } + pos += dataGenByField(field, buf, pos, prefix, k, ""); + } + + return pos; +} + + +static int csvGenRowTagData(char* buf, int size, SSuperTable* stb, int64_t index, int64_t* k) { + if (!buf || !stb || !k || size <= 0) { + return -1; + } + + // tbname + int pos = snprintf(buf, size, ",'%s%"PRId64"'", stb->childTblPrefix, index); + + // tags + pos += csvGenRowFields(buf + pos, size - pos, stb, GEN_ROW_FIELDS_TAG, k); + + return (pos > 0 && pos < size) ? pos : -1; +} + + +static int csvGenRowColData(char* buf, int size, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k) { + char ts_fmt[128] = {0}; + toolsFormatTimestamp(ts_fmt, ts, precision); + int pos = snprintf(buf, size, "\'%s\'", ts_fmt); + + // columns + pos += csvGenRowFields(buf + pos, size - pos, stb, GEN_ROW_FIELDS_COL, k); + return (pos > 0 && pos < size) ? pos : -1; +} + + +static void csvFreeCtbTagData(CsvThreadMeta* thread_meta, CsvRowTagsBuf* tags_buf_array) { + if (!thread_meta || !tags_buf_array) { + return; + } + + for (uint64_t i = 0 ; i < thread_meta->ctb_count; ++i) { + char* tags_buf = tags_buf_array[i].buf; + if (tags_buf) { + tmfree(tags_buf); + } else { + break; + } + } + tmfree(tags_buf_array); + return; +} + + +static CsvRowTagsBuf* csvGenCtbTagData(CsvWriteMeta* write_meta, CsvThreadMeta* thread_meta) { + SSuperTable* stb = write_meta->stb; + int ret = 0; + int64_t tk = 0; + + if (!write_meta || !thread_meta) { + return NULL; + } + + CsvRowTagsBuf* tags_buf_array = (CsvRowTagsBuf*)benchCalloc(thread_meta->ctb_count, sizeof(CsvRowTagsBuf), true); + if (!tags_buf_array) { + return NULL; + } + + char* tags_buf = NULL; + int tags_buf_size = TSDB_TABLE_NAME_LEN + stb->lenOfTags + stb->tags->size; + for (uint64_t i = 0; i < thread_meta->ctb_count; ++i) { + tags_buf = benchCalloc(1, tags_buf_size, true); + if (!tags_buf) { + goto error; + } + + tags_buf_array[i].buf = tags_buf; + + ret = csvGenRowTagData(tags_buf, tags_buf_size, stb, thread_meta->ctb_start_idx + i, &tk); + if (ret <= 0) { + goto error; + } + + tags_buf_array[i].length = ret; + } + thread_meta->tags_buf_size = tags_buf_size; + + return tags_buf_array; + +error: + csvFreeCtbTagData(thread_meta, tags_buf_array); return NULL; } -int csvTestProcess() { - pthread_t handle; - int ret = pthread_create(&handle, NULL, csvWriteThread, NULL); - if (ret != 0) { - errorPrint("pthread_create failed. error code =%d \n", ret); - return -1; + +static CsvFileHandle* csvOpen(const char* filename, CsvCompressionLevel compress_level) { + CsvFileHandle* fhdl = NULL; + bool failed = false; + + fhdl = (CsvFileHandle*)benchCalloc(1, sizeof(CsvFileHandle), false); + if (!fhdl) { + errorPrint("Failed to malloc csv file handle. filename: %s, compress level: %d.\n", + filename, compress_level); + return NULL; } - infoPrint("start output to csv %s ...\n", g_arguments->csvPath); - int64_t start = toolsGetTimestampMs(); - pthread_join(handle, NULL); - int64_t delay = toolsGetTimestampMs() - start; - infoPrint("output to csv %s finished. delay:%"PRId64"s \n", g_arguments->csvPath, delay/1000); - - return 0; -} - -int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir) { - // filename - int ret = 0; - char outFile[MAX_FILE_NAME_LEN] = {0}; - obtainCsvFile(outFile, db, stb, outDir); - FILE * fs = fopen(outFile, "w"); - if(fs == NULL) { - errorPrint("failed create csv file. file=%s, last errno=%d strerror=%s \n", outFile, errno, strerror(errno)); - return -1; - } - - int rowLen = TSDB_TABLE_NAME_LEN + stb->lenOfTags + stb->lenOfCols + stb->tags->size + stb->cols->size; - int bufLen = rowLen * g_arguments->reqPerReq; - char* buf = benchCalloc(1, bufLen, true); - - infoPrint("start write csv file: %s \n", outFile); - - if (stb->interlaceRows > 0) { - // interlace mode - ret = interlaceWriteCsv(db, stb, fs, buf, bufLen, rowLen * 2); + if (compress_level == CSV_COMPRESS_NONE) { + fhdl->handle.fp = fopen(filename, "w"); + failed = (!fhdl->handle.fp); } else { - // batch mode - ret = batchWriteCsv(db, stb, fs, buf, bufLen, rowLen * 2); + char mode[TINY_BUFF_LEN]; + (void)snprintf(mode, sizeof(mode), "wb%d", compress_level); + fhdl->handle.gf = gzopen(filename, mode); + failed = (!fhdl->handle.gf); } - tmfree(buf); - fclose(fs); - - succPrint("end write csv file: %s \n", outFile); - return ret; -} - - -void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir) { - sprintf(outFile, "%s%s-%s.csv", outDir, db->dbName, stb->stbName); -} - -int32_t writeCsvFile(FILE* f, char * buf, int32_t len) { - size_t size = fwrite(buf, 1, len, f); - if(size != len) { - errorPrint("failed to write csv file. expect write length:%d real write length:%d \n", len, (int32_t)size); - return -1; + if (failed) { + tmfree(fhdl); + errorPrint("Failed to open csv file handle. filename: %s, compress level: %d.\n", + filename, compress_level); + return NULL; + } else { + fhdl->filename = filename; + fhdl->compress_level = compress_level; + fhdl->result = CSV_ERR_OK; + return fhdl; } - return 0; } -int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain) { - int ret = 0; - int pos = 0; - int64_t tk = 0; - int64_t show = 0; - int tagDataLen = stb->lenOfTags + stb->tags->size + 256; - char * tagData = (char *) benchCalloc(1, tagDataLen, true); - int colDataLen = stb->lenOfCols + stb->cols->size + 256; - char * colData = (char *) benchCalloc(1, colDataLen, true); - - // gen child name - for (int64_t i = 0; i < stb->childTblCount; i++) { - int64_t ts = stb->startTimestamp; - int64_t ck = 0; - // tags - genTagData(tagData, stb, i, &tk); - // insert child column data - for(int64_t j = 0; j < stb->insertRows; j++) { - genColumnData(colData, stb, ts, db->precision, &ck); - // combine - pos += sprintf(buf + pos, "%s,%s\n", tagData, colData); - if (bufLen - pos < minRemain) { - // submit - ret = writeCsvFile(fs, buf, pos); - if (ret != 0) { - goto END; - } - - pos = 0; +static CsvIoError csvWrite(CsvFileHandle* fhdl, const char* buf, size_t size) { + if (fhdl->compress_level == CSV_COMPRESS_NONE) { + size_t ret = fwrite(buf, 1, size, fhdl->handle.fp); + if (ret != size) { + errorPrint("Failed to write csv file: %s. expected written %zu but %zu.\n", + fhdl->filename, size, ret); + if (ferror(fhdl->handle.fp)) { + perror("error"); } - - // ts move next - ts += stb->timestamp_step; - - // check cancel - if(g_arguments->terminate) { - infoPrint("%s", "You are cancel, exiting ...\n"); - ret = -1; - goto END; - } - - // print show - if (++show % SHOW_CNT == 0) { - infoPrint("batch write child table cnt = %"PRId64 " all rows = %" PRId64 "\n", i+1, show); - } - + fhdl->result = CSV_ERR_WRITE_FAILED; + return CSV_ERR_WRITE_FAILED; + } + } else { + int ret = gzwrite(fhdl->handle.gf, buf, size); + if (ret != size) { + errorPrint("Failed to write csv file: %s. expected written %zu but %d.\n", + fhdl->filename, size, ret); + int errnum; + const char* errmsg = gzerror(fhdl->handle.gf, &errnum); + errorPrint("gzwrite error: %s\n", errmsg); + fhdl->result = CSV_ERR_WRITE_FAILED; + return CSV_ERR_WRITE_FAILED; } } - if (pos > 0) { - ret = writeCsvFile(fs, buf, pos); - pos = 0; - } - -END: - // free - tmfree(tagData); - tmfree(colData); - return ret; + return CSV_ERR_OK; } -int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain) { - int ret = 0; - int pos = 0; - int64_t n = 0; // already inserted rows for one child table - int64_t tk = 0; - int64_t show = 0; - char **tagDatas = (char **)benchCalloc(stb->childTblCount, sizeof(char *), true); - int colDataLen = stb->lenOfCols + stb->cols->size + 256; - char * colData = (char *) benchCalloc(1, colDataLen, true); - int64_t last_ts = stb->startTimestamp; - - while (n < stb->insertRows ) { - for (int64_t i = 0; i < stb->childTblCount; i++) { - // start one table - int64_t ts = last_ts; - int64_t ck = 0; - // tags - if (tagDatas[i] == NULL) { - tagDatas[i] = genTagData(NULL, stb, i, &tk); +static void csvClose(CsvFileHandle* fhdl) { + if (!fhdl) { + return; + } + + if (fhdl->compress_level == CSV_COMPRESS_NONE) { + if (fhdl->handle.fp) { + fclose(fhdl->handle.fp); + fhdl->handle.fp = NULL; + } + } else { + if (fhdl->handle.gf) { + gzclose(fhdl->handle.gf); + fhdl->handle.gf = NULL; + } + } + + tmfree(fhdl); +} + + +static int csvWriteFile(CsvFileHandle* fhdl, uint64_t ctb_idx, int64_t cur_ts, int64_t* ck, CsvWriteMeta* write_meta, CsvThreadMeta* thread_meta) { + SDataBase* db = write_meta->db; + SSuperTable* stb = write_meta->stb; + CsvRowTagsBuf* tags_buf_array = thread_meta->tags_buf_array; + CsvRowTagsBuf* tags_buf = &tags_buf_array[ctb_idx]; + CsvRowColsBuf* cols_buf = thread_meta->cols_buf; + int ret = 0; + + + ret = csvGenRowColData(cols_buf->buf, cols_buf->buf_size, stb, cur_ts, db->precision, ck); + if (ret <= 0) { + errorPrint("Failed to generate csv column data. database: %s, super table: %s, naming type: %d, thread index: %zu, ctb index: %" PRIu64 ".\n", + db->dbName, stb->stbName, write_meta->naming_type, thread_meta->thread_id, ctb_idx); + return -1; + } + + cols_buf->length = ret; + + // write header + if (thread_meta->output_header) { + ret = csvWrite(fhdl, write_meta->csv_header, write_meta->csv_header_length); + if (ret != CSV_ERR_OK) { + errorPrint("Failed to write csv header data. database: %s, super table: %s, naming type: %d, thread index: %zu, ctb index: %" PRIu64 ".\n", + db->dbName, stb->stbName, write_meta->naming_type, thread_meta->thread_id, ctb_idx); + return -1; + } + + thread_meta->output_header = false; + } + + // write columns + ret = csvWrite(fhdl, cols_buf->buf, cols_buf->length); + if (ret != CSV_ERR_OK) { + errorPrint("Failed to write csv column data. database: %s, super table: %s, naming type: %d, thread index: %zu, ctb index: %" PRIu64 ".\n", + db->dbName, stb->stbName, write_meta->naming_type, thread_meta->thread_id, ctb_idx); + return -1; + } + + // write tags + ret = csvWrite(fhdl, tags_buf->buf, tags_buf->length); + if (ret != CSV_ERR_OK) { + errorPrint("Failed to write csv tag data. database: %s, super table: %s, naming type: %d, thread index: %zu, ctb index: %" PRIu64 ".\n", + db->dbName, stb->stbName, write_meta->naming_type, thread_meta->thread_id, ctb_idx); + return -1; + } + + // write line break + ret = csvWrite(fhdl, "\n", 1); + if (ret != CSV_ERR_OK) { + errorPrint("Failed to write csv line break data. database: %s, super table: %s, naming type: %d, thread index: %zu, ctb index: %" PRIu64 ".\n", + db->dbName, stb->stbName, write_meta->naming_type, thread_meta->thread_id, ctb_idx); + return -1; + } + + return 0; +} + + +static void* csvGenStbThread(void* arg) { + CsvThreadArgs* thread_arg = (CsvThreadArgs*)arg; + CsvWriteMeta* write_meta = thread_arg->write_meta; + CsvThreadMeta* thread_meta = &thread_arg->thread_meta; + SDataBase* db = write_meta->db; + SSuperTable* stb = write_meta->stb; + + int64_t cur_ts = 0; + int64_t slice_cur_ts = 0; + int64_t slice_end_ts = 0; + int64_t slice_batch_ts = 0; + int64_t slice_ctb_cur_ts = 0; + int64_t ck = 0; + uint64_t ctb_idx = 0; + int ret = 0; + CsvFileHandle* fhdl = NULL; + char fullname[MAX_PATH_LEN] = {0}; + + uint64_t total_rows = 0; + uint64_t pre_total_rows = 0; + uint64_t file_rows = 0; + int64_t start_print_ts = 0; + int64_t pre_print_ts = 0; + int64_t cur_print_ts = 0; + int64_t print_ts_elapse = 0; + + + // tags buffer + CsvRowTagsBuf* tags_buf_array = csvGenCtbTagData(write_meta, thread_meta); + if (!tags_buf_array) { + errorPrint("Failed to generate csv tag data. database: %s, super table: %s, naming type: %d, thread index: %zu.\n", + db->dbName, stb->stbName, write_meta->naming_type, thread_meta->thread_id); + return NULL; + } + + // column buffer + int buf_size = stb->lenOfCols + stb->cols->size; + char* buf = (char*)benchCalloc(1, buf_size, true); + if (!buf) { + errorPrint("Failed to malloc csv column buffer. database: %s, super table: %s, naming type: %d, thread index: %zu.\n", + db->dbName, stb->stbName, write_meta->naming_type, thread_meta->thread_id); + goto end; + } + + CsvRowColsBuf cols_buf = { + .buf = buf, + .buf_size = buf_size, + .length = 0 + }; + + thread_meta->tags_buf_array = tags_buf_array; + thread_meta->cols_buf = &cols_buf; + start_print_ts = toolsGetTimestampMs(); + + cur_ts = write_meta->start_ts; + while (cur_ts < write_meta->end_ts) { + // get filename + ret = csvGetFileFullname(write_meta, thread_meta, fullname, sizeof(fullname)); + if (ret < 0) { + errorPrint("Failed to generate csv filename. database: %s, super table: %s, naming type: %d, thread index: %zu.\n", + db->dbName, stb->stbName, write_meta->naming_type, thread_meta->thread_id); + goto end; + } + + // create fd + fhdl = csvOpen(fullname, stb->csv_compress_level); + if (fhdl == NULL) { + errorPrint("Failed to create csv file. thread index: %zu, file: %s, errno: %d, strerror: %s.\n", + thread_meta->thread_id, fullname, errno, strerror(errno)); + goto end; + } + + + thread_meta->output_header = stb->csv_output_header; + slice_cur_ts = cur_ts; + slice_end_ts = MIN(thread_meta->end_ts, write_meta->end_ts); + file_rows = 0; + pre_print_ts = toolsGetTimestampMs(); + + infoPrint("thread[%zu] begin to write csv file: %s.\n", thread_meta->thread_id, fullname); + + // write data + while (slice_cur_ts < slice_end_ts) { + slice_batch_ts = csvCalcSliceBatchTimestamp(write_meta, slice_cur_ts, slice_end_ts); + + for (ctb_idx = 0; ctb_idx < thread_meta->ctb_count; ++ctb_idx) { + for (slice_ctb_cur_ts = slice_cur_ts; slice_ctb_cur_ts < slice_batch_ts; slice_ctb_cur_ts += write_meta->stb->timestamp_step) { + ret = csvWriteFile(fhdl, ctb_idx, slice_ctb_cur_ts, &ck, write_meta, thread_meta); + if (ret) { + errorPrint("Failed to write csv file. thread index: %zu, file: %s, errno: %d, strerror: %s.\n", + thread_meta->thread_id, fullname, errno, strerror(errno)); + csvClose(fhdl); + goto end; + } + + ck += 1; + total_rows += 1; + file_rows += 1; + + cur_print_ts = toolsGetTimestampMs(); + print_ts_elapse = cur_print_ts - pre_print_ts; + if (print_ts_elapse > 30000) { + infoPrint("thread[%zu] has currently inserted rows: %" PRIu64 ", period insert rate: %.2f rows/s.\n", + thread_meta->thread_id, total_rows, (total_rows - pre_total_rows) * 1000.0 / print_ts_elapse); + + pre_print_ts = cur_print_ts; + pre_total_rows = total_rows; + } + + + if (g_arguments->terminate) { + csvClose(fhdl); + goto end; + } + } } - // calc need insert rows - int64_t needInserts = stb->interlaceRows; - if(needInserts > stb->insertRows - n) { - needInserts = stb->insertRows - n; - } + slice_cur_ts = slice_batch_ts; + } - for (int64_t j = 0; j < needInserts; j++) { - genColumnData(colData, stb, ts, db->precision, &ck); - // combine tags,cols - pos += sprintf(buf + pos, "%s,%s\n", tagDatas[i], colData); - if (bufLen - pos < minRemain) { - // submit - ret = writeCsvFile(fs, buf, pos); - if (ret != 0) { - goto END; - } - pos = 0; + csvClose(fhdl); + cur_ts = thread_meta->end_ts; + csvUpdateSliceRange(write_meta, thread_meta, slice_end_ts); + } + + cur_print_ts = toolsGetTimestampMs(); + print_ts_elapse = cur_print_ts - start_print_ts; + + succPrint("thread [%zu] has completed inserting rows: %" PRIu64 ", insert rate %.2f rows/s.\n", + thread_meta->thread_id, total_rows, total_rows * 1000.0 / print_ts_elapse); + +end: + thread_meta->total_rows = total_rows; + csvFreeCtbTagData(thread_meta, tags_buf_array); + tmfree(buf); + return NULL; +} + + +static int csvGenStbProcess(SDataBase* db, SSuperTable* stb) { + int ret = 0; + bool prompt = true; + uint64_t total_rows = 0; + int64_t start_ts = 0; + int64_t ts_elapse = 0; + + CsvWriteMeta* write_meta = NULL; + CsvThreadArgs* args = NULL; + pthread_t* pids = NULL; + + + write_meta = benchCalloc(1, sizeof(CsvWriteMeta), false); + if (!write_meta) { + ret = -1; + goto end; + } + + ret = csvInitWriteMeta(db, stb, write_meta); + if (ret < 0) { + ret = -1; + goto end; + } + + infoPrint("export csv mode: %s.\n", write_meta->mode); + + args = benchCalloc(write_meta->total_threads, sizeof(CsvThreadArgs), false); + if (!args) { + ret = -1; + goto end; + } + + pids = benchCalloc(write_meta->total_threads, sizeof(pthread_t), false); + if (!pids) { + ret = -1; + goto end; + } + + start_ts = toolsGetTimestampMs(); + for (uint32_t i = 0; (i < write_meta->total_threads && !g_arguments->terminate); ++i) { + CsvThreadArgs* arg = &args[i]; + arg->write_meta = write_meta; + csvInitThreadMeta(write_meta, i + 1, &arg->thread_meta); + + ret = pthread_create(&pids[i], NULL, csvGenStbThread, arg); + if (ret) { + perror("Failed to create thread"); + goto end; + } + } + + // wait threads + for (uint32_t i = 0; i < write_meta->total_threads; ++i) { + if (g_arguments->terminate && prompt) { + infoPrint("Operation cancelled by user, exiting gracefully...\n"); + prompt = false; + } + + infoPrint("pthread_join %u ...\n", i); + pthread_join(pids[i], NULL); + } + + // statistics + total_rows = 0; + for (uint32_t i = 0; i < write_meta->total_threads; ++i) { + CsvThreadArgs* arg = &args[i]; + CsvThreadMeta* thread_meta = &arg->thread_meta; + total_rows += thread_meta->total_rows; + } + + ts_elapse = toolsGetTimestampMs() - start_ts; + if (ts_elapse > 0) { + succPrint("Spent %.6f seconds to insert rows: %" PRIu64 " with %zu thread(s) into %s, at a rate of %.2f rows/s.\n", + ts_elapse / 1000.0, total_rows, write_meta->total_threads, g_arguments->output_path, total_rows * 1000.0 / ts_elapse); + } + + // export create db/stb sql + ret = csvExportCreateSql(write_meta); + +end: + tmfree(pids); + tmfree(args); + tmfree(write_meta); + return ret; +} + + +static void csvGenPrepare(SDataBase* db, SSuperTable* stb) { + stb->lenOfTags = accumulateRowLen(stb->tags, stb->iface); + stb->lenOfCols = accumulateRowLen(stb->cols, stb->iface); + + if (stb->childTblTo) { + stb->childTblCount = stb->childTblTo - stb->childTblFrom; + } + + return; +} + + +static int csvGenStb(SDataBase* db, SSuperTable* stb) { + // prepare + csvGenPrepare(db, stb); + + return csvGenStbProcess(db, stb); +} + + +static int csvWriteThread() { + for (size_t i = 0; i < g_arguments->databases->size && !g_arguments->terminate; ++i) { + // database + SDataBase* db = benchArrayGet(g_arguments->databases, i); + if (db->superTbls) { + for (size_t j = 0; j < db->superTbls->size && !g_arguments->terminate; ++j) { + // stb + SSuperTable* stb = benchArrayGet(db->superTbls, j); + if (stb->insertRows == 0) { + continue; } - // ts move next - ts += stb->timestamp_step; - - // check cancel - if(g_arguments->terminate) { - infoPrint("%s", "You are cancel, exiting ... \n"); - ret = -1; - goto END; + // parsing parameters + int ret = csvParseStbParameter(stb); + if (ret != 0) { + errorPrint("Failed to parse csv parameter. database: %s, super table: %s, error code: %d.\n", + db->dbName, stb->stbName, ret); + return -1; } - // print show - if (++show % SHOW_CNT == 0) { - infoPrint("interlace write child table index = %"PRId64 " all rows = %"PRId64 "\n", i+1, show); + // gen csv + ret = csvGenStb(db, stb); + if(ret != 0) { + errorPrint("Failed to generate csv files. database: %s, super table: %s, error code: %d.\n", + db->dbName, stb->stbName, ret); + return -1; } } - - // if last child table - if (i + 1 == stb->childTblCount ) { - n += needInserts; - last_ts = ts; - } } } - if (pos > 0) { - ret = writeCsvFile(fs, buf, pos); - pos = 0; - } - -END: - // free - for (int64_t m = 0 ; m < stb->childTblCount; m ++) { - tmfree(tagDatas[m]); - } - tmfree(tagDatas); - tmfree(colData); - return ret; -} - -// gen tag data -char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k) { - // malloc - char* tagData; - if (buf == NULL) { - int tagDataLen = TSDB_TABLE_NAME_LEN + stb->lenOfTags + stb->tags->size + 32; - tagData = benchCalloc(1, tagDataLen, true); - } else { - tagData = buf; - } - - int pos = 0; - // tbname - pos += sprintf(tagData, "\'%s%"PRId64"\'", stb->childTblPrefix, i); - // tags - pos += genRowByField(tagData + pos, stb->tags, stb->tags->size, stb->binaryPrefex, stb->ncharPrefex, k); - - return tagData; -} - -// gen column data -char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k) { - char szTime[128] = {0}; - toolsFormatTimestamp(szTime, ts, precision); - int pos = sprintf(colData, "\'%s\'", szTime); - - // columns - genRowByField(colData + pos, stb->cols, stb->cols->size, stb->binaryPrefex, stb->ncharPrefex, k); - return colData; + return 0; } -int32_t genRowByField(char* buf, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k) { - - // other cols data - int32_t pos1 = 0; - for(uint16_t i = 0; i < fieldCnt; i++) { - Field* fd = benchArrayGet(fields, i); - char* prefix = ""; - if(fd->type == TSDB_DATA_TYPE_BINARY || fd->type == TSDB_DATA_TYPE_VARBINARY) { - if(binanryPrefix) { - prefix = binanryPrefix; - } - } else if(fd->type == TSDB_DATA_TYPE_NCHAR) { - if(ncharPrefix) { - prefix = ncharPrefix; - } +int csvTestProcess() { + // parsing parameters + if (csvParseParameter() != 0) { + return -1; } - pos1 += dataGenByField(fd, buf, pos1, prefix, k, ""); - } - - return pos1; + infoPrint("Starting to output data to csv files in directory: %s ...\n", g_arguments->output_path); + int64_t start = toolsGetTimestampMs(); + int ret = csvWriteThread(); + if (ret != 0) { + return -1; + } + int64_t elapse = toolsGetTimestampMs() - start; + infoPrint("Generating csv files in directory: %s has been completed. Time elapsed: %.3f seconds\n", + g_arguments->output_path, elapse / 1000.0); + return 0; } diff --git a/tools/taos-tools/src/benchInsert.c b/tools/taos-tools/src/benchInsert.c index 885a6e529f..894e4621a0 100644 --- a/tools/taos-tools/src/benchInsert.c +++ b/tools/taos-tools/src/benchInsert.c @@ -1928,7 +1928,7 @@ static void *syncWriteInterlace(void *sarg) { bindv = createBindV(nBatchTable, tagCnt, stbInfo->cols->size + 1); } - bool oldInitStmt = stbInfo->autoTblCreating || database->superTbls->size > 1; + bool oldInitStmt = stbInfo->autoTblCreating; // not auto create table call once if(stbInfo->iface == STMT_IFACE && !oldInitStmt) { debugPrint("call prepareStmt for stable:%s\n", stbInfo->stbName); @@ -2900,7 +2900,7 @@ void *syncWriteProgressive(void *sarg) { tagData = benchCalloc(TAG_BATCH_COUNT, stbInfo->lenOfTags, false); } - bool oldInitStmt = stbInfo->autoTblCreating || database->superTbls->size > 1; + bool oldInitStmt = stbInfo->autoTblCreating; // stmt. not auto table create call on stmt if (stbInfo->iface == STMT_IFACE && !oldInitStmt) { if (prepareStmt(pThreadInfo->conn->stmt, stbInfo, tagData, w)) { @@ -3909,11 +3909,8 @@ int32_t initInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr if (NULL == pThreadInfo->conn) { goto END; } + // single always true for benchmark bool single = true; - if (database->superTbls->size > 1) { - single = false; - } - if (stbInfo->iface == STMT2_IFACE) { // stmt2 init if (pThreadInfo->conn->stmt2) diff --git a/tools/taos-tools/src/benchJsonOpt.c b/tools/taos-tools/src/benchJsonOpt.c index 4791385741..5b992b388e 100644 --- a/tools/taos-tools/src/benchJsonOpt.c +++ b/tools/taos-tools/src/benchJsonOpt.c @@ -1405,6 +1405,65 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { } } } + + // csv file prefix + tools_cJSON* csv_fp = tools_cJSON_GetObjectItem(stbInfo, "csv_file_prefix"); + if (csv_fp && csv_fp->type == tools_cJSON_String && csv_fp->valuestring != NULL) { + superTable->csv_file_prefix = csv_fp->valuestring; + } else { + superTable->csv_file_prefix = "data"; + } + + // csv timestamp format + tools_cJSON* csv_tf = tools_cJSON_GetObjectItem(stbInfo, "csv_ts_format"); + if (csv_tf && csv_tf->type == tools_cJSON_String && csv_tf->valuestring != NULL) { + superTable->csv_ts_format = csv_tf->valuestring; + } else { + superTable->csv_ts_format = NULL; + } + + // csv timestamp format + tools_cJSON* csv_ti = tools_cJSON_GetObjectItem(stbInfo, "csv_ts_interval"); + if (csv_ti && csv_ti->type == tools_cJSON_String && csv_ti->valuestring != NULL) { + superTable->csv_ts_interval = csv_ti->valuestring; + } else { + superTable->csv_ts_interval = "1d"; + } + + // csv output header + superTable->csv_output_header = true; + tools_cJSON* oph = tools_cJSON_GetObjectItem(stbInfo, "csv_output_header"); + if (oph && oph->type == tools_cJSON_String && oph->valuestring != NULL) { + if (0 == strcasecmp(oph->valuestring, "yes")) { + superTable->csv_output_header = true; + } else if (0 == strcasecmp(oph->valuestring, "no")) { + superTable->csv_output_header = false; + } + } + + // csv tbname alias + tools_cJSON* tba = tools_cJSON_GetObjectItem(stbInfo, "csv_tbname_alias"); + if (tba && tba->type == tools_cJSON_String && tba->valuestring != NULL) { + superTable->csv_tbname_alias = tba->valuestring; + } else { + superTable->csv_tbname_alias = "device_id"; + } + + // csv compression level + tools_cJSON* cl = tools_cJSON_GetObjectItem(stbInfo, "csv_compress_level"); + if (cl && cl->type == tools_cJSON_String && cl->valuestring != NULL) { + if (0 == strcasecmp(cl->valuestring, "none")) { + superTable->csv_compress_level = CSV_COMPRESS_NONE; + } else if (0 == strcasecmp(cl->valuestring, "fast")) { + superTable->csv_compress_level = CSV_COMPRESS_FAST; + } else if (0 == strcasecmp(cl->valuestring, "balance")) { + superTable->csv_compress_level = CSV_COMPRESS_BALANCE; + } else if (0 == strcasecmp(cl->valuestring, "best")) { + superTable->csv_compress_level = CSV_COMPRESS_BEST; + } + } else { + superTable->csv_compress_level = CSV_COMPRESS_NONE; + } } return 0; } @@ -1586,26 +1645,14 @@ static int getMetaFromCommonJsonFile(tools_cJSON *json) { } } - g_arguments->csvPath[0] = 0; - tools_cJSON *csv = tools_cJSON_GetObjectItem(json, "csvPath"); - if (csv && (csv->type == tools_cJSON_String) - && (csv->valuestring != NULL)) { - tstrncpy(g_arguments->csvPath, csv->valuestring, MAX_FILE_NAME_LEN); - } - - size_t len = strlen(g_arguments->csvPath); - - if(len == 0) { - // set default with current path - strcpy(g_arguments->csvPath, "./output/"); - mkdir(g_arguments->csvPath, 0775); + // output dir + tools_cJSON* opp = tools_cJSON_GetObjectItem(json, "output_dir"); + if (opp && opp->type == tools_cJSON_String && opp->valuestring != NULL) { + g_arguments->output_path = opp->valuestring; } else { - // append end - if (g_arguments->csvPath[len-1] != '/' ) { - strcat(g_arguments->csvPath, "/"); - } - mkdir(g_arguments->csvPath, 0775); + g_arguments->output_path = "./output/"; } + (void)mkdir(g_arguments->output_path, 0775); code = 0; return code; diff --git a/tools/taos-tools/src/benchMain.c b/tools/taos-tools/src/benchMain.c index 86ad795d05..e82da29468 100644 --- a/tools/taos-tools/src/benchMain.c +++ b/tools/taos-tools/src/benchMain.c @@ -153,7 +153,7 @@ int main(int argc, char* argv[]) { } } else if (g_arguments->test_mode == CSVFILE_TEST) { if (csvTestProcess()) { - errorPrint("%s", "query test process failed\n"); + errorPrint("%s", "generate csv process failed\n"); ret = -1; } } else if (g_arguments->test_mode == QUERY_TEST) { diff --git a/tools/tdgpt/README.md b/tools/tdgpt/README.md new file mode 100644 index 0000000000..0b61f8bcef --- /dev/null +++ b/tools/tdgpt/README.md @@ -0,0 +1,135 @@ +# Table of Contents + +1. [Introduction](#1-introduction) +1. [Documentation](#2-documentation) +1. [Prerequisites](#3-prerequisites) +1. [Building](#4-building) +1. [Packaging](#5-packaging) +1. [Installation](#6-installing) +1. [Running](#7-running) +1. [Testing](#8-testing) +1. [Releasing](#9-releasing) +1. [CI/CD](#10-cicd) +1. [Coverage](#11-coverage) +1. [Contributing](#12-contributing) + +# 1. Introduction +tdanalytics: an analytic platform for tdengine + +# 2. Documentation + +For user manual, system design and architecture, please refer to [TDengine Documentation](https://docs.tdengine.com/next) ([TDengine 文档](https://docs.taosdata.com/next)). + +# 3. Prerequisites + +List the software and tools required to work on the project. + +- python 3.10.12+ (for test) + +Step-by-step instructions to set up the prerequisites software. + +## 3.1 Install Python3.10 +Make sure Python3.10 or above is available before installing anode in your system. + +In case of Ubuntu, use the following instructions to install Python 3.10. + +``` +sudo apt-get install software-properties-common +sudo add-apt-repository ppa:deadsnakes/ppa +sudo apt update +sudo apt install python3.10 +sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 2 +sudo update-alternatives --config python3 +sudo apt install python3.10-venv +sudo apt install python3.10-dev +``` + +Install the Pip3.10 + +```bash +curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 +``` + +Add the ``~/.local/bin`` into ``~/.bashrc`` or ``~/.bash_profile`` + +```bash +export PATH=$PATH:~/.local/bin +``` + +# 4. Building +There is no need to build the taosanode, since it is implemented in Python, which is an interpreted language. + + +# 5. Packaging +In the base directory, you can use the following command to package to build an tarball. + +```bash +cd script && ./release.sh +``` + +After the packaging is completed, you will find the tarball in the `release` directory. + +```bash +ls -lht /root/tdanalytics/release + +-rw-rw-r-- 1 root root 74K Feb 21 17:04 TDengine-enterprise-anode-1.0.1.tar.gz +``` + +# 6. Installing + +## 6.1 Install taosanode + +Please use the following command to install taosanode in your system. + +```bash +./install.sh +``` + +During the installation, Python virtual environment will be established in `/var/lib/taos/taosanode/venv` by default, as well as the required libraries. +The taosanode will be installed as an system service, but will not automatic started when installed. You need to start the service mannually, by using the following command + +```bash +systemctl start taosanoded +``` + + +## 6.2 Configure the Service +taosanode provides the RESTFul service powered by `uWSGI`. You can config the options to tune the +performance by changing the default configuration file `taosanode.ini` located in `/etc/taos`, which is also the configuration directory for `taosd` service. + +```ini +# taosanode service ip:port +http = 127.0.0.1:6090 +``` + +# 7. Running +## 7.1 Start/Stop Service +`systemctl start/stop/restart taosanoded.service` will start/stop/restart the service of taosanode. + + +## 7.2 Uninstall +The command `rmtaosanode` will remove the installed taosanode from your system. Note that the python environment won't removed by this script, you need to remove it mannually. + +# 8. Testing +we use github Actions to run the test suit. Please refer to the file [.github/workflows/python-package.yml](https://github.com/taosdata/tdanalytics/.github/workflows/python-package.yml) for more details. + + +# 9 Releasing +For the complete list of taosanode Releases, please see Releases. + +# 10 CI/CD + +We use github Actions for CI/CD workflow configuration. Please refer to the file .github/workflows/python-package.yml for more details. + + +# 11 Coverage + + +# 12 Contributing + +Guidelines for contributing to the project: + +- Fork the repository +- Create a feature branch +- Submit a pull request + diff --git a/tools/tdgpt/cfg/taosanode.ini b/tools/tdgpt/cfg/taosanode.ini new file mode 100755 index 0000000000..51782bccd6 --- /dev/null +++ b/tools/tdgpt/cfg/taosanode.ini @@ -0,0 +1,81 @@ +#uwsgi --ini taosanode.ini +#uwsgi --reload taosanode.pid +#uwsgi --stop taosanode.pid + +[uwsgi] +# charset +env = LC_ALL = en_US.UTF-8 + +# ip:port +http = 127.0.0.1:6090 + +# the local unix socket file than communicate to Nginx +#socket = 127.0.0.1:8001 +#socket-timeout = 10 + +# base directory +chdir = /usr/local/taos/taosanode/lib + +# initialize python file +wsgi-file = /usr/local/taos/taosanode/lib/taosanalytics/app.py + +# call module of uWSGI +callable = app + +# auto remove unix Socket and pid file when stopping +vacuum = true + +# socket exec model +#chmod-socket = 664 + +# uWSGI pid +uid = root + +# uWSGI gid +gid = root + +# main process +master = true + +# the number of worker processes +processes = 2 + +# pid file +pidfile = /usr/local/taos/taosanode/taosanode.pid + +# enable threads +enable-threads = true + +# the number of threads for each process +threads = 4 + +# memory useage report +memory-report = true + +# smooth restart +reload-mercy = 10 + +# conflict with systemctl, so do NOT uncomment this +# daemonize = /var/log/taos/taosanode/taosanode.log + +# log directory +logto = /var/log/taos/taosanode/taosanode.log + +# wWSGI monitor port +stats = 127.0.0.1:8387 + +# python virtual environment directory +virtualenv = /usr/local/taos/taosanode/venv/ + +[taosanode] +# default app log file +app-log = /var/log/taos/taosanode/taosanode.app.log + +# model storage directory +model-dir = /usr/local/taos/taosanode/model/ + +# default log level +log-level = DEBUG + +# draw the query results +draw-result = 1 diff --git a/tools/tdgpt/cfg/taosanoded.service b/tools/tdgpt/cfg/taosanoded.service new file mode 100755 index 0000000000..a8d86cabe7 --- /dev/null +++ b/tools/tdgpt/cfg/taosanoded.service @@ -0,0 +1,22 @@ +[Unit] +Description=TaosANode Service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Environment=PATH=/usr/lib/taos/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ExecStart=/usr/local/taos/taosanode/bin/start.sh +ExecStop=/usr/local/taos/taosanode/bin/stop.sh +TimeoutStartSec=0 +TimeoutStopSec=120s +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +StandardOutput=null +Restart=always +StartLimitBurst=6 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/tools/tdgpt/requirements.txt b/tools/tdgpt/requirements.txt new file mode 100644 index 0000000000..b123e186e6 --- /dev/null +++ b/tools/tdgpt/requirements.txt @@ -0,0 +1,54 @@ +absl-py==2.1.0 +blinker==1.8.2 +click==8.1.7 +contourpy==1.3.0 +cycler==0.12.1 +Cython==3.0.11 +filelock==3.13.1 +Flask==3.0.3 +fonttools==4.54.1 +fsspec==2024.2.0 +h5py==3.12.1 +itsdangerous==2.2.0 +Jinja2==3.1.4 +joblib==1.4.2 +keras==3.6.0 +kiwisolver==1.4.7 +markdown-it-py==3.0.0 +MarkupSafe==3.0.1 +matplotlib==3.9.2 +mdurl==0.1.2 +ml_dtypes==0.5.0 +mpmath==1.3.0 +namex==0.0.8 +networkx==3.2.1 +numpy==1.26.4 +optree==0.13.0 +outlier-utils==0.0.5 +packaging==24.1 +pandas==1.5.0 +patsy==0.5.6 +pillow==10.4.0 +pmdarima==2.0.4 +pyculiarity==0.0.7 +Pygments==2.18.0 +pyparsing==3.1.4 +python-dateutil==2.9.0.post0 +pytz==2024.2 +rich==13.9.2 +rstl==0.1.3 +scikit-learn==1.5.2 +scipy==1.14.1 +six==1.16.0 +statsmodels==0.14.4 +sympy==1.12 +threadpoolctl==3.5.0 +--find-links https://download.pytorch.org/whl/torch/ +torch==2.4.1+cpu +typing_extensions==4.9.0 +urllib3==2.2.3 +uWSGI==2.0.27 +Werkzeug==3.0.4 +Flask-Testing==0.8.1 +xlsxwriter==3.2.1 +taospy==2.7.16 diff --git a/tools/tdgpt/script/install.sh b/tools/tdgpt/script/install.sh new file mode 100755 index 0000000000..9308b37cfc --- /dev/null +++ b/tools/tdgpt/script/install.sh @@ -0,0 +1,748 @@ +#!/bin/bash +# +# This file is used to install analysis platform on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e + +iplist="" +serverFqdn="" + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) +echo -e "${script_dir}" + +# Dynamic directory +PREFIX="taos" +PRODUCTPREFIX="taosanode" +serverName="${PRODUCTPREFIX}d" +configFile="taosanode.ini" +productName="TDengine Anode" +emailName="taosdata.com" +tarName="package.tar.gz" +logDir="/var/log/${PREFIX}/${PRODUCTPREFIX}" +moduleDir="/var/lib/${PREFIX}/${PRODUCTPREFIX}/model" +venvDir="/var/lib/${PREFIX}/${PRODUCTPREFIX}/venv" +global_conf_dir="/etc/${PREFIX}" +installDir="/usr/local/${PREFIX}/${PRODUCTPREFIX}" + +python_minor_ver=0 #check the python version +bin_link_dir="/usr/bin" + +#install main path +install_main_dir=${installDir} + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo >/dev/null; then + csudo="sudo " +fi + +update_flag=0 +prompt_force=0 + +initd_mod=0 +service_mod=2 +if ps aux | grep -v grep | grep systemd &>/dev/null; then + service_mod=0 +elif $(which service &>/dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &>/dev/null); then + initd_mod=1 + elif $(which insserv &>/dev/null); then + initd_mod=2 + elif $(which update-rc.d &>/dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) || : +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu"; then + # echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian"; then + # echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin"; then + # echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos"; then + # echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora"; then + # echo "This is fedora system" + os_type=2 +elif echo $osinfo | grep -qwi "Linux"; then + # echo "This is Linux system" + os_type=1 + service_mod=0 + initd_mod=0 + service_config_dir="/etc/systemd/system" +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact ${emailName} for support." + os_type=1 +fi + +# ============================= get input parameters ================================================= + +# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] + +# set parameters by default value +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] + +while getopts "hv:e:" arg; do + case $arg in + e) + #echo "interactiveFqdn=$OPTARG" + interactiveFqdn=$(echo $OPTARG) + ;; + v) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + h) + echo "Usage: $(basename $0) -v [server | client] -e [yes | no]" + exit 0 + ;; + ?) #unknow option + echo "unknown argument" + exit 1 + ;; + esac +done + +#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" + +services=(${serverName}) + +function install_services() { + for service in "${services[@]}"; do + install_service ${service} + done +} + +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 "$pid" || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + if [ ! -z "${install_main_dir}" ]; then + ${csudo}rm -rf ${install_main_dir} || : + fi + + ${csudo}mkdir -p ${install_main_dir} + ${csudo}mkdir -p ${install_main_dir}/cfg + ${csudo}mkdir -p ${install_main_dir}/bin + ${csudo}mkdir -p ${install_main_dir}/lib + ${csudo}mkdir -p ${global_conf_dir} +} + +function install_bin_and_lib() { + ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin + ${csudo}cp -r ${script_dir}/lib/* ${install_main_dir}/lib/ + + if [[ ! -e "${bin_link_dir}/rmtaosanode" ]]; then + ${csudo}ln -s ${install_main_dir}/bin/uninstall.sh ${bin_link_dir}/rmtaosanode + fi +} + +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in "${arr[@]}"; do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + + if grep -q "127.0.0.1 $1" /etc/hosts; then + return + else + ${csudo}chmod 666 /etc/hosts + ${csudo}echo "127.0.0.1 $1" >>/etc/hosts + fi +} + +function set_hostname() { + echo -e -n "${GREEN}Host name or IP (assigned to this machine) which can be accessed by your tools or apps (must not be 'localhost')${NC}" + read -e -p " : " -i "$(hostname)" newHostname + while true; do + if [ -z "$newHostname" ]; then + newHostname=$(hostname) + break + elif [ "$newHostname" != "localhost" ]; then + break + else + echo -e -n "${GREEN}Host name or IP (assigned to this machine) which can be accessed by your tools or apps (must not be 'localhost')${NC}" + read -e -p " : " -i "$(hostname)" newHostname + fi + done + + if [ -f ${global_conf_dir}/${configFile} ]; then + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${global_conf_dir}/${configFile} + else + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${script_dir}/cfg/${configFile} + fi + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]] && [[ ! $newHostname =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in "${arr[@]}"; do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}' | awk -F "/" '{print $1}') || : + if [ -z "$iplist" ]; then + iplist=$(ifconfig | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}' | awk -F ":" '{print $2}') || : + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + + if [ -f ${global_conf_dir}/${configFile} ]; then + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${global_conf_dir}/${configFile} + else + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${script_dir}/cfg/${configFile} + fi + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=$(echo $?) + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + if [ -f ${global_conf_dir}/${configFile} ]; then + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${global_conf_dir}/${configFile} + else + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${script_dir}/cfg/${configFile} + fi + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + set_hostname +} + +function install_anode_config() { + fileName="${script_dir}/cfg/${configFile}" + echo -e $fileName + + if [ -f ${fileName} ]; then + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$serverFqdn/" ${script_dir}/cfg/${configFile} + + if [ -f "${global_conf_dir}/${configFile}" ]; then + ${csudo}cp ${fileName} ${global_conf_dir}/${configFile}.new + else + ${csudo}cp ${fileName} ${global_conf_dir}/${configFile} + fi + fi + + ${csudo}ln -sf ${global_conf_dir}/${configFile} ${install_main_dir}/cfg +} + +function install_config() { + + [ ! -z $1 ] && return 0 || : # only install client + + if ((${update_flag} == 1)); then + install_taosd_config + return 0 + fi + + if [ "$interactiveFqdn" == "no" ]; then + install_taosd_config + return 0 + fi + + local_fqdn_check + install_anode_config + + echo + echo -e -n "${GREEN}Enter FQDN:port (like h1.${emailName}:6030) of an existing ${productName} cluster node to join${NC}" + echo + echo -e -n "${GREEN}OR leave it blank to build one${NC}:" + read firstEp + while true; do + if [ ! -z "$firstEp" ]; then + if [ -f ${global_conf_dir}/${configFile} ]; then + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${global_conf_dir}/${configFile} + else + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${script_dir}/cfg/${configFile} + fi + break + else + break + fi + done + + echo + echo -e -n "${GREEN}Enter your email address for priority support or enter empty to skip${NC}: " + read emailAddr + while true; do + if [ ! -z "$emailAddr" ]; then + email_file="${install_main_dir}/email" + ${csudo}bash -c "echo $emailAddr > ${email_file}" + break + else + break + fi + done +} + +function install_log() { + ${csudo}mkdir -p ${logDir} && ${csudo}chmod 777 ${logDir} + ${csudo}ln -sf ${logDir} ${install_main_dir}/log +} + +function install_module() { + ${csudo}mkdir -p ${moduleDir} && ${csudo}chmod 777 ${moduleDir} + ${csudo}ln -sf ${moduleDir} ${install_main_dir}/model +} + +function install_anode_venv() { + ${csudo}mkdir -p ${venvDir} && ${csudo}chmod 777 ${venvDir} + ${csudo}ln -sf ${venvDir} ${install_main_dir}/venv + + # build venv + ${csudo}python3.${python_minor_ver} -m venv ${venvDir} + + echo -e "active Python3 virtual env: ${venvDir}" + source ${venvDir}/bin/activate + + echo -e "install the required packages by pip3, this may take a while depending on the network condition" + ${csudo}${venvDir}/bin/pip3 install numpy==1.26.4 + ${csudo}${venvDir}/bin/pip3 install pandas==1.5.0 + + ${csudo}${venvDir}/bin/pip3 install scikit-learn + ${csudo}${venvDir}/bin/pip3 install outlier_utils + ${csudo}${venvDir}/bin/pip3 install statsmodels + ${csudo}${venvDir}/bin/pip3 install pyculiarity + ${csudo}${venvDir}/bin/pip3 install pmdarima + ${csudo}${venvDir}/bin/pip3 install flask + ${csudo}${venvDir}/bin/pip3 install matplotlib + ${csudo}${venvDir}/bin/pip3 install uwsgi + ${csudo}${venvDir}/bin/pip3 install torch --index-url https://download.pytorch.org/whl/cpu + ${csudo}${venvDir}/bin/pip3 install --upgrade keras + + echo -e "Install python library for venv completed!" +} + +function clean_service_on_sysvinit() { + if ps aux | grep -v grep | grep $1 &>/dev/null; then + ${csudo}service $1 stop || : + fi + + if ((${initd_mod} == 1)); then + if [ -e ${service_config_dir}/$1 ]; then + ${csudo}chkconfig --del $1 || : + fi + elif ((${initd_mod} == 2)); then + if [ -e ${service_config_dir}/$1 ]; then + ${csudo}insserv -r $1 || : + fi + elif ((${initd_mod} == 3)); then + if [ -e ${service_config_dir}/$1 ]; then + ${csudo}update-rc.d -f $1 remove || : + fi + fi + + ${csudo}rm -f ${service_config_dir}/$1 || : + + if $(which init &>/dev/null); then + ${csudo}init q || : + fi +} + +function install_service_on_sysvinit() { + if [ "$1" != "${serverName}" ]; then + return + fi + + clean_service_on_sysvinit $1 + sleep 1 + + if ((${os_type} == 1)); then + ${csudo}cp ${script_dir}/init.d/${serverName}.deb ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName} + elif ((${os_type} == 2)); then + ${csudo}cp ${script_dir}/init.d/${serverName}.rpm ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName} + fi + + if ((${initd_mod} == 1)); then + ${csudo}chkconfig --add $1 || : + ${csudo}chkconfig --level 2345 $1 on || : + elif ((${initd_mod} == 2)); then + ${csudo}insserv $1} || : + ${csudo}insserv -d $1 || : + elif ((${initd_mod} == 3)); then + ${csudo}update-rc.d $1 defaults || : + fi +} + +function clean_service_on_systemd() { + service_config="${service_config_dir}/$1.service" + + if systemctl is-active --quiet $1; then + echo "$1 is running, stopping it..." + ${csudo}systemctl stop $1 &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable $1 &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${service_config} +} + +function install_service_on_systemd() { + clean_service_on_systemd $1 + + cfg_source_dir=${script_dir}/cfg + if [[ "$1" == "${xname}" || "$1" == "${explorerName}" ]]; then + cfg_source_dir=${script_dir}/cfg + fi + + if [ -f ${cfg_source_dir}/$1.service ]; then + ${csudo}cp ${cfg_source_dir}/$1.service ${service_config_dir}/ || : + fi + + ${csudo}systemctl enable $1 + ${csudo}systemctl daemon-reload +} + +function install_service() { + if ((${service_mod} == 0)); then + install_service_on_systemd $1 + elif ((${service_mod} == 1)); then + install_service_on_sysvinit $1 + else + kill_process $1 + fi +} + +vercomp() { + if [[ $1 == $2 ]]; then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do + ver1[i]=0 + done + + for ((i = 0; i < ${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})); then + return 2 + fi + done + return 0 +} + +function is_version_compatible() { + + curr_version=$(ls ${script_dir}/driver/libtaos.so* | awk -F 'libtaos.so.' '{print $2}') + + if [ -f ${script_dir}/driver/vercomp.txt ]; then + min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt) + else + min_compatible_version=$(${script_dir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 5) + fi + + exist_version=$(${installDir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 3) + vercomp $exist_version "3.0.0.0" + case $? in + 2) + prompt_force=1 + ;; + esac + + vercomp $curr_version $min_compatible_version + echo "" # avoid $? value not update + + case $? in + 0) return 0 ;; + 1) return 0 ;; + 2) return 1 ;; + esac +} + +deb_erase() { + confirm="" + while [ "" == "${confirm}" ]; do + echo -e -n "${RED}Existing TDengine deb is detected, do you want to remove it? [yes|no] ${NC}:" + read confirm + if [ "yes" == "$confirm" ]; then + ${csudo}dpkg --remove tdengine || : + break + elif [ "no" == "$confirm" ]; then + break + fi + done +} + +rpm_erase() { + confirm="" + while [ "" == "${confirm}" ]; do + echo -e -n "${RED}Existing TDengine rpm is detected, do you want to remove it? [yes|no] ${NC}:" + read confirm + if [ "yes" == "$confirm" ]; then + ${csudo}rpm -e tdengine || : + break + elif [ "no" == "$confirm" ]; then + break + fi + done +} + +function updateProduct() { + # Check if version compatible + if ! is_version_compatible; then + echo -e "${RED}Version incompatible${NC}" + return 1 + fi + + # Start to update + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + + if echo $osinfo | grep -qwi "centos"; then + rpm -q tdengine 2>&1 >/dev/null && rpm_erase tdengine || : + elif echo $osinfo | grep -qwi "ubuntu"; then + dpkg -l tdengine 2>&1 | grep ii >/dev/null && deb_erase tdengine || : + fi + + tar -zxf ${tarName} + + echo "Start to update ${productName}..." + # Stop the service if running + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then + if ((${service_mod} == 0)); then + ${csudo}systemctl stop ${serverName} || : + elif ((${service_mod} == 1)); then + ${csudo}service ${serverName} stop || : + else + kill_process ${serverName} + fi + sleep 1 + fi + + install_main_path + install_log + install_module + install_config + + if [ -z $1 ]; then + install_bin + install_services + + echo + echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${global_conf_dir}/${configFile}" + [ -f ${global_conf_dir}/${adapterName}.toml ] && [ -f ${installDir}/bin/${adapterName} ] && + echo -e "${GREEN_DARK}To configure ${adapterName} ${NC}\t: edit ${global_conf_dir}/${adapterName}.toml" + echo -e "${GREEN_DARK}To configure ${explorerName} ${NC}\t: edit ${global_conf_dir}/explorer.toml" + if ((${service_mod} == 0)); then + echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}systemctl start ${serverName}${NC}" + elif ((${service_mod} == 1)); then + echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}service ${serverName} start${NC}" + else + echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ./${serverName}${NC}" + fi + + echo + echo "${productName} is updated successfully!" + echo + + else + install_bin + fi + + cd $script_dir + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") +} + +function installProduct() { + # Start to install + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + + tar -zxf ${tarName} + + echo "Start to install ${productName}..." + + install_main_path + install_log + install_anode_config + install_module + + install_bin_and_lib + install_services + + echo + echo -e "\033[44;32;1m${productName} is installed successfully!${NC}" + + echo + echo -e "\033[44;32;1mStart to create virtual python env in ${venvDir}${NC}" + install_anode_venv +} + +# check for python version, only the 3.10/3.11 is supported +check_python3_env() { + if ! command -v python3 &> /dev/null + then + echo -e "\033[31mWarning: Python3 command not found. Version 3.10/3.11 is required.\033[0m" + exit 1 + fi + + python3_version=$(python3 --version 2>&1 | awk -F' ' '{print $2}') + + python3_version_ok=false + python_minor_ver=$(echo "$python3_version" | cut -d"." -f2) + if [[ $(echo "$python3_version" | cut -d"." -f1) -eq 3 && $(echo "$python3_version" | cut -d"." -f2) -ge 10 ]]; then + python3_version_ok=true + fi + + if $python3_version_ok; then + echo -e "\033[32mPython3 ${python3_version} has been found.\033[0m" + else + if command -v python3.10 &> /dev/null + then + echo -e "\033[32mPython3.10 has been found.\033[0m" + python_minor_ver=10 + elif command -v python3.11 &> /dev/null + then + python_minor_ver=11 + echo -e "\033[32mPython3.11 has been found.\033[0m" + else + echo -e "\033[31mWarning: Python3.10/3.11 is required, only found python${python3_version}.\033[0m" + exit 1 + fi + fi + +# echo -e "Python3 minor version is:${python_minor_ver}" + + # check the existence pip3.10/pip3.11 + if ! command -v pip3 &> /dev/null + then + echo -e "\033[31mWarning: Pip3 command not found. Version 3.10/3.11 is required.\033[0m" + exit 1 + fi + + pip3_version=$(pip3 --version 2>&1 | awk -F' ' '{print $6}' | cut -d")" -f1) + major_ver=$(echo "${pip3_version}" | cut -d"." -f1) + minor_ver=$(echo "${pip3_version}" | cut -d"." -f2) + + pip3_version_ok=false; + if [[ ${major_ver} -eq 3 && ${minor_ver} -ge 10 ]]; then + pip3_version_ok=true + fi + + if $pip3_version_ok; then + echo -e "\033[32mpip3 ${pip3_version} has been found.\033[0m" + else + if command -v pip3.${python_minor_ver} &> /dev/null + then + echo -e "\033[32mpip3.${python_minor_ver} has been found.\033[0m" + else + echo -e "\033[31mWarning: pip3.10/3.11 is required, only found pip${pip3_version}.\033[0m" + exit 1 + fi + fi + +# if ! command -v python3.${python_minor_ver}-venv &> /dev/null +# then +# echo -e "\033[31mWarning: python3.${python_minor_ver}-venv command not found.\033[0m" +# exit 1 +# fi +} + +## ==============================Main program starts from here============================ +serverFqdn=$(hostname) + +if [ "$verType" == "server" ]; then + check_python3_env + installProduct +fi diff --git a/tools/tdgpt/script/release.sh b/tools/tdgpt/script/release.sh new file mode 100755 index 0000000000..c143357eb1 --- /dev/null +++ b/tools/tdgpt/script/release.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# Generate install package for all os system + +set -e +# set -x + +curr_dir=$(pwd) +compile_dir=$1 +version="1.0.1" +osType= +pagMode= +productName="TDengine-enterprise-anode" + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/..)" + +echo -e ${top_dir} + +serverName="taosanoded" +configFile="taosanode.ini" +tarName="package.tar.gz" + +# create compressed install file. +build_dir="${compile_dir}/build" +release_dir="${top_dir}/release" + +#package_name='linux' +install_dir="${release_dir}/${productName}-${version}" + +if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/${serverName} +fi + +cfg_dir="${top_dir}/cfg" +install_files="${script_dir}/install.sh" + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} + +if [ -f "${cfg_dir}/${serverName}.service" ]; then + cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || : +fi + +# python files +mkdir -p ${install_dir}/bin && mkdir -p ${install_dir}/lib + +# script to control start/stop/uninstall process +rm -r ${top_dir}/taosanalytics/*.pyc || : +cp -r ${top_dir}/taosanalytics/ ${install_dir}/lib/ && chmod a+x ${install_dir}/lib/ || : +cp -r ${top_dir}/script/st*.sh ${install_dir}/bin/ && chmod a+x ${install_dir}/bin/* || : +cp -r ${top_dir}/script/uninstall.sh ${install_dir}/bin/ && chmod a+x ${install_dir}/bin/* || : + +cd ${install_dir} + +#if [ "$osType" != "Darwin" ]; then +# tar -zcv -f ${tarName} ./bin/* || : +# rm -rf ${install_dir}/bin || : +#else +tar -zcv -f ${tarName} ./lib/* || : + +if [ ! -z "${install_dir}" ]; then + # shellcheck disable=SC2115 + rm -rf "${install_dir}"/lib || : +fi + +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${tarName} error !!!" + exit $exitcode +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} + +chmod a+x ${install_dir}/install.sh + +# Copy release note +# cp ${script_dir}/release_note ${install_dir} + +# exit 1 +cd ${release_dir} + +pkg_name=${install_dir} +echo -e "pkg_name is: ${pkg_name}" + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f "$(basename ${pkg_name}).tar.gz" "$(basename ${install_dir})" --remove-files || : +else + tar -zcv -f "$(basename ${pkg_name}).tar.gz" "$(basename ${install_dir})" || : + rm -rf "${install_dir}" ||: +fi + +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/tools/tdgpt/script/start.sh b/tools/tdgpt/script/start.sh new file mode 100755 index 0000000000..144ee08392 --- /dev/null +++ b/tools/tdgpt/script/start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# start the flask service by using uwsgi +/usr/local/taos/taosanode/venv/bin/uwsgi /usr/local/taos/taosanode/cfg/taosanode.ini \ No newline at end of file diff --git a/tools/tdgpt/script/stop.sh b/tools/tdgpt/script/stop.sh new file mode 100755 index 0000000000..c6f81d05cc --- /dev/null +++ b/tools/tdgpt/script/stop.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# stop the uwsgi server +/usr/local/taos/taosanode/venv/bin/uwsgi --stop /usr/local/taos/taosanode/taosanode.pid \ No newline at end of file diff --git a/tools/tdgpt/script/uninstall.sh b/tools/tdgpt/script/uninstall.sh new file mode 100755 index 0000000000..29c62f9782 --- /dev/null +++ b/tools/tdgpt/script/uninstall.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# uninstall the deployed app info, not remove the python virtual environment + +set -e +#set -x + +osType=`uname` + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +MAIN_NAME="taosanode" +installDir="/usr/local/taos/taosanode" +venv_dir="/usr/local/taos/taosanode/venv" +serverName="${MAIN_NAME}d" +uninstallName="rmtaosanode" +productName="TDengine Enterprise ANode" + +if [ "$osType" != "Darwin" ]; then + bin_link_dir="/usr/bin" +else + bin_link_dir="/usr/local/bin" +fi + +#install main path +bin_dir=${installDir}/bin +lib_dir=${installDir}/lib +local_log_dir=${installDir}/log +local_conf_dir=${installDir}/cfg +local_model_dir=${installDir}/model + +global_log_dir="/var/log/taos/${MAIN_NAME}" +global_conf_dir="/etc/taos/" + +service_config_dir="/etc/systemd/system" + +services=(${serverName} ${uninstallName}) + +csudo="" +if command -v sudo >/dev/null; then + csudo="sudo " +fi + +initd_mod=0 +service_mod=2 + +if ps aux | grep -v grep | grep systemd &>/dev/null; then + service_mod=0 +elif $(which service &>/dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &>/dev/null); then + initd_mod=1 + elif $(which insserv &>/dev/null); then + initd_mod=2 + elif $(which update-rc.d &>/dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +kill_service_of() { + _service=$1 + pid=$(ps -ef | grep $_service | grep -v grep | awk '{print $2}') + if [ -n "$pid" ]; then + "${csudo}"${installDir}/bin/stop.sh: + fi +} + +clean_service_on_systemd_of() { + _service=$1 + _service_config="${service_config_dir}/${_service}.service" + if systemctl is-active --quiet ${_service}; then + echo "${_service} is running, stopping it..." + "${csudo}"systemctl stop ${_service} &>/dev/null || echo &>/dev/null + fi + + "${csudo}"systemctl disable ${_service} &>/dev/null || echo &>/dev/null + + if [[ ! -z "${_service_config}" && -f "${_service_config}" ]]; then + ${csudo}rm ${_service_config} + fi +} + +clean_service_on_sysvinit_of() { + _service=$1 + if pidof ${_service} &>/dev/null; then + echo "${_service} is running, stopping it..." + "${csudo}"service ${_service} stop || : + fi + if ((${initd_mod} == 1)); then + if [ -e ${service_config_dir}/${_service} ]; then + # shellcheck disable=SC2086 + ${csudo}chkconfig --del ${_service} || : + fi + elif ((${initd_mod} == 2)); then + if [ -e ${service_config_dir}/${_service} ]; then + ${csudo}insserv -r ${_service} || : + fi + elif ((${initd_mod} == 3)); then + if [ -e ${service_config_dir}/${_service} ]; then + ${csudo}update-rc.d -f ${_service} remove || : + fi + fi + + # shellcheck disable=SC2236 + if [ ! -z "${service_config_dir}" ]; then + echo -e "rm ${service_config_dir}/${_service}" + fi + + #${csudo}rm ${service_config_dir}/${_service} || : + + if $(which init &>/dev/null); then + ${csudo}init q || : + fi +} + +clean_service_of() { + if ((${service_mod} == 0)); then + clean_service_on_systemd_of $_service + elif ((${service_mod} == 1)); then + clean_service_on_sysvinit_of $_service + else + kill_service_of $_service + fi +} + +remove_service_of() { + _service=$1 + clean_service_of ${_service} + if [[ -e "${bin_link_dir}/${_service}" ]]; then + ${csudo}rm "${bin_link_dir}"/${_service} + echo "${_service} is removed successfully!" + fi +} + +remove_service() { + for _service in "${services[@]}"; do + remove_service_of "${_service}" + done +} + +function clean_venv() { + # Remove python virtual environment + #${csudo}rm ${venv_dir}/* || : + if [ ! -z "${venv_dir}" ]; then + echo -e "${csudo}rm -rf ${venv_dir}/*" + fi +} + +function clean_module() { + if [ ! -z "${local_model_dir}" ]; then + ${csudo}unlink ${local_model_dir} || : + fi +} + +function clean_config() { + # Remove config file + if [ ! -z "${global_conf_dir}" ]; then + ${csudo}rm -f ${global_conf_dir}/taosanode.ini || : + fi + + if [ ! -z "${local_conf_dir}" ]; then + # shellcheck disable=SC2086 + ${csudo}rm -rf ${local_conf_dir} || : + fi +} + +function clean_log() { + # Remove log files + if [ ! -z "${global_log_dir}" ]; then + ${csudo}rm -rf ${global_log_dir} || : + fi + + if [ ! -z "${local_log_dir}" ]; then + ${csudo}rm -rf ${local_log_dir} || : + fi +} + +function remove_deploy_binary() { + if [ ! -z "${bin_dir}" ]; then + ${csudo}rm -rf ${bin_dir} || : + fi + + if [ ! -z "${lib_dir}" ]; then + ${csudo}rm -rf ${lib_dir} + fi +} + +remove_service +clean_log # Remove link log directory +clean_config # Remove link configuration file +remove_deploy_binary +clean_venv + +if [[ -e /etc/os-release ]]; then + osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +else + osinfo="" +fi + +if echo $osinfo | grep -qwi "ubuntu"; then + # echo "this is ubuntu system" + ${csudo}dpkg --force-all -P tdengine >/dev/null 2>&1 || : +elif echo $osinfo | grep -qwi "debian"; then + # echo "this is debian system" + ${csudo}dpkg --force-all -P tdengine >/dev/null 2>&1 || : +elif echo $osinfo | grep -qwi "centos"; then + # echo "this is centos system" + ${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || : +fi + +command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true +echo +echo "${productName} is uninstalled successfully!" +echo diff --git a/tools/tdgpt/taosanalytics/__init__.py b/tools/tdgpt/taosanalytics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/tdgpt/taosanalytics/algo/__init__.py b/tools/tdgpt/taosanalytics/algo/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/tdgpt/taosanalytics/algo/ad/__init__.py b/tools/tdgpt/taosanalytics/algo/ad/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/tdgpt/taosanalytics/algo/ad/autoencoder.py b/tools/tdgpt/taosanalytics/algo/ad/autoencoder.py new file mode 100644 index 0000000000..e58db3f54b --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/ad/autoencoder.py @@ -0,0 +1,117 @@ +# encoding:utf-8 +# pylint: disable=c0103 +""" auto encoder algorithms to detect anomaly for time series data""" +import os.path + +import joblib +import numpy as np +import pandas as pd + +from taosanalytics.conf import app_logger, conf +from taosanalytics.service import AbstractAnomalyDetectionService +from taosanalytics.util import create_sequences + + +class _AutoEncoderDetectionService(AbstractAnomalyDetectionService): + name = 'ad_encoder' + desc = "anomaly detection based on auto encoder" + + def __init__(self): + super().__init__() + + self.table_name = None + self.mean = None + self.std = None + self.threshold = None + self.time_interval = None + self.model = None + self.dir = 'ad_autoencoder' + + self.root_path = conf.get_model_directory() + + self.root_path = self.root_path + f'/{self.dir}/' + + if not os.path.exists(self.root_path): + app_logger.log_inst.error( + "%s ad algorithm failed to locate default module directory:" + "%s, not active", self.__class__.__name__, self.root_path) + else: + app_logger.log_inst.info("%s ad algorithm root path is: %s", self.__class__.__name__, + self.root_path) + + def execute(self): + if self.input_is_empty(): + return [] + + if self.model is None: + raise FileNotFoundError("not load autoencoder model yet, or load model failed") + + array_2d = np.reshape(self.list, (len(self.list), 1)) + df = pd.DataFrame(array_2d) + + # normalize input data using z-score + normalized_list = (df - self.mean.value) / self.std.value + seq = create_sequences(normalized_list.values, self.time_interval) + + # Get test MAE loss. + pred_list = self.model.predict(seq) + mae_loss = np.mean(np.abs(pred_list - seq), axis=1) + mae = mae_loss.reshape((-1)) + + # Detect all the samples which are anomalies. + anomalies = mae > self.threshold + + # syslogger.log_inst( + # "Number of anomaly samples: %f, Indices of anomaly samples:{}". + # format(np.sum(anomalies), np.where(anomalies)) + # ) + + # data i is an anomaly if samples [(i - timesteps + 1) to (i)] are anomalies + ad_indices = [] + for data_idx in range(self.time_interval - 1, + len(normalized_list) - self.time_interval + 1): + if np.all(anomalies[data_idx - self.time_interval + 1: data_idx]): + ad_indices.append(data_idx) + + return [-1 if i in ad_indices else 1 for i in range(len(self.list))] + + def set_params(self, params): + + if "model" not in params: + raise ValueError("model needs to be specified") + + name = params['model'] + + module_file_path = f'{self.root_path}/{name}.dat' + module_info_path = f'{self.root_path}/{name}.info' + + app_logger.log_inst.info("try to load module:%s", module_file_path) + + if os.path.exists(module_file_path): + self.model = joblib.load(module_file_path) + else: + app_logger.log_inst.error("failed to load autoencoder model file: %s", module_file_path) + raise FileNotFoundError(f"{module_file_path} not found") + + if os.path.exists(module_info_path): + info = joblib.load(module_info_path) + else: + app_logger.log_inst.error("failed to load autoencoder model file: %s", module_file_path) + raise FileNotFoundError("%s not found", module_info_path) + + if info is not None: + self.mean = info["mean"] + self.std = info["std"] + self.threshold = info["threshold"] + self.time_interval = info["timesteps"] + + app_logger.log_inst.info( + "load ac module success, mean: %f, std: %f, threshold: %f, time_interval: %d", + self.mean[0], self.std[0], self.threshold, self.time_interval + ) + else: + app_logger.log_inst.error("failed to load %s model", name) + raise RuntimeError(f"failed to load model {name}") + + def get_params(self): + return {"dir": self.dir + '/*'} diff --git a/tools/tdgpt/taosanalytics/algo/ad/grubbs.py b/tools/tdgpt/taosanalytics/algo/ad/grubbs.py new file mode 100644 index 0000000000..6318c109f5 --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/ad/grubbs.py @@ -0,0 +1,42 @@ +# encoding:utf-8 +""" grubbs algorithm class""" + +from outliers import smirnov_grubbs as grubbs +from taosanalytics.service import AbstractAnomalyDetectionService + + +class _GrubbsService(AbstractAnomalyDetectionService): + """ Grubbs algorithm is to check the anomaly data in the input list """ + name = 'grubbs' + desc = """Grubbs' test is to detect the presence of one outlier in a data set that is normally + distributed""" + + def __init__(self, alpha_val=0.95): + super().__init__() + + if alpha_val <= 0 or alpha_val >= 1: + raise ValueError("invalid alpha value, valid range is (0, 1)") + self.alpha = 1 - alpha_val + + def execute(self): + """perform Grubbs' test and identify (if any) the outlier""" + if self.input_is_empty(): + return [] + + res = grubbs.test(self.list, alpha=self.alpha) + + error_indicator = [1 if k in set(res) else -1 for k in self.list] + return error_indicator + + def set_params(self, params): + """ set the value of alpha """ + super().set_params(params) + + if "alpha".lower() in params: + # raise ValueError("alpha parameter is missing for grubbs algorithm") + alpha_val = float(params["alpha"]) + + if alpha_val <= 0 or alpha_val >= 1: + raise ValueError("invalid alpha value, valid range is (0, 1)") + + self.alpha = 1 - alpha_val diff --git a/tools/tdgpt/taosanalytics/algo/ad/iqr.py b/tools/tdgpt/taosanalytics/algo/ad/iqr.py new file mode 100644 index 0000000000..c0918a5090 --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/ad/iqr.py @@ -0,0 +1,29 @@ +# encoding:utf-8 +"""iqr class definition""" +import numpy as np +from taosanalytics.service import AbstractAnomalyDetectionService + + +class _IqrService(AbstractAnomalyDetectionService): + """ IQR algorithm is to check the anomaly data in the input list """ + name = 'iqr' + desc = """found the anomaly data according to the inter-quartile range""" + + def __init__(self): + super().__init__() + + def execute(self): + if self.input_is_empty(): + return [] + + lower = np.quantile(self.list, 0.25) + upper = np.quantile(self.list, 0.75) + + min_val = lower - 1.5 * (upper - lower) + max_val = upper + 1.5 * (upper - lower) + + threshold = [min_val, max_val] + return [-1 if k < threshold[0] or k > threshold[1] else 1 for k in self.list] + + def set_params(self, params): + pass diff --git a/tools/tdgpt/taosanalytics/algo/ad/ksigma.py b/tools/tdgpt/taosanalytics/algo/ad/ksigma.py new file mode 100644 index 0000000000..9d872dd11a --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/ad/ksigma.py @@ -0,0 +1,47 @@ +# encoding:utf-8 +"""ksigma class definition""" + +import numpy as np +from taosanalytics.service import AbstractAnomalyDetectionService + + +class _KSigmaService(AbstractAnomalyDetectionService): + """ KSigma algorithm is to check the anomaly data in the input list """ + name = "ksigma" + desc = """the k-sigma algorithm (or 3σ rule) expresses a conventional heuristic that nearly all + values are taken to lie within k (usually three) standard deviations of the mean, and thus + it is empirically useful to treat 99.7% probability as near certainty""" + + def __init__(self, k_val=3): + super().__init__() + self.k_val = k_val + + def execute(self): + def get_k_sigma_range(vals, k_value): + """ Return the k-sigma value range """ + avg = np.mean(vals) + std = np.std(vals) + + upper = avg + k_value * std + lower = avg - k_value * std + return [float(lower), float(upper)] + + if self.input_is_empty(): + return [] + + threshold = get_k_sigma_range(self.list, self.k_val) + return [-1 if k < threshold[0] or k > threshold[1] else 1 for k in self.list] + + def set_params(self, params): + super().set_params(params) + + if "k" in params: + k = int(params["k"]) + + if k < 1 or k > 3: + raise ValueError("k value out of range, valid range [1, 3]") + + self.k_val = k + + def get_params(self): + return {"k": self.k_val} diff --git a/tools/tdgpt/taosanalytics/algo/ad/lof.py b/tools/tdgpt/taosanalytics/algo/ad/lof.py new file mode 100644 index 0000000000..9f7b8fdc04 --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/ad/lof.py @@ -0,0 +1,43 @@ +# encoding:utf-8 +"""local outlier factor class definition""" + +import numpy as np +import sklearn.neighbors as neighbor +from taosanalytics.service import AbstractAnomalyDetectionService + + +class _LofService(AbstractAnomalyDetectionService): + """ LOF(local outlier factor) algorithm is to check the anomaly data in the input list """ + name = 'lof' + desc = """Local Outlier Factor, Ref: M. M. Breunig, H. P. Kriegel, R. T. Ng, J. Sander. + LOF:Identifying Density-based Local Outliers. SIGMOD, 2000.""" + + def __init__(self, n_neighbors=10, algo="auto"): + super().__init__() + + self.neighbors = n_neighbors + self.algorithm = algo + + def execute(self): + """perform LOF(local outlier factor) test and identify (if any) the outlier""" + if self.input_is_empty(): + return [] + + checker = neighbor.LocalOutlierFactor(n_neighbors=self.neighbors, algorithm=self.algorithm) + + arr_2d = np.reshape(self.list, (len(self.list), 1)) + res = checker.fit_predict(arr_2d) + + print(f"The negative outlier factor is:{checker.negative_outlier_factor_}") + return res + + def set_params(self, params): + super().set_params(params) + + if "neighbors" in params: # todo check value range + self.neighbors = int(params["neighbors"]) + if "algorithm" in params: + self.algorithm = params["algorithm"] + + def get_params(self): + return {"neighbors": self.neighbors, "algorithm": self.algorithm} diff --git a/tools/tdgpt/taosanalytics/algo/ad/shesd.py b/tools/tdgpt/taosanalytics/algo/ad/shesd.py new file mode 100644 index 0000000000..e105743bb5 --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/ad/shesd.py @@ -0,0 +1,44 @@ +# encoding:utf-8 +"""shesd algorithm class definition""" + +from pandas import Series +from pyculiarity import detect_vec +from taosanalytics.service import AbstractAnomalyDetectionService + + +class _SHESDService(AbstractAnomalyDetectionService): + """ s-h-esd algorithm is to check the anomaly data in the input list """ + name = 'shesd' + desc = "" + + def __init__(self, n_period=0, direction="both", anoms=0.05): + super().__init__() + + self.period = n_period + self.direction = direction + self.max_anoms = anoms + + def execute(self): + """perform SHESD test and identify (if any) the outlier""" + if self.input_is_empty(): + return [] + + results = detect_vec(Series(self.list), max_anoms=self.max_anoms, direction=self.direction, + period=self.period) + + res_val = results['anoms']['anoms'] + + return [-1 if k in set(res_val) else 1 for k in self.list] + + def set_params(self, params): + super().set_params(params) + + if "period" in params: # todo check value range + self.period = int(params["period"]) + if "direction" in params: + self.direction = params["direction"] + if "max_anoms" in params: + self.max_anoms = float(params["max_anoms"]) + + def get_params(self): + return {"period": self.period, "direction": self.direction, "max_anoms": self.max_anoms} diff --git a/tools/tdgpt/taosanalytics/algo/anomaly.py b/tools/tdgpt/taosanalytics/algo/anomaly.py new file mode 100644 index 0000000000..5f04283c06 --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/anomaly.py @@ -0,0 +1,49 @@ +# encoding:utf-8 +# pylint: disable=c0103 +""" anomaly detection register/display functions """ + +from matplotlib import pyplot as plt +from taosanalytics.conf import app_logger, conf +from taosanalytics.servicemgmt import loader + + +def do_ad_check(input_list, ts_list, algo_name, params): + """ actual anomaly detection handler """ + s = loader.get_service(algo_name) + + if s is None: + s = loader.get_service("ksigma") + + if s is None: + raise ValueError(f"failed to load {algo_name} or ksigma analysis service") + + s.set_input_list(input_list, ts_list) + s.set_params(params) + + res = s.execute() + + n_error = abs(sum(filter(lambda x: x == -1, res))) + app_logger.log_inst.debug("There are %d in input, and %d anomaly points found: %s", + len(input_list), + n_error, + res) + + draw_ad_results(input_list, res, algo_name) + return res + + +def draw_ad_results(input_list, res, fig_name): + """ draw the detected anomaly points """ + + # not in debug, do not visualize the anomaly detection result + if not conf.get_draw_result_option(): + return + + plt.clf() + for index, val in enumerate(res): + if val != -1: + continue + plt.scatter(index, input_list[index], marker='o', color='r', alpha=0.5, s=100, zorder=3) + + plt.plot(input_list, label='sample') + plt.savefig(fig_name) diff --git a/tools/tdgpt/taosanalytics/algo/fc/__init__.py b/tools/tdgpt/taosanalytics/algo/fc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/tdgpt/taosanalytics/algo/fc/arima.py b/tools/tdgpt/taosanalytics/algo/fc/arima.py new file mode 100644 index 0000000000..9e087a5e9e --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/fc/arima.py @@ -0,0 +1,114 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""arima class definition""" +import pmdarima as pm + +from taosanalytics.algo.forecast import insert_ts_list +from taosanalytics.conf import app_logger +from taosanalytics.service import AbstractForecastService + + +class _ArimaService(AbstractForecastService): + """ ARIMA algorithm is to do the fc in the input list """ + name = "arima" + desc = "do time series data fc by using ARIMA model" + + def __init__(self): + super().__init__() + + self.diff = 0 + self.start_p = 0 + self.max_p = 10 + self.start_q = 0 + self.max_q = 10 + + def set_params(self, params): + super().set_params(params) + + self.start_p = int(params['start_p']) if 'start_p' in params else 0 + self.max_p = int(params['max_p']) if 'max_p' in params else 0 + self.start_q = int(params['start_q']) if 'start_q' in params else 0 + self.max_q = int(params['max_q']) if 'max_q' in params else 0 + + def get_params(self): + """ get the default value for fc algorithms """ + p = super().get_params() + p.update( + { + "start_p": self.start_p, "max_p": self.max_p, "start_q": self.start_q, + "max_q": self.max_q, "diff": self.diff + } + ) + + return p + + def __do_forecast_helper(self, fc_rows): + """ do arima fc """ + # plot_acf(self.list, lags=25, title='raw_acf') + # plot_pacf(self.list, lags=25, title='raw_pacf') + # plt.show() + + seasonal = self.period > 0 + + # Fit model + model = pm.auto_arima(self.list, + start_p=self.start_p, + start_q=self.start_q, + max_p=self.max_p, + max_q=self.max_q, + d=1, + m=self.period, + seasonal=seasonal, + start_P=0, + D=self.diff) + + app_logger.log_inst.debug(model.summary()) + + # predict N steps into the future + fc = model.predict(n_periods=fc_rows, return_conf_int=self.return_conf, + alpha=self.conf) + + # plt.plot(source_data, label='training') + # plt.plot(xrange, actual_data, label='actual') + + # fc_list = fc.tolist() + # fc_without_diff = restore_from_diff(self.list, fc_list, 2) + # print(fc_without_diff) + + # plt.plot(xrange, fc_without_diff, label='fc') + + # residuals = pd.DataFrame(model.arima_res_.resid) + # wn = is_white_noise(residuals) + # print("residual is white noise:", wn) + + # fig, ax = plt.subplots(1, 2) + # residuals.plot(title="Residuals", ax=ax[0]) + # residuals.plot(kind='kde', title='Density', ax=ax[1]) + # plt.show() + + res1 = [fc[0].tolist(), fc[1][:, 0].tolist(), + fc[1][:, 1].tolist()] if self.return_conf else [fc.tolist()] + + return ( + res1, + model.arima_res_.mse, + f"SARIMAX{model.order}x{model.seasonal_order}" + ) + + def execute(self): + """ do fc the time series data""" + + if self.list is None or len(self.list) < self.period: + raise ValueError("number of input data is less than the periods") + + if self.fc_rows <= 0: + raise ValueError("fc rows is not specified yet") + + res, mse, model_info = self.__do_forecast_helper(self.fc_rows) + insert_ts_list(res, self.start_ts, self.time_step, self.fc_rows) + + return { + "mse": mse, + "model_info": model_info, + "res": res + } diff --git a/tools/tdgpt/taosanalytics/algo/fc/holtwinters.py b/tools/tdgpt/taosanalytics/algo/fc/holtwinters.py new file mode 100644 index 0000000000..d8225eaa5a --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/fc/holtwinters.py @@ -0,0 +1,79 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""holt winters definition""" + +from statsmodels.tsa.holtwinters import ExponentialSmoothing, SimpleExpSmoothing + +from taosanalytics.algo.forecast import insert_ts_list +from taosanalytics.service import AbstractForecastService + + +class _HoltWintersService(AbstractForecastService): + """ Holt winters algorithm is to do the fc in the input list """ + name = "holtwinters" + desc = "forecast algorithm by using exponential smoothing" + + def __init__(self): + super().__init__() + + self.trend_option = None + self.seasonal_option = None + + def set_params(self, params): + super().set_params(params) + + self.trend_option = params['trend'] if 'trend' in params else None + + if self.trend_option is not None: + if self.trend_option not in ('add', 'mul'): + raise ValueError("trend parameter can only be 'mul' or 'add'") + + self.seasonal_option = params['seasonal'] if 'seasonal' in params else None + if self.seasonal_option is not None: + if self.seasonal_option not in ('add', 'mul'): + raise ValueError("seasonal parameter can only be 'mul' or 'add'") + + def get_params(self): + p = super().get_params() + p.update({'trend': self.trend_option, 'seasonal': self.seasonal_option}) + return p + + def __do_forecast_helper(self, source_data, fc_rows): + """ do holt winters impl """ + if self.trend_option is None: + fitted_model = SimpleExpSmoothing(source_data).fit() + else: + if self.period == 0 or self.seasonal_option is None: + # no valid seasonal periods, so not need to specify the seasonal parameters + fitted_model = ExponentialSmoothing(source_data, trend=self.trend_option).fit() + else: # seasonal attributes + fitted_model = ExponentialSmoothing( + source_data, + trend=self.trend_option, + seasonal=self.seasonal_option, + seasonal_periods=self.period + ).fit() + + fc = fitted_model.forecast(fc_rows) + + if self.return_conf: + return [fc.tolist(), fc.tolist(), fc.tolist()], fitted_model.sse + else: + return [fc.tolist()], fitted_model.sse + + def execute(self): + """ do fc the time series data""" + if self.list is None or len(self.list) < self.period: + raise ValueError("number of input data is less than the periods") + + if self.fc_rows <= 0: + raise ValueError("fc rows is not specified yet") + + res, mse = self.__do_forecast_helper(self.list, self.fc_rows) + insert_ts_list(res, self.start_ts, self.time_step, self.fc_rows) + + # add the conf range if required + return { + "mse": mse, + "res": res + } diff --git a/tools/tdgpt/taosanalytics/algo/forecast.py b/tools/tdgpt/taosanalytics/algo/forecast.py new file mode 100644 index 0000000000..e1e321a7b0 --- /dev/null +++ b/tools/tdgpt/taosanalytics/algo/forecast.py @@ -0,0 +1,110 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""forecast helper methods""" + +import numpy as np +import pandas as pd +from matplotlib import pyplot as plt + +from taosanalytics.conf import app_logger, conf +from taosanalytics.servicemgmt import loader + + +def do_forecast(input_list, ts_list, algo_name, params): + """ data fc handler """ + s = loader.get_service(algo_name) + + if s is None: + s = loader.get_service("holtwinters") + + if s is None: + raise ValueError(f"failed to load {algo_name} or holtwinters analysis service") + + s.set_input_list(input_list, ts_list) + s.set_params(params) + + app_logger.log_inst.debug("start to do forecast") + res = s.execute() + + app_logger.log_inst.debug("forecast done") + + res["period"] = s.period + res["algo"] = algo_name + + check_fc_results(res) + + fc = res["res"] + draw_fc_results(input_list, len(fc) > 2, fc, len(fc[0]), algo_name) + return res + + +def do_add_fc_params(params, json_obj): + """ add params into parameters """ + if "forecast_rows" in json_obj: + params["fc_rows"] = int(json_obj["forecast_rows"]) + + if "start" in json_obj: + params["start_ts"] = int(json_obj["start"]) + + if "every" in json_obj: + params["time_step"] = int(json_obj["every"]) + + if "conf" in json_obj: + params["conf"] = int(json_obj["conf"]) + + if "return_conf" in json_obj: + params["return_conf"] = int(json_obj["return_conf"]) + + +def insert_ts_list(res, start_ts, time_step, fc_rows): + """ insert the ts list before return results """ + ts_list = [start_ts + i * time_step for i in range(fc_rows)] + res.insert(0, ts_list) + return res + + +def draw_fc_results(input_list, return_conf, fc, n_rows, fig_name): + """Visualize the forecast results """ + # controlled by option, do not visualize the anomaly detection result + if not conf.get_draw_result_option(): + return + + app_logger.log_inst.debug('draw forecast result in debug model') + plt.clf() + + x = np.arange(len(input_list), len(input_list) + n_rows, 1) + + # draw the range of conf + if return_conf: + lower_series = pd.Series(fc[2], index=x) + upper_series = pd.Series(fc[3], index=x) + + plt.fill_between(lower_series.index, lower_series, upper_series, color='k', alpha=.15) + + plt.plot(input_list) + plt.plot(x, fc[1], c='blue') + plt.savefig(fig_name) + + app_logger.log_inst.debug("draw results completed in debug model") + + +def check_fc_results(res): + app_logger.log_inst.debug("start to check forecast result") + + if "res" not in res: + raise ValueError("forecast result is empty") + + fc = res["res"] + if len(fc) < 2: + raise ValueError("result length should greater than or equal to 2") + + n_rows = len(fc[0]) + if n_rows != len(fc[1]): + raise ValueError("result length is not identical, ts rows:%d res rows:%d" % ( + n_rows, len(fc[1]))) + + if len(fc) > 2 and (len(fc[2]) != n_rows or len(fc[3]) != n_rows): + raise ValueError( + "result length is not identical in confidence, ts rows:%d, lower confidence rows:%d, " + "upper confidence rows%d" % + (n_rows, len(fc[2]), len(fc[3]))) diff --git a/tools/tdgpt/taosanalytics/app.py b/tools/tdgpt/taosanalytics/app.py new file mode 100644 index 0000000000..7be41489e7 --- /dev/null +++ b/tools/tdgpt/taosanalytics/app.py @@ -0,0 +1,163 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""the main route definition for restful service""" +import os.path, sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../") + +from flask import Flask, request + +from taosanalytics.algo.anomaly import do_ad_check +from taosanalytics.algo.forecast import do_forecast, do_add_fc_params +from taosanalytics.conf import conf +from taosanalytics.model import get_avail_model +from taosanalytics.servicemgmt import loader +from taosanalytics.util import app_logger, validate_pay_load, get_data_index, get_ts_index, is_white_noise, \ + parse_options, convert_results_to_windows + +app = Flask(__name__) + +# load the all algos +app_logger.set_handler(conf.get_log_path()) +app_logger.set_log_level(conf.get_log_level()) +loader.load_all_service() + + +@app.route("/") +def start(): + """ default rsp """ + return "TDengine© Time Series Data Analytics Platform (ver 1.0.1)" + + +@app.route("/status") +def server_status(): + """ return server status """ + return { + 'protocol': 1.0, + 'status': 'ready' + } + + +@app.route("/list") +def list_all_services(): + """ + API function to return all available services, including both fc and anomaly detection + """ + return loader.get_service_list() + + +@app.route("/models") +def list_all_models(): + """ list all available models """ + return get_avail_model() + + +@app.route("/anomaly-detect", methods=['POST']) +def handle_ad_request(): + """handle the anomaly detection requests""" + app_logger.log_inst.info('recv ad request from %s', request.remote_addr) + app_logger.log_inst.debug('req payload: %s', request.json) + + algo = request.json["algo"].lower() if "algo" in request.json else "ksigma" + + # 1. validate the input data in json format + try: + validate_pay_load(request.json) + except ValueError as e: + return {"msg": str(e), "rows": -1} + + payload = request.json["data"] + + # 2. white noise data check + wn_check = request.json["wncheck"] if "wncheck" in request.json else 1 + + data_index = get_data_index(request.json["schema"]) + ts_index = get_ts_index(request.json["schema"]) + + if wn_check: + try: + data = payload[data_index] + if is_white_noise(data): + app_logger.log_inst.debug("wn data, not process") + return {"msg": "white noise can not be check", "rows": -1} + except Exception as e: + return {"msg": str(e), "rows": -1} + + # 3. parse the options for different ad services + # the default options is like following: "algo=ksigma,k=2,invalid_option=44" + options = request.json["option"] if "option" in request.json else None + params = parse_options(options) + + # 4. do anomaly detection + try: + res_list = do_ad_check(payload[data_index], payload[ts_index], algo, params) + ano_window = convert_results_to_windows(res_list, payload[ts_index]) + + result = {"algo": algo, "option": options, "res": ano_window, "rows": len(ano_window)} + app_logger.log_inst.debug("anomaly-detection result: %s", str(result)) + + return result + + except Exception as e: + result = {"res": {}, "rows": 0, "msg": str(e)} + app_logger.log_inst.error("failed to do anomaly-detection, %s", str(e)) + + return result + + +@app.route("/forecast", methods=['POST']) +def handle_forecast_req(): + """handle the fc request """ + app_logger.log_inst.info('recv fc from %s', request.remote_addr) + app_logger.log_inst.debug('req payload: %s', request.json) + + # holt-winters by default + algo = request.json['algo'].lower() if 'algo' in request.json else 'holtwinters' + + # 1. validate the input data in json format + try: + validate_pay_load(request.json) + except ValueError as e: + app_logger.log_inst.error('validate req json failed, %s', e) + return {"msg": str(e), "rows": -1} + + payload = request.json["data"] + + # 2. white noise data check + wn_check = request.json["wncheck"] if "wncheck" in request.json else 1 + data_index = get_data_index(request.json["schema"]) + ts_index = get_ts_index(request.json["schema"]) + + if wn_check: + try: + data = payload[data_index] + if is_white_noise(data): + app_logger.log_inst.debug("%s wn data, not process", data) + return {"msg": "white noise can not be check", "rows": -1} + except Exception as e: + return {"msg": str(e), "rows": -1} + + options = request.json["option"] if "option" in request.json else None + params = parse_options(options) + + try: + do_add_fc_params(params, request.json) + except ValueError as e: + app_logger.log_inst.error("invalid fc params: %s", e) + return {"msg": f"{e}", "rows": -1} + + try: + res1 = do_forecast(payload[data_index], payload[ts_index], algo, params) + res = {"option": options, "rows": params["fc_rows"]} + res.update(res1) + + app_logger.log_inst.debug("forecast result: %s", res) + + return res + except Exception as e: + app_logger.log_inst.error('forecast failed, %s', str(e)) + return {"msg": str(e), "rows": -1} + + +if __name__ == '__main__': + app.run(port=6090) diff --git a/tools/tdgpt/taosanalytics/conf.py b/tools/tdgpt/taosanalytics/conf.py new file mode 100644 index 0000000000..c255b8e258 --- /dev/null +++ b/tools/tdgpt/taosanalytics/conf.py @@ -0,0 +1,105 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""configuration model definition""" +import configparser +import logging + +_ANODE_SECTION_NAME = "taosanode" + + +class Configure: + """ configuration class """ + + def __init__(self, conf_path="/etc/taos/taosanode.ini"): + self.path = None + + self._log_path = 'taosanode.app.log' + self._log_level = logging.INFO + self._model_directory = '/var/lib/taos/taosanode/model/' + self._draw_result = 0 + + self.conf = configparser.ConfigParser() + self.reload(conf_path) + + def get_log_path(self) -> str: + """ return log file full path """ + return self._log_path + + def get_log_level(self): + """ return the log level specified by configuration file """ + return self._log_level + + def get_model_directory(self): + """ return model directory """ + return self._model_directory + + def get_draw_result_option(self): + """ get the option for draw results or not""" + return self._draw_result + + def reload(self, new_path: str): + """ load the info from config file """ + self.path = new_path + + self.conf.read(self.path) + + if self.conf.has_option(_ANODE_SECTION_NAME, 'app-log'): + self._log_path = self.conf.get(_ANODE_SECTION_NAME, 'app-log') + + if self.conf.has_option(_ANODE_SECTION_NAME, 'log-level'): + log_level = self.conf.get(_ANODE_SECTION_NAME, 'log-level') + + log_flag = { + 'DEBUG': logging.DEBUG, 'INFO': logging.INFO, 'CRITICAL': logging.CRITICAL, + 'ERROR': logging.ERROR, 'WARN': logging.WARN + } + + if log_level.upper() in log_flag: + self._log_level = log_flag[log_level.upper()] + else: + self._log_level = logging.INFO + + if self.conf.has_option(_ANODE_SECTION_NAME, 'model-dir'): + self._model_directory = self.conf.get(_ANODE_SECTION_NAME, 'model-dir') + + if self.conf.has_option(_ANODE_SECTION_NAME, 'draw-result'): + self._draw_result = self.conf.get(_ANODE_SECTION_NAME, 'draw-result') + + +class AppLogger(): + """ system log_inst class """ + LOG_STR_FORMAT = '%(asctime)s - %(threadName)s - %(levelname)s - %(message)s' + + def __init__(self): + self.log_inst = logging.getLogger(__name__) + self.log_inst.setLevel(logging.INFO) + + def set_handler(self, file_path: str): + """ set the log_inst handler """ + + handler = logging.FileHandler(file_path) + handler.setFormatter(logging.Formatter(self.LOG_STR_FORMAT)) + + self.log_inst.addHandler(handler) + + def set_log_level(self, log_level): + """adjust log level""" + try: + self.log_inst.setLevel(log_level) + self.log_inst.info("set log level:%d", log_level) + except ValueError as e: + self.log_inst.error("failed to set log level: %d, %s", log_level, str(e)) + + +conf = Configure() +app_logger = AppLogger() + + +def setup_log_info(name: str): + """ prepare the log info for unit test """ + app_logger.set_handler(name) + + try: + app_logger.set_log_level(logging.DEBUG) + except ValueError as e: + print("set log level failed:%s", e) diff --git a/tools/tdgpt/taosanalytics/misc/__init__.py b/tools/tdgpt/taosanalytics/misc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/tdgpt/taosanalytics/model.py b/tools/tdgpt/taosanalytics/model.py new file mode 100644 index 0000000000..6efd85544e --- /dev/null +++ b/tools/tdgpt/taosanalytics/model.py @@ -0,0 +1,22 @@ +# encoding:utf-8 +# pylint: disable=c0103 + +def get_avail_model(): + return [ + { + "name": "ad_encoder_keras", + "algo": "auto-encoder", + "type": "anomaly-detection", + "src-table": "*", + "build-time": "2024-10-07 13:21:44" + } + ] + + +def train_model(): + pass + + +if __name__ == '__main__': + a = get_avail_model() + print(a) diff --git a/tools/tdgpt/taosanalytics/service.py b/tools/tdgpt/taosanalytics/service.py new file mode 100644 index 0000000000..79244aae8c --- /dev/null +++ b/tools/tdgpt/taosanalytics/service.py @@ -0,0 +1,110 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""main service module""" +from abc import abstractmethod, ABC + + +class AnalyticsService: + """ Analytics root class with only one method""" + + @abstractmethod + def execute(self): + """ the main execute method to perform fc or anomaly detection """ + + def get_desc(self) -> str: + """algorithm description""" + return "" + + def get_params(self) -> dict: + """return exist params """ + return {} + + +class AbstractAnalyticsService(AnalyticsService, ABC): + """ abstract base analytics service class definition""" + name = '' + desc = '' + + def __init__(self): + self.list = None + self.ts_list = None + + def set_input_list(self, input_list: list, input_ts_list: list = None): + """ set the input list """ + self.list = input_list + self.ts_list = input_ts_list + + def set_params(self, params: dict) -> None: + """set the parameters for current algo """ + if params is None: + return + + if not isinstance(params, dict): + raise ValueError('invalid parameter type, only dict allowed') + + def get_desc(self) -> str: + return self.desc + + +class AbstractAnomalyDetectionService(AbstractAnalyticsService, ABC): + """ abstract anomaly detection service, all anomaly detection algorithm class should be + inherent from this class""" + + def __init__(self): + super().__init__() + self.type = "anomaly-detection" + + def input_is_empty(self): + """ check if the input list is empty or None """ + return (self.list is None) or (len(self.list) == 0) + + +class AbstractForecastService(AbstractAnalyticsService, ABC): + """abstract forecast service, all forecast algorithms class should be inherent from + this base class""" + + def __init__(self): + super().__init__() + self.type = "forecast" + + self.period = 0 + self.start_ts = 0 + self.time_step = 0 + self.fc_rows = 0 + + self.return_conf = 1 + self.conf = 0.05 + + def set_params(self, params: dict) -> None: + if not {'start_ts', 'time_step', 'fc_rows'}.issubset(params.keys()): + raise ValueError('params are missing, start_ts, time_step, fc_rows are all required') + + self.start_ts = int(params['start_ts']) + + self.time_step = int(params['time_step']) + + if self.time_step <= 0: + raise ValueError('time_step should be greater than 0') + + self.fc_rows = int(params['fc_rows']) + + if self.fc_rows <= 0: + raise ValueError('fc rows is not specified yet') + + self.period = int(params['period']) if 'period' in params else 0 + if self.period < 0: + raise ValueError("periods should be greater than 0") + + self.conf = float(params['conf']) if 'conf' in params else 95 + + self.conf = 1.0 - self.conf / 100.0 + if self.conf < 0 or self.conf >= 1.0: + raise ValueError("invalid value of conf, should between 0 and 100") + + self.return_conf = int(params['return_conf']) if 'return_conf' in params else 1 + + def get_params(self): + return { + "period": self.period, "start": self.start_ts, "every": self.time_step, + "forecast_rows": self.fc_rows, "return_conf": self.return_conf, "conf": self.conf + } diff --git a/tools/tdgpt/taosanalytics/servicemgmt.py b/tools/tdgpt/taosanalytics/servicemgmt.py new file mode 100644 index 0000000000..5b20c73249 --- /dev/null +++ b/tools/tdgpt/taosanalytics/servicemgmt.py @@ -0,0 +1,120 @@ +# encoding:utf-8 +"""load and return the available services""" +import copy +import importlib +import inspect +import os +from collections import defaultdict +from taosanalytics.conf import app_logger +from taosanalytics.service import AbstractAnomalyDetectionService, AbstractForecastService + +os.environ['KERAS_BACKEND'] = 'torch' + + +class AnalyticsServiceLoader: + """ Singleton register for multiple anomaly detection algorithms and fc algorithms""" + + def __init__(self): + self.services = defaultdict(list) + + def get_service(self, name): + """ get the required service """ + serv = self.services.get(name, [])[0] if self.services.get(name) else None + return copy.copy(serv) + + def get_typed_services(self, type_str: str) -> list: + """ get specified type service """ + all_items = [] + for key, val in self.services.items(): + if val[0].type == type_str: + try: + one = {"name": key, "desc": val[0].get_desc(), "params": val[0].get_params()} + all_items.append(one) + except AttributeError as e: + app_logger.log_inst.error("failed to get service: %s info, reason: %s", key, e); + + return all_items + + def get_service_list(self): + """ return all available service info """ + info = { + "protocol": 1.0, + "version": 0.1, + "details": [ + self.get_forecast_algo_list(), + self.get_anomaly_detection_algo_list() + ] + } + + return info + + def get_anomaly_detection_algo_list(self): + """ get all available service list """ + return { + "type": "anomaly-detection", + "algo": self.get_typed_services("anomaly-detection") + } + + def get_forecast_algo_list(self): + """ get all available service list """ + return { + "type": "forecast", + "algo": self.get_typed_services("forecast") + } + + def load_all_service(self) -> None: + """ load all algorithms in the specified directory""" + + def register_service(container, name: str, service): + """ register service for both anomaly detection and fc """ + app_logger.log_inst.info("register service: %s", name) + container[name].append(service) + + def do_load_service(cur_directory, lib_prefix, sub_directory): + """ the implementation of load services """ + service_directory = cur_directory + sub_directory + + if not os.path.exists(service_directory): + app_logger.log_inst.fatal( + "service directory:%s not lib exists, failed to load service", + service_directory) + raise FileNotFoundError(f"service directory:{service_directory} not found") + + all_files = os.listdir(service_directory) + + for item in all_files: + if item in ('__init__.py', '__pycache__') or not item.endswith('py'): + continue + + full_path = os.path.join(service_directory, item) + if os.path.isdir(full_path): + continue + + # do load algorithm + name = lib_prefix + item.split('.')[0] + module = importlib.import_module(name) + + app_logger.log_inst.info("load algorithm:%s", name) + + for (class_name, _) in inspect.getmembers(module, inspect.isclass): + + if class_name in ( + AbstractAnomalyDetectionService.__name__, + AbstractForecastService.__name__ + ) or (not class_name.startswith('_')): + continue + + algo_cls = getattr(module, class_name) + + if algo_cls is not None: + obj = algo_cls() + register_service(self.services, algo_cls.name, obj) + + # start to load all services + current_directory = os.path.dirname(os.path.abspath(__file__)) + + do_load_service(current_directory, 'taosanalytics.algo.ad.', '/algo/ad/') + do_load_service(current_directory, 'taosanalytics.algo.fc.', '/algo/fc/') + + +loader: AnalyticsServiceLoader = AnalyticsServiceLoader() diff --git a/tools/tdgpt/taosanalytics/test/__init__.py b/tools/tdgpt/taosanalytics/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/tdgpt/taosanalytics/test/anomaly_test.py b/tools/tdgpt/taosanalytics/test/anomaly_test.py new file mode 100644 index 0000000000..f44a7f0d52 --- /dev/null +++ b/tools/tdgpt/taosanalytics/test/anomaly_test.py @@ -0,0 +1,170 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""anomaly detection unit test""" +import unittest, sys, os.path +import pandas as pd + +sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../../") + +from taosanalytics.algo.anomaly import draw_ad_results +from taosanalytics.conf import setup_log_info, app_logger +from taosanalytics.servicemgmt import loader + + +class AnomalyDetectionTest(unittest.TestCase): + """ anomaly detection unit test class""" + input_list = [5, 14, 15, 15, 14, 19, 17, 16, 20, 22, 8, 21, 28, 11, 9, 29, 40] + large_list = [ + 13, 14, 8, 10, 16, 26, 32, 27, 18, 32, 36, 24, + 22, 23, 22, 18, 25, 21, 21, 14, 8, 11, 14, 23, + 18, 17, 19, 20, 22, 19, 13, 26, 13, 14, 22, 24, + 21, 22, 26, 21, 23, 24, 27, 41, 31, 27, 35, 26, + 28, 36, 39, 21, 17, 22, 17, 19, 15, 34, 10, 15, + 22, 18, 15, 20, 15, 22, 19, 16, 30, 27, 29, 23, + 20, 16, 21, 21, 25, 16, 18, 15, 18, 14, 10, 15, + 8, 15, 6, 11, 8, 7, 13, 10, 23, 16, 15, 25, + 22, 20, 16 + ] + + @classmethod + def setUpClass(cls): + """ set up environment for unit test, set the log file path """ + setup_log_info("unit_test.log") + loader.load_all_service() + + def test_ksigma(self): + """ + Test the ksigma algorithm for anomaly detection. This test case verifies the + functionality of the ksigma algorithm by setting up the input data, + executing the algorithm, and asserting the expected results. + """ + + s = loader.get_service("ksigma") + s.set_input_list(AnomalyDetectionTest.input_list, None) + s.set_params({"k": 2}) + + r = s.execute() + draw_ad_results(AnomalyDetectionTest.input_list, r, "ksigma") + + self.assertEqual(r[-1], -1) + self.assertEqual(len(r), len(AnomalyDetectionTest.input_list)) + + def test_iqr(self): + """ + Test the IQR(Interquartile Range) algorithm for anomaly detection. This test case verifies the functionality + of the IQR algorithm by setting up the input data, executing the algorithm, and asserting the expected results. + """ + + s = loader.get_service("iqr") + s.set_input_list(AnomalyDetectionTest.input_list, None) + + try: + s.set_params({"k": 2}) + except ValueError as e: + self.assertEqual(1, 0, e) + + r = s.execute() + draw_ad_results(AnomalyDetectionTest.input_list, r, "iqr") + + self.assertEqual(r[-1], -1) + self.assertEqual(len(r), len(AnomalyDetectionTest.input_list)) + + def test_grubbs(self): + """ + Test the Grubbs algorithm for anomaly detection. + + This test case verifies the functionality of the Grubbs algorithm by setting up the input data, + executing the algorithm, and asserting the expected results. + """ + + s = loader.get_service("grubbs") + s.set_input_list(AnomalyDetectionTest.input_list, None) + s.set_params({"alpha": 0.95}) + + r = s.execute() + draw_ad_results(AnomalyDetectionTest.input_list, r, "grubbs") + + self.assertEqual(r[-1], -1) + self.assertEqual(len(r), len(AnomalyDetectionTest.input_list)) + + def test_shesd(self): + """ + Test the SHESD (Seasonal Hybrid ESD) algorithm for anomaly detection. + + This test case verifies the functionality of the SHESD algorithm by setting up the input data, + executing the algorithm, and asserting the expected results. + """ + + s = loader.get_service("shesd") + s.set_params({"period": 3}) + s.set_input_list(AnomalyDetectionTest.input_list, None) + + r = s.execute() + draw_ad_results(AnomalyDetectionTest.input_list, r, "shesd") + + self.assertEqual(r[-1], -1) + + def test_lof(self): + """ + Test the LOF (Local Outlier Factor) algorithm for anomaly detection. + + This test case verifies the functionality of the LOF algorithm by setting up the input data, + executing the algorithm, and asserting the expected results. + """ + s = loader.get_service("lof") + s.set_params({"period": 3}) + s.set_input_list(AnomalyDetectionTest.input_list, None) + + r = s.execute() + draw_ad_results(AnomalyDetectionTest.input_list, r, "lof") + + self.assertEqual(r[-1], -1) + self.assertEqual(r[-2], -1) + + def test_multithread_safe(self): + """ Test the multithread safe function""" + s1 = loader.get_service("shesd") + s2 = loader.get_service("shesd") + + s1.set_params({"period": 3}) + self.assertNotEqual(s1.period, s2.period) + + def __load_remote_data_for_ad(self): + """load the remote data for anomaly detection""" + + url = ("https://raw.githubusercontent.com/numenta/NAB/master/data/artificialWithAnomaly/" + "art_daily_jumpsup.csv") + + remote_data = pd.read_csv(url, parse_dates=True, index_col="timestamp") + k = remote_data.values.ravel().tolist() + return k + + def test_autoencoder_ad(self): + """for local test only, disabled it in github action""" + pass + + # data = self.__load_remote_data_for_ad() + # + # s = loader.get_service("ad_encoder") + # s.set_input_list(data) + # + # try: + # s.set_params({"model": "ad_encoder_"}) + # except ValueError as e: + # app_logger.log_inst.error(f"failed to set the param for auto_encoder algorithm, reason:{e}") + # return + # + # r = s.execute() + # + # num_of_error = -(sum(filter(lambda x: x == -1, r))) + # self.assertEqual(num_of_error, 109) + # + # draw_ad_results(data, r, "autoencoder") + + def test_get_all_services(self): + """Test get all services""" + loader.get_anomaly_detection_algo_list() + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/tdgpt/taosanalytics/test/forecast_test.py b/tools/tdgpt/taosanalytics/test/forecast_test.py new file mode 100644 index 0000000000..1e4874b8c8 --- /dev/null +++ b/tools/tdgpt/taosanalytics/test/forecast_test.py @@ -0,0 +1,115 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""forecast unit test cases""" + +import unittest, os.path, sys +import pandas as pd + +sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../../") + +from taosanalytics.algo.forecast import draw_fc_results +from taosanalytics.conf import setup_log_info +from taosanalytics.servicemgmt import loader + + +class ForecastTest(unittest.TestCase): + """forecast unit test cases""" + + @classmethod + def setUpClass(cls): + """ set up the environment for unit test """ + setup_log_info("unit_test.log") + loader.load_all_service() + + def get_input_list(self): + """ load data from csv """ + url = ('https://raw.githubusercontent.com/jbrownlee/Datasets/refs/heads/master/' + 'airline-passengers.csv') + data = pd.read_csv(url, index_col='Month', parse_dates=True) + + ts_list = data[['Passengers']].index.tolist() + dst_list = [int(item.timestamp()) for item in ts_list] + + return data[['Passengers']].values.tolist(), dst_list + + def test_holt_winters_forecast(self): + """ test holt winters forecast with invalid and then valid parameters""" + s = loader.get_service("holtwinters") + data, ts = self.get_input_list() + + s.set_input_list(data, ts) + self.assertRaises(ValueError, s.execute) + + s.set_params({"fc_rows": 10, "start_ts": 171000000, "time_step": 86400 * 30}) + + r = s.execute() + draw_fc_results(data, len(r["res"]) > 2, r["res"], len(r["res"][0]), "holtwinters") + + def test_holt_winters_forecast_2(self): + """test holt winters with valid parameters""" + s = loader.get_service("holtwinters") + data, ts = self.get_input_list() + + s.set_input_list(data, ts) + s.set_params( + { + "fc_rows": 10, "trend": 'mul', "seasonal": 'mul', "start_ts": 171000000, + "time_step": 86400 * 30, "period": 12 + } + ) + + r = s.execute() + + draw_fc_results(data, len(r["res"]) > 2, r["res"], len(r["res"][0]), "holtwinters") + + def test_holt_winter_invalid_params(self): + """parameters validation check""" + s = loader.get_service("holtwinters") + + self.assertRaises(ValueError, s.set_params, {"trend": "mul"}) + + self.assertRaises(ValueError, s.set_params, {"trend": "mul"}) + + self.assertRaises(ValueError, s.set_params, {"trend": "mul", "fc_rows": 10}) + + self.assertRaises(ValueError, s.set_params, {"trend": "multi"}) + + self.assertRaises(ValueError, s.set_params, {"seasonal": "additive"}) + + self.assertRaises(ValueError, s.set_params, { + "fc_rows": 10, "trend": 'multi', "seasonal": 'addi', "start_ts": 171000000, + "time_step": 86400 * 30, "period": 12} + ) + + self.assertRaises(ValueError, s.set_params, + {"fc_rows": 10, "trend": 'mul', "seasonal": 'add', "time_step": 86400 * 30, "period": 12} + ) + + s.set_params({"fc_rows": 10, "start_ts": 171000000, "time_step": 86400 * 30}) + + self.assertRaises(ValueError, s.set_params, {"fc_rows": 'abc', "start_ts": 171000000, "time_step": 86400 * 30}) + + self.assertRaises(ValueError, s.set_params, {"fc_rows": 10, "start_ts": "aaa", "time_step": "30"}) + + self.assertRaises(ValueError, s.set_params, {"fc_rows": 10, "start_ts": 171000000, "time_step": 0}) + + def test_arima(self): + """arima algorithm check""" + s = loader.get_service("arima") + data, ts = self.get_input_list() + + s.set_input_list(data, ts) + self.assertRaises(ValueError, s.execute) + + s.set_params( + {"fc_rows": 10, "start_ts": 171000000, "time_step": 86400 * 30, "period": 12, + "start_p": 0, "max_p": 10, "start_q": 0, "max_q": 10} + ) + r = s.execute() + + rows = len(r["res"][0]) + draw_fc_results(data, len(r["res"]) > 1, r["res"], rows, "arima") + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/tdgpt/taosanalytics/test/install_test.py b/tools/tdgpt/taosanalytics/test/install_test.py new file mode 100644 index 0000000000..9c5aa9238f --- /dev/null +++ b/tools/tdgpt/taosanalytics/test/install_test.py @@ -0,0 +1,27 @@ +"""perform the build release package and install and then test the restful service""" + +import unittest +import os + + +class ForecastTest(unittest.TestCase): + + def test_release(self): + """ test the package """ + pass + + # print("build install package") + # os.system("../../script/release.sh") + # print("build completed") + # + # self.assertEqual(os.path.exists("../../release/TDengine-enterprise-anode-1.0.0.tar.gz"), 1) + + def test_install(self): + """ test """ + pass + + # print("start to install package") + # os.system("tar zxvf ../../release/TDengine-enterprise-anode-1.0.0.tar.gz") + # os.chdir("../../release/TDengine-enterprise-anode-1.0.0/") + # + # os.system("./install.sh") diff --git a/tools/tdgpt/taosanalytics/test/restful_api_test.py b/tools/tdgpt/taosanalytics/test/restful_api_test.py new file mode 100644 index 0000000000..6463343e00 --- /dev/null +++ b/tools/tdgpt/taosanalytics/test/restful_api_test.py @@ -0,0 +1,259 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""flask restful api test module""" + +import sys, os.path + +sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../../") + +from flask_testing import TestCase +from taosanalytics.app import app +from taosanalytics.conf import setup_log_info + + +class RestfulTest(TestCase): + """ restful api test class """ + + def create_app(self): + app.testing = True + setup_log_info("restfull_test.log") + return app + + def test_access_main_page(self): + """ test asscess default main page """ + response = self.client.get('/') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content_length, len("TDengine© Time Series Data Analytics Platform (ver 1.0.1)") + 1) + + def test_load_status(self): + """ test load the server status """ + response = self.client.get('/status') + self.assertEqual(response.status_code, 200) + res = response.json + + self.assertEqual(res['protocol'], 1.0) + self.assertEqual(res['status'], 'ready') + + def test_load_algos(self): + """ test load provided algos""" + response = self.client.get('/list') + self.assertEqual(response.status_code, 200) + + res = response.json + self.assertEqual(res['version'], 0.1) + self.assertEqual(res['protocol'], 1.0) + + d = res['details'] + self.assertEqual(len(d), 2) + + def test_forecast(self): + """test forecast api""" + response = self.client.post("/forecast", json={ + "schema": [ + ["ts", "TIMESTAMP", 8], + ["val", "INT", 4] + ], + "data": [ + [ + 1577808000000, 1577808001000, 1577808002000, 1577808003000, 1577808004000, + 1577808005000, 1577808006000, 1577808007000, 1577808008000, 1577808009000, + 1577808010000, 1577808011000, 1577808012000, 1577808013000, 1577808014000, + 1577808015000, 1577808016000, 1577808017000, 1577808018000, 1577808019000, + 1577808020000, 1577808021000, 1577808022000, 1577808023000, 1577808024000, + 1577808025000, 1577808026000, 1577808027000, 1577808028000, 1577808029000, + 1577808030000, 1577808031000, 1577808032000, 1577808033000, 1577808034000, + 1577808035000, 1577808036000, 1577808037000, 1577808038000, 1577808039000, + 1577808040000, 1577808041000, 1577808042000, 1577808043000, 1577808044000, + 1577808045000, 1577808046000, 1577808047000, 1577808048000, 1577808049000, + 1577808050000, 1577808051000, 1577808052000, 1577808053000, 1577808054000, + 1577808055000, 1577808056000, 1577808057000, 1577808058000, 1577808059000, + 1577808060000, 1577808061000, 1577808062000, 1577808063000, 1577808064000, + 1577808065000, 1577808066000, 1577808067000, 1577808068000, 1577808069000, + 1577808070000, 1577808071000, 1577808072000, 1577808073000, 1577808074000, + 1577808075000, 1577808076000, 1577808077000, 1577808078000, 1577808079000, + 1577808080000, 1577808081000, 1577808082000, 1577808083000, 1577808084000, + 1577808085000, 1577808086000, 1577808087000, 1577808088000, 1577808089000, + 1577808090000, 1577808091000, 1577808092000, 1577808093000, 1577808094000, + 1577808095000 + ], + [ + 13, 14, 8, 10, 16, 26, 32, 27, 18, 32, 36, 24, 22, 23, 22, 18, 25, 21, 21, + 14, 8, 11, 14, 23, 18, 17, 19, 20, 22, 19, 13, 26, 13, 14, 22, 24, 21, 22, + 26, 21, 23, 24, 27, 41, 31, 27, 35, 26, 28, 36, 39, 21, 17, 22, 17, 19, 15, + 34, 10, 15, 22, 18, 15, 20, 15, 22, 19, 16, 30, 27, 29, 23, 20, 16, 21, 21, + 25, 16, 18, 15, 18, 14, 10, 15, 8, 15, 6, 11, 8, 7, 13, 10, 23, 16, 15, 25 + ] + ], + "option": "algo=holtwinters", + "algo": "holtwinters", + "prec": "ms", + "wncheck": 1, + "return_conf": 1, + "forecast_rows": 10, + "conf": 95, + "start": 1577808096000, + "every": 1000, + "rows": 96, + "protocol": 1.0 + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json["algo"], "holtwinters") + self.assertEqual(response.json["rows"], 10) + self.assertEqual(response.json["period"], 0) + self.assertEqual(response.json["res"][0][0], 1577808096000) + self.assertEqual(response.json["res"][0][-1], 1577808105000) + self.assertEqual(len(response.json["res"][0]), response.json["rows"]) + self.assertEqual(len(response.json["res"]), 4) + + def test_ad(self): + """test anomaly detect api""" + response = self.client.post("/anomaly-detect", json={ + "schema": [ + ["ts", "TIMESTAMP", 8], + ["val", "INT", 4] + ], + "data": [ + [1577808000000, 1577808001000, 1577808002000, 1577808003000, 1577808004000, + 1577808005000, 1577808006000, 1577808007000, 1577808008000, 1577808009000, + 1577808010000, 1577808011000, 1577808012000, 1577808013000, 1577808014000, + 1577808015000, 1577808016000], + [5, 14, 15, 15, 14, 19, 17, 16, 20, 22, 8, 21, 28, 11, 9, 29, 40] + ], + "rows": 17, + "algo": "iqr" + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json["rows"], 1) + self.assertEqual(response.json["algo"], "iqr") + + def test_ad_error_get(self): + """1. invalid http method""" + response = self.client.get("/anomaly-detect", json={ + "schema": [ + ["ts", "TIMESTAMP", 8], + ["val", "INT", 4] + ], + "data": [ + [1577808000000, 1577808001000, 1577808002000, 1577808003000, 1577808004000, + 1577808005000, 1577808006000, 1577808007000, 1577808008000, 1577808009000, + 1577808010000, 1577808011000, 1577808012000, 1577808013000, 1577808014000, + 1577808015000, 1577808016000], + [5, 14, 15, 15, 14, 19, 17, 16, 20, 22, 8, 21, 28, 11, 9, 29, 40] + ], + "rows": 17, + "algo": "iqr" + }) + + self.assertEqual(response.status_code, 405) + + def test_ad_error_empty_payload(self): + """2. list that is going to apply anomaly detection is empty or less value than the threshold + , which is [10, 100000]""" + response = self.client.post("/anomaly-detect", json={ + "schema": [ + ["ts", "TIMESTAMP", 8], + ["val", "INT", 4] + ], + "data": [ + [1577808000000, 1577808001000, 1577808002000, 1577808003000, 1577808004000, + 1577808005000, 1577808006000, 1577808007000, 1577808008000], + [5, 14, 15, 15, 14, 19, 17, 16, 20] + ], + "rows": 9, + "algo": "iqr" + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json["rows"], -1) + + def test_ad_error_single_col(self): + """3. only one column""" + response = self.client.post("/anomaly-detect", json={ + "schema": [ + ["ts", "TIMESTAMP", 8], + ["val", "INT", 4] + ], + "data": [ + [1577808000000, 1577808001000, 1577808002000, 1577808003000, 1577808004000, + 1577808005000, 1577808006000, 1577808007000, 1577808008000] + ], + "rows": 9, + "algo": "iqr" + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json["rows"], -1) + + def test_ad_error_three_cols(self): + """4. there are three input columns """ + response = self.client.post("/anomaly-detect", json={ + "schema": [ + ["ts", "TIMESTAMP", 8], + ["val", "INT", 4], + ["val1", "INT", 4] + ], + "data": [ + [1577808000000, 1577808001000, 1577808002000, 1577808003000, 1577808004000, + 1577808005000, 1577808006000, 1577808007000, 1577808008000, 1577808009000], + [5, 14, 15, 15, 14, 19, 17, 16, 20, 44], + [5, 14, 15, 15, 14, 19, 17, 16, 20, 44] + ], + "rows": 10, + "algo": "iqr" + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json["rows"], -1) + + def test_ad_disorder_cols(self): + """5. disorder two columns """ + response = self.client.post("/anomaly-detect", json={ + "schema": [ + ["val", "INT", 4], + ["ts", "TIMESTAMP", 8] + ], + "data": [ + [5, 14, 15, 15, 14, 19, 17, 16, 20, 44], + [1577808000000, 1577808001000, 1577808002000, 1577808003000, 1577808004000, + 1577808005000, 1577808006000, 1577808007000, 1577808008000, 1577808009000], + ], + "rows": 10, + "algo": "iqr" + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json["rows"], 2) + + def test_missing_schema(self): + """6. missing schema info""" + response = self.client.post("/anomaly-detect", json={ + "data": [ + [5, 14, 15, 15, 14, 19, 17, 16, 20, 44], + [1577808000000, 1577808001000, 1577808002000, 1577808003000, 1577808004000, + 1577808005000, 1577808006000, 1577808007000, 1577808008000, 1577808009000], + ], + "rows": 10, + "algo": "iqr" + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json["rows"], -1) + + def test_invalid_schema_info(self): + """7. invalid schema info""" + response = self.client.post("/anomaly-detect", json={ + "schema": [ + ["ts", "TIMESTAMP", 8] + ], + "data": [ + [1577808000000, 1577808001000, 1577808002000, 1577808003000, 1577808004000, + 1577808005000, 1577808006000, 1577808007000, 1577808008000, 1577808009000], + ], + "rows": 10, + "algo": "iqr" + }) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json["rows"], -1) diff --git a/tools/tdgpt/taosanalytics/test/unit_test.py b/tools/tdgpt/taosanalytics/test/unit_test.py new file mode 100644 index 0000000000..f6ecdf0d5b --- /dev/null +++ b/tools/tdgpt/taosanalytics/test/unit_test.py @@ -0,0 +1,106 @@ +# encoding:utf-8 +# pylint: disable=c0103 +"""unit test module""" +import os.path +import unittest +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../../") + +from taosanalytics.servicemgmt import loader +from taosanalytics.util import convert_results_to_windows, is_white_noise, parse_options, is_stationary + + +class UtilTest(unittest.TestCase): + """utility test cases""" + + def test_generate_anomaly_window(self): + # Test case 1: Normal input + wins = convert_results_to_windows([1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, -1], + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) + print(f"The result window is:{wins}") + + # Assert the number of windows + self.assertEqual(len(wins), 2) + + # Assert the first window + self.assertListEqual(wins[0], [7, 9]) + + # Assert the second window + self.assertListEqual(wins[1], [12, 12]) + + # Test case 2: Anomaly input list is empty + wins = convert_results_to_windows([], [1, 2]) + self.assertListEqual(wins, []) + + # Test case 3: Anomaly input list is None + wins = convert_results_to_windows([], None) + self.assertListEqual(wins, []) + + # Test case 4: Timestamp list is None + wins = convert_results_to_windows(None, []) + self.assertListEqual(wins, []) + + def test_validate_input_data(self): + pass + + def test_validate_pay_load(self): + pass + + def test_validate_forecast_input_data(self): + pass + + def test_convert_results_to_windows(self): + pass + + def test_is_white_noise(self): + """ + Test the is_white_noise function. + This function tests the functionality of the is_white_noise function by providing a list and asserting the expected result. + """ + list1 = [] + wn = is_white_noise(list1) + self.assertFalse(wn) + + def test_is_stationary(self): + """test whether data is stationary or not""" + st = is_stationary([1, 2, 3, 4, 5, 7, 5, 1, 54, 3, 6, 87, 45, 14, 24]) + self.assertEquals(st, False) + + def test_parse_options(self): + """test case for parse key/value string into k/v pair""" + option_str = "algo=ksigma,k=2,invalid_option=invalid_str" + opt = parse_options(option_str) + + self.assertEqual(len(opt), 3) + self.assertDictEqual(opt, {'algo': 'ksigma', 'k': '2', 'invalid_option': 'invalid_str'}) + + def test_get_data_index(self): + """ test the get the data index method""" + schema = [ + ["val", "INT", 4], + ["ts", "TIMESTAMP", 8] + ] + for index, val in enumerate(schema): + if val[0] == "val": + return index + + +class ServiceTest(unittest.TestCase): + def setUp(self): + """ load all service before start unit test """ + loader.load_all_service() + + def test_get_all_algos(self): + service_list = loader.get_service_list() + self.assertEqual(len(service_list["details"]), 2) + + for item in service_list["details"]: + if item["type"] == "anomaly-detection": + self.assertEqual(len(item["algo"]), 6) + else: + self.assertEqual(len(item["algo"]), 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/tdgpt/taosanalytics/util.py b/tools/tdgpt/taosanalytics/util.py new file mode 100644 index 0000000000..b9b292c3b4 --- /dev/null +++ b/tools/tdgpt/taosanalytics/util.py @@ -0,0 +1,126 @@ +# encoding:utf-8 +"""utility methods to helper query processing""" +import numpy as np +from statsmodels.stats.diagnostic import acorr_ljungbox +from statsmodels.tsa.stattools import adfuller + +from taosanalytics.conf import app_logger + + +def validate_pay_load(json_obj): + """ validate the input payload """ + if "data" not in json_obj: + raise ValueError('data attr does not exist in json') + + data = json_obj["data"] + + if len(data) <= 1: + raise ValueError('only one column, primary timestamp column should be provided') + + if len(data) > 2: + raise ValueError('too many columns') + + rows = len(data[0]) + + if rows != len(data[1]): + raise ValueError('data inconsistent, number of rows are not identical') + + if rows < 10 or rows > 40000: + raise ValueError(f'number of rows should between 10 and 40000, actual {rows} rows') + + if "schema" not in json_obj: + raise ValueError('schema is missing') + + index = get_data_index(json_obj["schema"]) + if index == -1: + raise ValueError('invalid schema info, data column is missing') + + +def convert_results_to_windows(result, ts_list): + """generate the window according to anomaly detection result""" + skey, ekey = -1, -1 + wins = [] + + if ts_list is None or result is None or len(result) != len(ts_list): + return wins + + for index, val in enumerate(result): + if val == -1: + ekey = ts_list[index] + if skey == -1: + skey = ts_list[index] + else: + if ekey != -1: + wins.append([skey, ekey]) + skey, ekey = -1, -1 + + if ekey != -1: + wins.append([skey, ekey]) + + return wins + + +def is_white_noise(input_list): + """ determine whether the input list is a white noise list or not """ + if len(input_list) < 16: # the number of items in the list is insufficient + return False + + res = acorr_ljungbox(input_list, lags=[6, 12, 16], boxpierce=True, return_df=True) + q_lb = res.lb_pvalue.array[2] + return q_lb >= 0.05 + + +def is_stationary(input_list): + """ determine whether the input list is weak stationary or not """ + adf, pvalue, usedlag, nobs, critical_values, _ = adfuller(input_list, autolag='AIC') + app_logger.log_inst.info("adf is:%f critical value is:%s" % (adf, critical_values)) + return pvalue < 0.05 + + +def parse_options(option_str) -> dict: + """ + the option format is like the following string: "algo=ksigma,k=2,invalid_option=invalid_str" + convert it to the dict format + """ + options = {} + + if option_str is None or len(option_str) == 0: + return options + + opt_list = option_str.split(",") + for line in opt_list: + if "=" not in line or len(line.strip()) < 3: + continue + + kv_pair = line.strip().split("=") + if kv_pair[0].strip() == '' or kv_pair[1].strip() == '': + continue + + options[kv_pair[0].strip()] = kv_pair[1].strip() + + return options + + +def get_data_index(schema): + """get the data index according to the schema info""" + for index, val in enumerate(schema): + if val[0] == "val": + return index + + return -1 + + +def get_ts_index(schema): + """get the timestamp index according to the schema info""" + for index, val in enumerate(schema): + if val[0] == "ts": + return index + return -1 + + +def create_sequences(values, time_steps): + """ create sequences for training model """ + output = [] + for i in range(len(values) - time_steps + 1): + output.append(values[i: (i + time_steps)]) + return np.stack(output)