diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 7ce0a5ce48..b7b97cee8e 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -9,17 +9,12 @@ on: paths-ignore: - 'packaging/**' - 'docs/**' - repository_dispatch: - types: [trigger-tests-from-tdinternal] concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.event.client_payload.ref}}-${{ github.event_name == 'repository_dispatch' && 'dispatch' || ''}} + group: ${{ github.workflow }}-${{ github.ref }}-TDengine cancel-in-progress: true env: - CONTAINER_NAME: 'taosd-test' - WKDIR: '/var/lib/jenkins/workspace' - WK: '/var/lib/jenkins/workspace/TDinternal' WKC: '/var/lib/jenkins/workspace/TDinternal/community' jobs: @@ -28,439 +23,80 @@ jobs: group: CI labels: [self-hosted, Linux, X64, testing] outputs: - tdinternal: ${{ steps.parameters.outputs.tdinternal }} run_function_test: ${{ steps.parameters.outputs.run_function_test }} run_tdgpt_test: ${{ steps.parameters.outputs.run_tdgpt_test }} - source_branch: ${{ steps.parameters.outputs.source_branch }} - target_branch: ${{ steps.parameters.outputs.target_branch }} - pr_number: ${{ steps.parameters.outputs.pr_number }} steps: - name: Determine trigger source and fetch parameters id: parameters run: | set -euo pipefail - # check the trigger source and get branch information - if [ "${{ github.event_name }}" == "repository_dispatch" ]; then - tdinternal="true" - source_branch=${{ github.event.client_payload.tdinternal_source_branch }} - target_branch=${{ github.event.client_payload.tdinternal_target_branch }} - pr_number=${{ github.event.client_payload.tdinternal_pr_number }} + target_branch=${{ github.event.pull_request.base.ref }} + + # Check whether to run tdgpt test cases + cd ${{ env.WKC }} + changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) + echo "changed files exclude doc: ${changed_files_non_doc}" + + if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then run_tdgpt_test="true" + else + run_tdgpt_test="false" + fi + echo "run tdgpt test: ${run_tdgpt_test}" + + # Check whether to run function test cases + changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ + grep -v "^docs/en/" | \ + grep -v "^docs/zh/" | \ + grep -v ".md$" | \ + grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ + tr '\n' ' ' || :) + echo "changed files exclude tdgpt: ${changed_files_non_tdgpt}" + + if [ -n "$changed_files_non_tdgpt" ]; then run_function_test="true" else - tdinternal="false" - source_branch=${{ github.event.pull_request.head.ref }} - target_branch=${{ github.event.pull_request.base.ref }} - pr_number=${{ github.event.pull_request.number }} - - # check whether to run tdgpt test cases - cd ${{ env.WKC }} - changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) - echo “changed files exclude doc, ${changed_files_non_doc}" - if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then - run_tdgpt_test="true" - else - run_tdgpt_test="false" - fi - echo "run tdgpt test: ${run_tdgpt_test}" - - # check whether to run function test cases - changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ - grep -v "^docs/en/" | \ - grep -v "^docs/zh/" | \ - grep -v ".md$" | \ - grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ - tr '\n' ' ' || :) - echo "changed file exclude tdgpt: ${changed_files_non_tdgpt}" - if [ -n "$changed_files_non_tdgpt" ]; then - run_function_test="true" - else - run_function_test="false" - fi - echo "run function test: ${run_function_test}" + run_function_test="false" fi - echo "tdinternal=$tdinternal" >> $GITHUB_OUTPUT + echo "run function test: ${run_function_test}" + + # Output the results for GitHub Actions echo "run_function_test=$run_function_test" >> $GITHUB_OUTPUT echo "run_tdgpt_test=$run_tdgpt_test" >> $GITHUB_OUTPUT - echo "source_branch=$source_branch" >> $GITHUB_OUTPUT - echo "target_branch=$target_branch" >> $GITHUB_OUTPUT - echo "pr_number=$pr_number" >> $GITHUB_OUTPUT + + echo ${{ github.event.pull_request.head.ref }} + echo ${{ github.event.pull_request.base.ref }} + echo ${{ github.event.pull_request.number }} run-tests-on-linux: + uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml needs: fetch-parameters - runs-on: - group: CI - labels: [self-hosted, Linux, X64, testing] - timeout-minutes: 200 - env: - IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }} - RUN_RUNCTION_TEST: ${{ needs.fetch-parameters.outputs.run_function_test }} - RUN_TDGPT_TEST: ${{ needs.fetch-parameters.outputs.run_tdgpt_test }} - SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }} - TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }} - PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }} - steps: - - name: Output the environment information - run: | - echo "::group::Environment Info" - date - hostname - env - echo "Runner: ${{ runner.name }}" - echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}" - echo "Workspace: ${{ env.WKDIR }}" - git --version - echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}" - echo "::endgroup::" - - - name: Prepare repositories - run: | - set -euo pipefail - prepare_environment() { - cd "$1" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "$2" - } - prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}" - prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}" - - - name: Get latest codes and logs for TDinternal PR - if: ${{ env.IS_TDINTERNAL == 'true' }} - run: | - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Get latest codes and logs for TDengine PR - if: ${{ env.IS_TDINTERNAL == 'false' }} - run: | - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Update submodule - run: | - cd ${{ env.WKC }} - git submodule update --init --recursive - - name: Output the 'file_no_doc_changed' information to the file - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - mkdir -p ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }} - cd ${{ env.WKC }} - changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ env.TARGET_BRANCH }}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :) - echo $changed_files_non_doc > ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt - - name: Check assert testing - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - ./run_check_assert_container.sh -d ${{ env.WKDIR }} - - name: Check void function testing - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - ./run_check_void_container.sh -d ${{ env.WKDIR }} - - name: Build docker container - if: ${{ env.RUN_RUNCTION_TEST == 'true' }} - run: | - date - rm -rf ${{ env.WKC }}/debug - cd ${{ env.WKC }}/tests/parallel_test - time ./container_build.sh -w ${{ env.WKDIR }} -e - - name: Get parameters for testing - id: get_param - run: | - log_server_file="/home/log_server.json" - timeout_cmd="" - extra_param="" - - if [ -f "$log_server_file" ]; then - log_server_enabled=$(jq '.enabled' "$log_server_file") - timeout_param=$(jq '.timeout' "$log_server_file") - if [ "$timeout_param" != "null" ] && [ "$timeout_param" != "0" ]; then - timeout_cmd="timeout $timeout_param" - fi - - if [ "$log_server_enabled" == "1" ]; then - log_server=$(jq '.server' "$log_server_file" | sed 's/\\\"//g') - if [ "$log_server" != "null" ] && [ "$log_server" != "" ]; then - extra_param="-w $log_server" - fi - fi - fi - echo "timeout_cmd=$timeout_cmd" >> $GITHUB_OUTPUT - echo "extra_param=$extra_param" >> $GITHUB_OUTPUT - - name: Run function returns with a null pointer scan testing - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - ./run_scan_container.sh -d ${{ env.WKDIR }} -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -f ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt ${{ steps.get_param.outputs.extra_param }} - - name: Run tdgpt test cases - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' && env.RUN_TDGPT_TEST == 'true' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - export DEFAULT_RETRY_TIME=2 - date - timeout 600 time ./run.sh -e -m /home/m.json -t tdgpt_cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 300 ${{ steps.get_param.outputs.extra_param }} - - name: Run function test cases - if: ${{ env.RUN_RUNCTION_TEST == 'true'}} - run: | - cd ${{ env.WKC }}/tests/parallel_test - export DEFAULT_RETRY_TIME=2 - date - ${{ steps.get_param.outputs.timeout_cmd }} time ./run.sh -e -m /home/m.json -t cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 1200 ${{ steps.get_param.outputs.extra_param }} + if: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || needs.fetch-parameters.outputs.run_function_test == 'true' }} + with: + tdinternal: false + run_function_test: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' || false }} + run_tdgpt_test: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || false }} + source_branch: ${{ github.event.pull_request.head.ref }} + target_branch: ${{ github.event.pull_request.base.ref }} + pr_number: ${{ github.event.pull_request.number }} run-tests-on-mac: + uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} - runs-on: - group: CI - labels: [self-hosted, macOS, testing] - timeout-minutes: 60 - env: - IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }} - SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }} - TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }} - PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }} - steps: - - name: Output the environment information - run: | - echo "::group::Environment Info" - date - hostname - env - echo "Runner: ${{ runner.name }}" - echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}" - echo "Workspace: ${{ env.WKDIR }}" - git --version - echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}" - echo "::endgroup::" - - name: Prepare repositories - run: | - set -euo pipefail - prepare_environment() { - cd "$1" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "$2" - } - prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}" - prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}" - - name: Get latest codes and logs for TDinternal PR - if: ${{ env.IS_TDINTERNAL == 'true' }} - run: | - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Get latest codes and logs for TDengine PR - if: ${{ env.IS_TDINTERNAL == 'false' }} - run: | - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Update submodule - run: | - cd ${{ env.WKC }} - git submodule update --init --recursive - - name: Run tests - run: | - date - cd ${{ env.WK }} - rm -rf debug - mkdir debug - cd ${{ env.WK }}/debug - echo $PATH - echo "PATH=/opt/homebrew/bin:$PATH" >> $GITHUB_ENV - cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false -DCMAKE_BUILD_TYPE=Release - make -j10 - ctest -j10 || exit 7 - date + with: + tdinternal: false + source_branch: ${{ github.event.pull_request.head.ref }} + target_branch: ${{ github.event.pull_request.base.ref }} + pr_number: ${{ github.event.pull_request.number }} run-tests-on-windows: + uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} - runs-on: - group: CI - labels: [self-hosted, Windows, X64, testing] - timeout-minutes: 126 - env: - IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }} - SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }} - TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }} - PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }} - WIN_INTERNAL_ROOT: "C:\\workspace\\0\\TDinternal" - WIN_COMMUNITY_ROOT: "C:\\workspace\\0\\TDinternal\\community" - WIN_SYSTEM_TEST_ROOT: "C:\\workspace\\0\\TDinternal\\community\\tests\\system-test" - WIN_VS_PATH: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" - WIN_CPU_TYPE: "x64" - steps: - - name: Output the environment information - run: | - hostname - taskkill /f /t /im python.exe - taskkill /f /t /im bash.exe - taskkill /f /t /im taosd.exe - ipconfig - set - date /t - time /t - rd /s /Q "%WIN_INTERNAL_ROOT%\debug" || exit 0 - shell: cmd - - name: Prepare repositories - run: | - :: Prepare internal repository - if exist "%WIN_INTERNAL_ROOT%" ( - cd /d "%WIN_INTERNAL_ROOT%" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "%TARGET_BRANCH%" - ) else ( - echo Directory does not exist: "%WIN_INTERNAL_ROOT%" - exit 1 - ) - - :: Prepare community repository - if exist "%WIN_COMMUNITY_ROOT%" ( - cd /d "%WIN_COMMUNITY_ROOT%" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "%TARGET_BRANCH%" - ) else ( - echo Directory does not exist: "%WIN_COMMUNITY_ROOT%" - exit 1 - ) - shell: cmd - - name: Get latest codes and logs for TDinternal PR - if: ${{ env.IS_TDINTERNAL == 'true' }} - run: | - cd %WIN_INTERNAL_ROOT% - git pull origin %TARGET_BRANCH% - git fetch origin +refs/pull/%PR_NUMBER%/merge - git checkout -qf FETCH_HEAD - cd %WIN_COMMUNITY_ROOT% - git remote prune origin - git pull - shell: cmd - - name: Get latest codes and logs for TDengine PR - if: ${{ env.IS_TDINTERNAL == 'false' }} - run: | - cd %WIN_INTERNAL_ROOT% - git pull origin %TARGET_BRANCH% - cd %WIN_COMMUNITY_ROOT% - git remote prune origin - git pull origin %TARGET_BRANCH% - git fetch origin +refs/pull/%PR_NUMBER%/merge - git checkout -qf FETCH_HEAD - shell: cmd - - name: Output branch and log information - run: | - cd %WIN_INTERNAL_ROOT% - git branch - git log -5 - - cd %WIN_COMMUNITY_ROOT% - git branch - git log -5 - shell: cmd - - name: Update submodule - run: | - cd %WIN_COMMUNITY_ROOT% - git submodule update --init --recursive - shell: cmd - - name: Build on windows - run: | - echo "building ..." - time /t - cd %WIN_INTERNAL_ROOT% - mkdir debug - cd debug - time /t - call "%WIN_VS_PATH%" %WIN_CPU_TYPE% - set CL=/MP8 - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" - time /t - cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true || exit 7 - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" - time /t - jom -j 6 || exit 8 - time /t - - cd %WIN_COMMUNITY_ROOT%/tests/ci - pip3 install taospy==2.7.21 - pip3 install taos-ws-py==0.3.8 - xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 - shell: cmd - - name: Run ctest - run: | - echo "windows ctest ..." - time /t - cd %WIN_INTERNAL_ROOT%\\debug - ctest -j 1 || exit 7 - time /t - shell: cmd - - name: Run function test - run: | - echo "windows test ..." - xcopy /e/y/i/f "%WIN_INTERNAL_ROOT%\debug\build\lib\taos.dll" C:\Windows\System32 - ls -l "C:\Windows\System32\taos.dll" - time /t - cd %WIN_SYSTEM_TEST_ROOT% - echo "testing ..." - test-all.bat ci - time /t - shell: cmd + with: + tdinternal: false + source_branch: ${{ github.event.pull_request.head.ref }} + target_branch: ${{ github.event.pull_request.base.ref }} + pr_number: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/tdgpt-ci.yml b/.github/workflows/tdgpt-ci.yml index abe446f9c2..c8fbaccd7d 100644 --- a/.github/workflows/tdgpt-ci.yml +++ b/.github/workflows/tdgpt-ci.yml @@ -1,7 +1,6 @@ name: TDGpt CI Pipeline on: - push: pull_request: branches: - '3.0' diff --git a/.github/workflows/tdgpt-update-service.yml b/.github/workflows/tdgpt-update-service.yml index b45e6d1ffc..03bff257c1 100644 --- a/.github/workflows/tdgpt-update-service.yml +++ b/.github/workflows/tdgpt-update-service.yml @@ -34,3 +34,8 @@ jobs: ./install.sh fi systemctl restart taosanoded + + - name: Clean up + if: always() + run: | + if [[ -f ${{ env.WKC }}/tools/tdgpt/release/TDengine-enterprise-anode-1.0.1 ]] then rm -rf ${{ env.WKC }}/tools/tdgpt/release/TDengine-enterprise-anode-1.0.1; fi diff --git a/docs/en/05-basic/03-query.md b/docs/en/05-basic/03-query.md index 463f58fd6e..d0aef17f4d 100644 --- a/docs/en/05-basic/03-query.md +++ b/docs/en/05-basic/03-query.md @@ -145,20 +145,19 @@ Query OK, 10 row(s) in set (2.415961s) In TDengine, you can use the window clause to perform aggregation queries by time window partitioning, which is particularly suitable for scenarios requiring analysis of large amounts of time-series data, such as smart meters collecting data every 10s but needing to query the average temperature every 1min. -The window clause allows you to partition the queried data set by windows and aggregate the data within each window, including: - -- Time window (time window) -- State window (status window) -- Session window (session window) -- Event window (event window) - -The logic of window partitioning is shown in the following image: +The window clause allows you to partition the queried data set by windows and aggregate the data within each window. The logic of window partitioning is shown in the following image:
Windowing description
Figure 1. Windowing logic
+- Time Window: Data is divided based on time intervals, supporting sliding and tumbling time windows, suitable for data aggregation over fixed time periods. +- Status Window: Windows are divided based on changes in device status values, with data of the same status value grouped into one window, which closes when the status value changes. +- Session Window: Sessions are divided based on the differences in record timestamps, with records having a timestamp interval less than the predefined value belonging to the same session. +- Event Window: Windows are dynamically divided based on the start and end conditions of events, opening when the start condition is met and closing when the end condition is met. +- Count Window: Windows are divided based on the number of data rows, with each window consisting of a specified number of rows for aggregation calculations. + The syntax for the window clause is as follows: ```sql diff --git a/docs/en/06-advanced/05-data-in/07-mqtt.md b/docs/en/06-advanced/05-data-in/07-mqtt.md index 73ef3b534c..47ffa82bf2 100644 --- a/docs/en/06-advanced/05-data-in/07-mqtt.md +++ b/docs/en/06-advanced/05-data-in/07-mqtt.md @@ -86,9 +86,15 @@ The keep alive interval is the time interval negotiated between the client and t In **Clean Session**, choose whether to clear the session. The default value is true. -Fill in the Topic names to be consumed in **Subscription Topics and QoS Configuration**. Use the following format: `topic1::0,topic2::1`. +In the **Topics Qos Config**, fill in the topic name and QoS to subscribe. Use the following format: `{topic_name}::{qos}` (e.g., `my_topic::0`). MQTT protocol 5.0 supports shared subscriptions, allowing multiple clients to subscribe to the same topic for load balancing. Use the following format: `$share/{group_name}/{topic_name}::{qos}`, where `$share` is a fixed prefix indicating the enablement of shared subscription, and `group_name` is the client group name, similar to Kafka's consumer group. -Click the **Check Connectivity** button to check if the data source is available. +In the **Topic Analysis**, fill in the MQTT topic parsing rules. The format is the same as the MQTT Topic, parsing each level of the MQTT Topic into corresponding variable names, with `_` indicating that the current level is ignored during parsing. For example: if the MQTT Topic `a/+/c` corresponds to the parsing rule `v1/v2/_`, it means assigning the first level `a` to variable `v1`, the value of the second level (where the wildcard `+` represents any value) to variable `v2`, and ignoring the value of the third level `c`, which will not be assigned to any variable. In the `payload parsing` below, the variables obtained from Topic parsing can also participate in various transformations and calculations. + +In the **Compression**, configure the message body compression algorithm. After receiving the message, taosX uses the corresponding compression algorithm to decompress the message body and obtain the original data. Options include none (no compression), gzip, snappy, lz4, and zstd, with the default being none. + +In the **Char Encoding**, configure the message body encoding format. After receiving the message, taosX uses the corresponding encoding format to decode the message body and obtain the original data. Options include UTF_8, GBK, GB18030, and BIG5, with the default being UTF_8. + +Click the **Check Connection** button to check if the data source is available.
diff --git a/docs/en/08-operation/03-deployment/03-kubernetes.md b/docs/en/08-operation/03-deployment/03-kubernetes.md index dbb6022ce6..fdec50b9d6 100644 --- a/docs/en/08-operation/03-deployment/03-kubernetes.md +++ b/docs/en/08-operation/03-deployment/03-kubernetes.md @@ -339,7 +339,7 @@ Helm operates Kubernetes using kubectl and kubeconfig configurations, which can The TDengine Chart has not yet been released to the Helm repository, it can currently be downloaded directly from GitHub: ```shell -wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-enterpise-3.5.0.tgz +wget https://github.com/taosdata/TDengine-Operator/raw/refs/heads/3.0/helm/tdengine-enterprise-3.5.0.tgz ``` Note that it's for the enterprise edition, and the community edition is not yet available. diff --git a/docs/en/14-reference/01-components/01-taosd.md b/docs/en/14-reference/01-components/01-taosd.md index 4527a7fcac..5ce9a40ecb 100644 --- a/docs/en/14-reference/01-components/01-taosd.md +++ b/docs/en/14-reference/01-components/01-taosd.md @@ -43,7 +43,7 @@ After modifying configuration file parameters, you need to restart the *taosd* s |resolveFQDNRetryTime | Cancelled after 3.x |Not supported |Number of retries when FQDN resolution fails| |timeToGetAvailableConn | Cancelled after 3.3.4.x |Maximum waiting time to get an available connection, range 10-50000000, in milliseconds, default value 500000| |maxShellConns | Cancelled after 3.x |Supported, effective after restart|Maximum number of connections allowed| -|maxRetryWaitTime | |Supported, effective after restart|Maximum timeout for reconnection,calculated from the time of retry,range is 0-86400000,in milliseconds, default value 10000| +|maxRetryWaitTime | |Supported, effective after restart|Maximum timeout for reconnection,calculated from the time of retry,range is 3000-86400000,in milliseconds, default value 10000| |shareConnLimit |Added in 3.3.4.0 |Supported, effective after restart|Number of requests a connection can share, range 1-512, default value 10| |readTimeout |Added in 3.3.4.0 |Supported, effective after restart|Minimum timeout for a single request, range 64-604800, in seconds, default value 900| diff --git a/docs/en/26-tdinternal/09-stream.md b/docs/en/26-tdinternal/09-stream.md index 2c8d41cccb..69424652dc 100644 --- a/docs/en/26-tdinternal/09-stream.md +++ b/docs/en/26-tdinternal/09-stream.md @@ -94,7 +94,7 @@ The sink task is responsible for receiving the output results from the agg task The above three types of tasks each play their roles in the stream computing architecture, distributed at different levels. Clearly, the number of source tasks directly depends on the number of vnodes, with each source task independently handling the data in its vnode without interference from other source tasks, and there are no sequential constraints. However, it is worth noting that if the final stream computing results converge to one table, then only one sink task will be deployed on the vnode where that table is located. The collaborative relationship between these three types of tasks is shown in the following diagram, together forming the complete execution process of stream computing tasks.
-Relationships between tasks +Relationships between tasks
Figure 3. Relationships between tasks
diff --git a/docs/en/27-train-faq/02-dst.md b/docs/en/27-train-faq/02-dst.md new file mode 100644 index 0000000000..6fda8f8a16 --- /dev/null +++ b/docs/en/27-train-faq/02-dst.md @@ -0,0 +1,293 @@ +--- +title: DST(Daylight Saving Time) Usage +description: Explanation and suggestions for using DST(Daylight Saving Time) in TDengine +--- + +## Background + +In the use of time-series databases, there are times when Daylight Saving Time (DST) is encountered. We analyze and explain the use and issues of DST in TDengine to help you use TDengine more smoothly. + +## Definitions + +### Time Zone + +A time zone is a region on Earth that uses the same standard time. Due to the Earth's rotation, to ensure that the time in each place is coordinated with the local sunrise and sunset, the world is divided into multiple time zones. + +### IANA Time Zone + +The IANA (Internet Assigned Numbers Authority) time zone database, also known as the tz database, provides a standard reference for global time zone information. It is the basis for modern systems and software to handle time zone-related operations. + +IANA uses the "Region/City" format (e.g., Europe/Berlin) to clearly identify time zones. + +TDengine supports the use of IANA time zones in different components (except for the time zone settings in Windows taos.cfg). + +### Standard Time and Local Time + +Standard time is the time determined based on a fixed meridian on Earth. It provides a unified reference point for each time zone. + +- Greenwich Mean Time (GMT): Historically used reference time, located at the 0° meridian. +- Coordinated Universal Time (UTC): The modern time standard, similar to GMT but more precise. + +The relationship between standard time and time zones is as follows: + +- Reference: Standard time (e.g., UTC) is the reference point for setting time zones. +- Offset: Different time zones are defined by their offset from standard time. For example, UTC+1 means 1 hour ahead of UTC. +- Regional Division: The world is divided into multiple time zones, each using one or more standard times. + +Relative to standard time, each region sets its local time based on its time zone: + +- Time Zone Offset: Local time equals standard time plus the offset of the time zone. For example, UTC+2 means 2 hours ahead of UTC. +- Daylight Saving Time (DST): Some regions adjust their local time during specific periods, such as moving the clock forward by one hour. See the next section for details. + +### Daylight Saving Time + +Daylight Saving Time (DST) is a system that advances the time by one hour to make better use of daylight and save energy. It usually starts in spring and ends in autumn. The specific start and end times of DST vary by region. The following explanation uses Berlin time as an example to illustrate DST and its effects. + +![DST Berlin](./02-dst/dst-berlin.png) + +According to this rule, you can see: + +- The time between 02:00:00 and 03:00:00 (excluding 03:00:00) on March 31, 2024, in Berlin local time does not exist (jump). +- The time between 02:00:00 and 03:00:00 (excluding 03:00:00) on October 27, 2024, in Berlin local time appears twice. + +#### DST and the IANA Time Zone Database + +- Recording Rules: The IANA time zone database records detailed DST rules for each region, including the start and end dates and times. +- Automatic Adjustment: Many operating systems and software use the IANA database to automatically handle DST adjustments. +- Historical Changes: The IANA database also tracks historical DST changes to ensure accuracy. + +#### DST and Timestamp Conversion + +- Converting a timestamp to local time is deterministic. For example, 1729990654 is Berlin time DST 2024-10-27 02:57:34, and 1729994254 is Berlin time standard time 2024-10-27 02:57:34 (these two local times are the same except for the time offset). +- Without specifying the time offset, converting local time to a timestamp is indeterminate. The time skipped during DST does not exist and cannot be converted to a timestamp, such as Berlin time 2024-03-31 02:34:56 does not exist and cannot be converted to a timestamp. The repeated time during the end of DST cannot determine which timestamp it is, such as 2024-10-27 02:57:34 without specifying the time offset cannot determine whether it is 1729990654 or 1729994254. Specifying the time offset can determine the timestamp, such as 2024-10-27 02:57:34 CEST(+02:00), specifying DST 2024-10-27 02:57:34 timestamp 1729990654. + +### RFC3339 Time Format + +RFC 3339 is an internet time format standard used to represent dates and times. It is based on the ISO 8601 standard but specifies some format details more specifically. + +The format is as follows: + +- Basic Format: `YYYY-MM-DDTHH:MM:SSZ` +- Time Zone Representation: + - Z represents Coordinated Universal Time (UTC). + - Offset format, such as +02:00, represents the time difference from UTC. + +With explicit time zone offsets, the RFC 3339 format can accurately parse and compare times globally. + +The advantages of RFC 3339 include: + +- Standardization: Provides a unified format for easy cross-system data exchange. +- Clarity: Clearly indicates time zone information, avoiding time misunderstandings. + +TDengine uses the RFC3339 format for display in REST API and Explorer UI. In SQL statements, you can use the RFC3339 format to write timestamp data: + +```sql +insert into t1 values('2024-10-27T01:59:59.000Z', 0); +select * from t1 where ts >= '2024-10-27T01:59:59.000Z'; +``` + +### Undefined Behavior + +Undefined behavior refers to specific code or operations that do not have a clearly defined result and do not guarantee compatibility with that result. TDengine may modify the current behavior in a future version without notifying users. Therefore, users should not rely on the current undefined behavior for judgment or application in TDengine. + +## Writing and Querying DST in TDengine + +We use the following table to show the impact of DST on writing and querying. + +![DST Table](./02-dst/dst-table.png) + +### Table Explanation + +- **TIMESTAMP**: TDengine uses a 64-bit integer to store raw timestamps. +- **UTC**: The UTC time representation corresponding to the timestamp. +- **Europe/Berlin**: The RFC3339 format time corresponding to the Europe/Berlin time zone. +- **Local**: The local time corresponding to the Europe/Berlin time zone (without time zone). + +### Table Analysis + +- At the **start of DST** (Berlin time March 31, 02:00), the time jumps directly from 02:00 to 03:00 (one hour forward). + - Light green is the timestamp one hour before the start of DST; + - Dark green is the timestamp one hour after the start of DST; + - Red indicates that the nonexistent local time was inserted into the TDengine database: + - Using SQL `INSERT INTO t1 values('2024-03-31 02:59:59',..)` to insert data from `2024-03-31 02:00:00` to `2024-03-31 02:59:59` will be automatically adjusted to -1000 (in TDengine, this is undefined behavior, currently this value is related to the database precision, millisecond database is -1000, microsecond database is -1000000, nanosecond database is -1000000000), because that moment does not exist in local time; +- At the **end of DST** (Berlin time October 27, 03:00), the time jumps from 03:00 to 02:00 (one hour back). + - Light blue indicates the timestamp one hour before the clock jump; + - Dark blue indicates the timestamp within one hour after the clock jump, its local time without time zone is the same as the previous hour. + - Purple indicates the timestamp one hour after the clock jump; +- **Local Time Changes**: It can be seen that due to the adjustment of DST, local time changes, which may cause some time periods to appear repeated or missing. +- **UTC Time Unchanged**: UTC time remains unchanged, ensuring the consistency and order of time. +- **RFC3339**: The RFC3339 format time shows the change in time offset, changing to +02:00 after the start of DST and to +01:00 after the end of DST. +- **Conditional Query**: + - At the **start of DST**, the skipped time (`[03-31 02:00:00,03-31 03:00:00)`) does not exist, so using that time for queries results in undefined behavior: `SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'` (the nonexistent local timestamp is converted to `-1000`): + + ```sql + taos> SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'; + ts | + ================= + -1000 | + Query OK, 1 row(s) in set (0.003635s) + ``` + + When the nonexistent timestamp is used together with the existing timestamp, the result is also not as expected, as shown below where the start local time does not exist: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 03:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + -1000 | 1969-12-31T23:59:59.000Z | + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + 1711846800000 | 2024-03-31T01:00:00.000Z | + 1711846801000 | 2024-03-31T01:00:01.000Z | + Query OK, 5 row(s) in set (0.003339s) + ``` + + In the following statements, the first SQL query end time does not exist, and the second end time exists. The first SQL query result is not as expected: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 02:00:00'; + Query OK, 0 row(s) in set (0.000930s) + + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 01:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + Query OK, 2 row(s) in set (0.001227s) + ``` + + - At the end of DST, the repeated time (`[10-27 02:00:00,10-27 03:00:00)` excluding `10-27 03:00:00`) appears twice, and using that time range for queries in TDengine is also undefined behavior. + - Querying the data between `[2024-10-27 02:00:00, 2024-10-27 03:00:00]` includes the repeated timestamps and the data at `2024-10-27 03:00:00`: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts BETWEEN '2024-10-27 02:00:00' AND '2024-10-27 03:00:00'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + 1729994400000 | 2024-10-27T02:00:00.000Z | 2024-10-27 03:00:00 | + Query OK, 5 row(s) in set (0.001370s) + ``` + + - However, the following query for the range [2024-10-27 02:00:00.000,2024-10-27 02:57:34.999] can only find the data at the first 2024-10-27 02:00:00 time point: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:00.999'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 1 row(s) in set (0.004480s) + ``` + + - The following query for the range [2024-10-27 02:00:01,2024-10-27 02:57:35] can find 3 rows of data (including one row of local time data at 02:59:59): + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:35';; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + =============================================================================================== + 2024-10-27 02:00:00.000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 2024-10-27 02:59:59.000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 2024-10-27 02:00:00.000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 3 row(s) in set (0.004428s) + ``` + +## Summary and Suggestions + +### Summary + +This explanation only addresses the impact of using local time. Using UNIX timestamps or RFC3339 has no impact. + +- Writing: + - It is not possible to write data for nonexistent times during the DST transition. + - Writing data for repeated times during the DST transition is undefined behavior. +- Querying: + - Querying with conditions that specify the skipped time during the start of DST results in undefined behavior. + - Querying with conditions that specify the repeated time during the end of DST results in undefined behavior. +- Display: + - Displaying with time zones is not affected. + - Displaying local time is accurate, but repeated times during the end of DST cannot be distinguished. + - Users should be cautious when using time without time zones for display and application. + +### Suggestions + +To avoid unnecessary impacts of DST on querying and writing in TDengine, it is recommended to use explicit time offsets for writing and querying. + +- Use UNIX Timestamps: Using UNIX timestamps can avoid time zone issues. + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1711846799000 | 2024-03-31T00:59:59.000Z | 2024-03-31T01:59:59.000+01:00 | 2024-03-31 01:59:59 | + | 1711846800000 | 2024-03-31T01:00:00.000Z | 2024-03-31T03:00:00.000+02:00 | 2024-03-31 03:00:00 | + + ```sql + taos> insert into t1 values(1711846799000, 1)(1711846800000, 2); + Insert OK, 2 row(s) affected (0.001434s) + + taos> select * from t1 where ts between 1711846799000 and 1711846800000; + ts | v1 | + =============================== + 1711846799000 | 1 | + 1711846800000 | 2 | + Query OK, 2 row(s) in set (0.003503s) + ``` + +- Use RFC3339 Time Format: The RFC3339 time format with time zone offsets can effectively avoid the uncertainty of DST. + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27T02:00:00.000+02:00 | 2024-10-27 02:00:00 | + | 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27T02:59:59.000+02:00 | 2024-10-27 02:59:59 | + | 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27T02:00:00.000+01:00 | 2024-10-27 02:00:00 | + | 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27T02:59:59.000+01:00 | 2024-10-27 02:59:59 | + + ```sql + taos> insert into t1 values ('2024-10-27T02:00:00.000+02:00', 1) + ('2024-10-27T02:59:59.000+02:00', 2) + ('2024-10-27T02:00:00.000+01:00', 3) + ('2024-10-27T02:59:59.000+01:00', 4); + Insert OK, 4 row(s) affected (0.001514s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+01:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 3 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 4 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 4 row(s) in set (0.004275s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+02:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 2 row(s) in set (0.004275s) + ``` + +- Pay Attention to Time Zone Settings When Querying: When querying and displaying, if local time is needed, be sure to consider the impact of DST. + - taosAdapter: When using the REST API, it supports setting the IANA time zone, and the result is returned in RFC3339 format. + + ```shell + $ curl -uroot:taosdata 'localhost:6041/rest/sql?tz=Europe/Berlin'\ + -d "select ts from tz1.t1" + {"code":0,"column_meta":[["ts","TIMESTAMP",8]],"data":[["1970-01-01T00:59:59.000+01:00"],["2024-03-31T01:00:00.000+01:00"],["2024-03-31T01:59:59.000+01:00"],["2024-03-31T03:00:00.000+02:00"],["2024-03-31T03:00:01.000+02:00"],["2024-10-27T02:00:00.000+02:00"],["2024-10-27T02:59:59.000+02:00"],["2024-10-27T02:00:00.000+01:00"],["2024-10-27T02:59:59.000+01:00"],["2024-10-27T03:00:00.000+01:00"]],"rows":10} + ``` + + - Explorer: When using the Explorer page for SQL queries, users can configure the client time zone to display in RFC3339 format. + + ![Explorer DST](./02-dst/explorer-with-tz.png) + +## Reference Documents + +- IANA Time Zone Database: [https://www.iana.org/time-zones](https://www.iana.org/time-zones) +- RFC3339: [https://datatracker.ietf.org/doc/html/rfc3339](https://datatracker.ietf.org/doc/html/rfc3339) diff --git a/docs/en/27-train-faq/02-dst/dst-berlin.png b/docs/en/27-train-faq/02-dst/dst-berlin.png new file mode 100644 index 0000000000..fa1125de25 Binary files /dev/null and b/docs/en/27-train-faq/02-dst/dst-berlin.png differ diff --git a/docs/en/27-train-faq/02-dst/dst-table.png b/docs/en/27-train-faq/02-dst/dst-table.png new file mode 100644 index 0000000000..f5eddbe010 Binary files /dev/null and b/docs/en/27-train-faq/02-dst/dst-table.png differ diff --git a/docs/en/27-train-faq/02-dst/explorer-with-tz.png b/docs/en/27-train-faq/02-dst/explorer-with-tz.png new file mode 100644 index 0000000000..95758f117a Binary files /dev/null and b/docs/en/27-train-faq/02-dst/explorer-with-tz.png differ diff --git a/docs/en/assets/architecture-01.png b/docs/en/assets/architecture-01.png index a9e2bf9f70..71f6b06f5a 100644 Binary files a/docs/en/assets/architecture-01.png and b/docs/en/assets/architecture-01.png differ diff --git a/docs/en/assets/architecture-02.png b/docs/en/assets/architecture-02.png index e558a830ae..92580995c1 100644 Binary files a/docs/en/assets/architecture-02.png and b/docs/en/assets/architecture-02.png differ diff --git a/docs/en/assets/architecture-03.png b/docs/en/assets/architecture-03.png index 467580248c..60d786a782 100644 Binary files a/docs/en/assets/architecture-03.png and b/docs/en/assets/architecture-03.png differ diff --git a/docs/en/assets/architecture-04.png b/docs/en/assets/architecture-04.png index e034f97f89..06ecf72a26 100644 Binary files a/docs/en/assets/architecture-04.png and b/docs/en/assets/architecture-04.png differ diff --git a/docs/en/assets/data-compression-01.png b/docs/en/assets/data-compression-01.png index 64a36f4a42..31abb35926 100644 Binary files a/docs/en/assets/data-compression-01.png and b/docs/en/assets/data-compression-01.png differ diff --git a/docs/en/assets/data-querying-01.png b/docs/en/assets/data-querying-01.png index 381d6ec88e..36cc854fe2 100644 Binary files a/docs/en/assets/data-querying-01.png and b/docs/en/assets/data-querying-01.png differ diff --git a/docs/en/assets/data-querying-03.png b/docs/en/assets/data-querying-03.png index 416a14a0a9..b1f4024322 100644 Binary files a/docs/en/assets/data-querying-03.png and b/docs/en/assets/data-querying-03.png differ diff --git a/docs/en/assets/data-querying-04.png b/docs/en/assets/data-querying-04.png index fc83f00193..a249dac3c3 100644 Binary files a/docs/en/assets/data-querying-04.png and b/docs/en/assets/data-querying-04.png differ diff --git a/docs/en/assets/data-subscription-engine-01.png b/docs/en/assets/data-subscription-engine-01.png index d9306cdb74..088d0070be 100644 Binary files a/docs/en/assets/data-subscription-engine-01.png and b/docs/en/assets/data-subscription-engine-01.png differ diff --git a/docs/en/assets/data-subscription-engine-02.png b/docs/en/assets/data-subscription-engine-02.png index 0c9a061107..0bbd9ff0af 100644 Binary files a/docs/en/assets/data-subscription-engine-02.png and b/docs/en/assets/data-subscription-engine-02.png differ diff --git a/docs/en/assets/data-subscription-engine-03.png b/docs/en/assets/data-subscription-engine-03.png index 0ddb005d66..df61f32208 100644 Binary files a/docs/en/assets/data-subscription-engine-03.png and b/docs/en/assets/data-subscription-engine-03.png differ diff --git a/docs/en/assets/data-subscription-engine-04.png b/docs/en/assets/data-subscription-engine-04.png index 1bb4bb3014..921d6d7de6 100644 Binary files a/docs/en/assets/data-subscription-engine-04.png and b/docs/en/assets/data-subscription-engine-04.png differ diff --git a/docs/en/assets/data-subscription-engine-05.png b/docs/en/assets/data-subscription-engine-05.png index fbc2147c91..5e7867170c 100644 Binary files a/docs/en/assets/data-subscription-engine-05.png and b/docs/en/assets/data-subscription-engine-05.png differ diff --git a/docs/en/assets/data-subscription-engine-06.png b/docs/en/assets/data-subscription-engine-06.png index dd2b472890..256c6abc9c 100644 Binary files a/docs/en/assets/data-subscription-engine-06.png and b/docs/en/assets/data-subscription-engine-06.png differ diff --git a/docs/en/assets/data-subscription-engine-07.png b/docs/en/assets/data-subscription-engine-07.png index 6cd1670b64..978fa022ee 100644 Binary files a/docs/en/assets/data-subscription-engine-07.png and b/docs/en/assets/data-subscription-engine-07.png differ diff --git a/docs/en/assets/mqtt-05.png b/docs/en/assets/mqtt-05.png index 5345b3923f..5644330e14 100644 Binary files a/docs/en/assets/mqtt-05.png and b/docs/en/assets/mqtt-05.png differ diff --git a/docs/en/assets/query-engine-01.png b/docs/en/assets/query-engine-01.png index 08d42dc038..da850c3116 100644 Binary files a/docs/en/assets/query-engine-01.png and b/docs/en/assets/query-engine-01.png differ diff --git a/docs/en/assets/query-engine-02.png b/docs/en/assets/query-engine-02.png index ae7ab92c0a..dcb5360f62 100644 Binary files a/docs/en/assets/query-engine-02.png and b/docs/en/assets/query-engine-02.png differ diff --git a/docs/en/assets/storage-engine-01.png b/docs/en/assets/storage-engine-01.png index 40cba22780..f5cc5867bc 100644 Binary files a/docs/en/assets/storage-engine-01.png and b/docs/en/assets/storage-engine-01.png differ diff --git a/docs/en/assets/storage-engine-02.png b/docs/en/assets/storage-engine-02.png index c6ff24bccb..f201b4dbc4 100644 Binary files a/docs/en/assets/storage-engine-02.png and b/docs/en/assets/storage-engine-02.png differ diff --git a/docs/en/assets/storage-engine-03.png b/docs/en/assets/storage-engine-03.png index fdc252a4ab..d083906367 100644 Binary files a/docs/en/assets/storage-engine-03.png and b/docs/en/assets/storage-engine-03.png differ diff --git a/docs/en/assets/storage-engine-04.png b/docs/en/assets/storage-engine-04.png index 606bd4eaeb..d265fcd6a7 100644 Binary files a/docs/en/assets/storage-engine-04.png and b/docs/en/assets/storage-engine-04.png differ diff --git a/docs/en/assets/storage-engine-05.png b/docs/en/assets/storage-engine-05.png index 4211ac2343..36febdfc85 100644 Binary files a/docs/en/assets/storage-engine-05.png and b/docs/en/assets/storage-engine-05.png differ diff --git a/docs/en/assets/storage-engine-06.png b/docs/en/assets/storage-engine-06.png index d310929b31..4f8eaa1547 100644 Binary files a/docs/en/assets/storage-engine-06.png and b/docs/en/assets/storage-engine-06.png differ diff --git a/docs/en/assets/storage-engine-07.png b/docs/en/assets/storage-engine-07.png index 7c82e017e9..bfbf0c03d3 100644 Binary files a/docs/en/assets/storage-engine-07.png and b/docs/en/assets/storage-engine-07.png differ diff --git a/docs/en/assets/storage-engine-08.png b/docs/en/assets/storage-engine-08.png index ecd1262e94..a2c4201fe7 100644 Binary files a/docs/en/assets/storage-engine-08.png and b/docs/en/assets/storage-engine-08.png differ diff --git a/docs/en/assets/storage-engine-09.png b/docs/en/assets/storage-engine-09.png index d07acee6ae..304926b04e 100644 Binary files a/docs/en/assets/storage-engine-09.png and b/docs/en/assets/storage-engine-09.png differ diff --git a/docs/en/assets/storage-engine-10.png b/docs/en/assets/storage-engine-10.png index 475dbae2fb..deba103a86 100644 Binary files a/docs/en/assets/storage-engine-10.png and b/docs/en/assets/storage-engine-10.png differ diff --git a/docs/en/assets/storage-engine-11.png b/docs/en/assets/storage-engine-11.png index 4d942ad728..43a3d625c8 100644 Binary files a/docs/en/assets/storage-engine-11.png and b/docs/en/assets/storage-engine-11.png differ diff --git a/docs/en/assets/stream-processing-engine-01.png b/docs/en/assets/stream-processing-engine-01.png index c8a8087273..407eb3c542 100644 Binary files a/docs/en/assets/stream-processing-engine-01.png and b/docs/en/assets/stream-processing-engine-01.png differ diff --git a/docs/en/assets/stream-processing-engine-02.png b/docs/en/assets/stream-processing-engine-02.png index 16c3be56b9..04ecf582de 100644 Binary files a/docs/en/assets/stream-processing-engine-02.png and b/docs/en/assets/stream-processing-engine-02.png differ diff --git a/docs/en/assets/stream-processing-engine-03.png b/docs/en/assets/stream-processing-engine-03.png index d9fae4908d..138f4eda39 100644 Binary files a/docs/en/assets/stream-processing-engine-03.png and b/docs/en/assets/stream-processing-engine-03.png differ diff --git a/docs/zh/05-basic/03-query.md b/docs/zh/05-basic/03-query.md index 0b2f290667..52b825c47c 100644 --- a/docs/zh/05-basic/03-query.md +++ b/docs/zh/05-basic/03-query.md @@ -4,6 +4,10 @@ title: TDengine 数据查询 toc_max_heading_level: 4 --- +import win from './window.png'; +import swin from './session-window.png'; +import ewin from './event-window.png'; + 相较于其他众多时序数据库和实时数据库，TDengine 的一个独特优势在于，自其首个版本发布之初便支持标准的 SQL 查询功能。这一特性极大地降低了用户在使用过程中的学习难度。本章将以智能电表的数据模型为例介绍如何在 TDengine 中运用 SQL 查询来处理时序数据。如果需要进一步了解 SQL 语法的细节和功能，建议参阅 TDengine 的官方文档。通过本章的学习，你将能够熟练掌握 TDengine 的 SQL 查询技巧，进而高效地对时序数据进行操作和分析。 ## 基本查询 @@ -136,16 +140,15 @@ Query OK, 10 row(s) in set (2.415961s) 在 TDengine 中，你可以使用窗口子句来实现按时间窗口切分方式进行聚合结果查询，这种查询方式特别适用于需要对大量时间序列数据进行分析的场景，例如智能电表每 10s 采集一次数据，但需要查询每隔 1min 的温度平均值。 -窗口子句允许你针对查询的数据集合按照窗口进行切分，并对每个窗口内的数据进行聚合，包含： -- 时间窗口（time window） -- 状态窗口（status window） -- 会话窗口（session window） -- 事件窗口（event window） -- 计数窗口（count window） +窗口子句允许你针对查询的数据集合按照窗口进行切分，并对每个窗口内的数据进行聚合。窗口划分逻辑如下图所示。 -窗口划分逻辑如下图所示： +常用窗口划分逻辑 -![常用窗口划分逻辑](./window.png) +- 时间窗口（time window）：根据时间间隔划分数据，支持滑动时间窗口和翻转时间窗口，适用于按固定时间周期进行数据聚合。 +- 状态窗口（status window）：基于设备状态值的变化划分窗口，相同状态值的数据归为一个窗口，状态值改变时窗口关闭。 +- 会话窗口（session window）：根据记录的时间戳差异划分会话，时间戳间隔小于预设值的记录属于同一会话。 +- 事件窗口（event window）：基于事件的开始条件和结束条件动态划分窗口，满足开始条件时窗口开启，满足结束条件时窗口关闭。 +- 计数窗口（count window）：根据数据行数划分窗口，每达到指定行数即为一个窗口，并进行聚合计算。 窗口子句语法如下： @@ -408,7 +411,8 @@ Query OK, 22 row(s) in set (0.153403s) 会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示，如果设置时间戳的连续的间隔小于等于 12 秒，则以下 6 条记录构成 2 个会话窗口，分别是：[2019-04-28 14:22:10，2019-04-28 14:22:30] 和 [2019-04-28 14:23:10，2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒，超过了连续时间间隔（12 秒）。 -![会话窗口示意图](./session-window.png) +会话窗口示意图 + 在 tol_value 时间间隔范围内的结果都认为归属于同一个窗口，如果连续的两条记录的时间超过 tol_val，则自动开启下一个窗口。 @@ -461,7 +465,7 @@ Query OK, 10 row(s) in set (0.043489s) select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c2 < 10 ``` -![事件窗口示意图](./event-window.png) +事件窗口示意图 示例 SQL: diff --git a/docs/zh/05-basic/event-window.png b/docs/zh/05-basic/event-window.png index 3a2a628ffe..a249dac3c3 100644 Binary files a/docs/zh/05-basic/event-window.png and b/docs/zh/05-basic/event-window.png differ diff --git a/docs/zh/05-basic/session-window.png b/docs/zh/05-basic/session-window.png index 416a14a0a9..b1f4024322 100644 Binary files a/docs/zh/05-basic/session-window.png and b/docs/zh/05-basic/session-window.png differ diff --git a/docs/zh/05-basic/window.png b/docs/zh/05-basic/window.png index 5e3efb8a3b..a6f5883917 100644 Binary files a/docs/zh/05-basic/window.png and b/docs/zh/05-basic/window.png differ diff --git a/docs/zh/06-advanced/06-TDgpt/07-faq.md b/docs/zh/06-advanced/06-TDgpt/07-faq.md index 9747dc18f4..4cfb5abf06 100644 --- a/docs/zh/06-advanced/06-TDgpt/07-faq.md +++ b/docs/zh/06-advanced/06-TDgpt/07-faq.md @@ -3,7 +3,12 @@ title: "常见问题" sidebar_label: "常见问题" --- -1. 创建 anode 失败，返回指定服务无法访问 +### 1. 安装过程中编译 uWSGI 失败，如何处理 +TDgpt 安装过程中需要在本地编译 uWSGI，某些环境的 Python（例如：anaconda）安装 uWSGI 会出现冲突导致编译失败，安装流程因此无法继续下去。这种情况下可以尝试在安装过程中忽略 uWSGI的安装。 +由于忽略了 uWSGI 安装，后续启动 taosasnode 服务的时候，需要手动输入命令进行启动 `python3.10 /usr/local/taos/taosanode/lib/taosanalytics/app.py` 。 执行该命令的时候请确保使用了虚拟环境中的 Python 程序才能加载依赖库。 + + +### 2. 创建 anode 失败，返回指定服务无法访问 ```bash taos> create anode '127.0.0.1:6090'; @@ -26,7 +31,7 @@ curl: (7) Failed to connect to 127.0.0.1 port 6090: Connection refused >请勿使用 systemctl status taosanode 检查 taosanode 是否正常 -2. 服务正常，查询过程返回服务不可用 +### 3. 服务正常，查询过程返回服务不可用 ```bash taos> select _frowts,forecast(current, 'algo=arima, alpha=95, wncheck=0, rows=20') from d1 where ts<='2017-07-14 10:40:09.999'; @@ -34,7 +39,7 @@ DB error: Analysis service can't access[0x80000441] (60.195613s) ``` 数据分析默认超时时间是 60s，出现这个问题的原因是输入数据分析过程超过默认的最长等待时间，请尝试采用限制数据输入范围的方式将输入数据规模减小或者更换分析算法再次尝试。 -3. 返回结果出现非法 JSON 格式错误 (Invalid json format) +### 4. 返回结果出现非法 JSON 格式错误 (Invalid json format) 从 anode 返回到 TDengine 的分析结果有误，请检查 anode 运行日志 `/var/log/taos/taosanode/taosanode.app.log`，以便于获得具体的错误信息。 diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 2216cab915..1e6667b31f 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -147,7 +147,7 @@ taosd 命令行参数如下 - 类型：整数 - 单位：毫秒 - 默认值：10000 -- 最小值：0 +- 最小值：3000 - 最大值：86400000 - 动态修改：支持通过 SQL 修改，重启后生效 - 支持版本：v3.3.4.0 引入 diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 9c8e2ebb20..ca7fb18b9f 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -65,7 +65,6 @@ ASIN(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - #### ATAN ```sql @@ -84,7 +83,6 @@ ATAN(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - #### CEIL ```sql @@ -121,6 +119,66 @@ COS(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 +#### DEGREES + +```sql +DEGREES(expr) +``` + +**功能说明**：计算指定参数由弧度值转为角度后的值。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，则返回 NULL。 +- degree = radian * 180 / π。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select degrees(PI()); + degrees(pi()) | +============================ + 180.000000000000000 | +``` + +#### EXP + +```sql +EXP(expr) +``` +**功能说明**：返回 e（自然对数的底）的指定乘方后的值。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select exp(2); + exp(2) | +============================ + 7.389056098930650 | +``` + #### FLOOR ```sql @@ -130,6 +188,37 @@ FLOOR(expr) **功能说明**：获得指定字段的向下取整数的结果。 其他使用说明参见 CEIL 函数描述。 +#### LN + +```sql +LN(expr) +``` + +**功能说明**：返回指定参数的自然对数。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 如果 `epxr` 小于等于 0，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select ln(10); + ln(10) | +============================ + 2.302585092994046 | +``` + #### LOG ```sql @@ -148,6 +237,71 @@ LOG(expr1[, expr2]) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 +#### MOD + +```sql +MOD(expr1, expr2) +``` + +**功能说明**：计算 expr1 % expr2 的结果。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr2` 为 0 则返回 NULL。 +- 如果 `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +``` sql +taos> select mod(10,3); + mod(10,3) | +============================ + 1.000000000000000 | + +taos> select mod(1,0); + mod(1,0) | +============================ + NULL | +``` + +#### PI + +```sql +PI() +``` + +**功能说明**：返回圆周率 π 的值。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：无。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- π ≈ 3.141592653589793。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select pi(); + pi() | +============================ + 3.141592653589793 | +``` #### POW @@ -167,8 +321,85 @@ POW(expr1, expr2) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 +#### RADIANS + +```sql +RADIANS(expr) +``` + +**功能说明**：计算指定参数由角度值转为弧度后的值。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，则返回 NULL。 +- radian = degree * π / 180。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select radians(180); + radians(180) | +============================ + 3.141592653589793 | +``` + +#### RAND + +```sql +RAND([seed]) +``` + +**功能说明**：返回一个从0到1均匀分布的随机数。 + +**版本**：v3.3.3.0 + +**返回结果类型**：DOUBLE。 + +**适用数据类型**： +- `seed`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果指定了 `seed` 值，那么将会用指定的 `seed` 作为随机种子，确保生成的随机数序列具有确定性。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +``` sql +taos> select rand(); + rand() | +============================ + 0.202092426923147 | + +taos> select rand(); + rand() | +============================ + 0.131537788143166 | + +taos> select rand(1); + rand(1) | +============================ + 0.000007826369259 | + +taos> select rand(1); + rand(1) | +============================ + 0.000007826369259 | +``` #### ROUND + ```sql ROUND(expr[, digits]) ``` @@ -208,6 +439,49 @@ taos> select round(8888.88,-1); 8890.000000000000000 | ``` +#### SIGN + +```sql +SIGN(expr) +``` + +**功能说明**：返回指定参数的符号。 + +**版本**：v3.3.3.0 + +**返回结果类型**：与指定字段的原始数据类型一致。 + +**适用数据类型**：数值类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为负，返回 -1。 +- 如果 `expr` 为正，返回 1。 +- 如果 `expr` 为 0，返回 0。 +- 如果 `expr` 为 NULL，返回 NULL。 +- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 + +**举例**： +```sql +taos> select sign(-1); + sign(-1) | +======================== + -1 | + +taos> select sign(1); + sign(1) | +======================== + 1 | + +taos> select sign(0); + sign(0) | +======================== + 0 | +``` + #### SIN ```sql @@ -262,36 +536,8 @@ TAN(expr) **使用说明**：只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 -#### PI -```sql -PI() -``` - -**功能说明**：返回圆周率 π 的值。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：无。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- π ≈ 3.141592653589793。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select pi(); - pi() | -============================ - 3.141592653589793 | -``` - ##### TRUNCATE + ```sql TRUNCATE(expr, digits) ``` @@ -332,249 +578,88 @@ taos> select truncate(8888.88, -1); 8880.000000000000000 | ``` -#### EXP -```sql -EXP(expr) -``` -**功能说明**：返回 e（自然对数的底）的指定乘方后的值。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，返回 NULL。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select exp(2); - exp(2) | -============================ - 7.389056098930650 | -``` - -#### LN -```sql -LN(expr) -``` - -**功能说明**：返回指定参数的自然对数。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，返回 NULL。 -- 如果 `epxr` 小于等于 0，返回 NULL。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select ln(10); - ln(10) | -============================ - 2.302585092994046 | -``` - -#### MOD -```sql -MOD(expr1, expr2) -``` - -**功能说明**：计算 expr1 % expr2 的结果。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr2` 为 0 则返回 NULL。 -- 如果 `expr1` 或 `expr2` 为 NULL，返回 NULL。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -``` sql -taos> select mod(10,3); - mod(10,3) | -============================ - 1.000000000000000 | - -taos> select mod(1,0); - mod(1,0) | -============================ - NULL | -``` - -#### RAND -```sql -RAND([seed]) -``` - -**功能说明**：返回一个从0到1均匀分布的随机数。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**： -- `seed`：INTEGER。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果指定了 `seed` 值，那么将会用指定的 `seed` 作为随机种子，确保生成的随机数序列具有确定性。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -``` sql -taos> select rand(); - rand() | -============================ - 0.202092426923147 | - -taos> select rand(); - rand() | -============================ - 0.131537788143166 | - -taos> select rand(1); - rand(1) | -============================ - 0.000007826369259 | - -taos> select rand(1); - rand(1) | -============================ - 0.000007826369259 | -``` - -#### SIGN -```sql -SIGN(expr) -``` - -**功能说明**：返回指定参数的符号。 - -**版本**：v3.3.3.0 - -**返回结果类型**：与指定字段的原始数据类型一致。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为负，返回 -1。 -- 如果 `expr` 为正，返回 1。 -- 如果 `expr` 为 0，返回 0。 -- 如果 `expr` 为 NULL，返回 NULL。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select sign(-1); - sign(-1) | -======================== - -1 | - -taos> select sign(1); - sign(1) | -======================== - 1 | - -taos> select sign(0); - sign(0) | -======================== - 0 | -``` - -#### DEGREES -```sql -DEGREES(expr) -``` - -**功能说明**：计算指定参数由弧度值转为角度后的值。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，则返回 NULL。 -- degree = radian * 180 / π。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select degrees(PI()); - degrees(pi()) | -============================ - 180.000000000000000 | -``` - -#### RADIANS -```sql -RADIANS(expr) -``` - -**功能说明**：计算指定参数由角度值转为弧度后的值。 - -**版本**：v3.3.3.0 - -**返回结果类型**：DOUBLE。 - -**适用数据类型**：数值类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，则返回 NULL。 -- radian = degree * π / 180。 -- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用。 - -**举例**： -```sql -taos> select radians(180); - radians(180) | -============================ - 3.141592653589793 | -``` ### 字符串函数 字符串函数的输入参数为字符串类型，返回结果为数值类型或字符串类型。 +#### ASCII + +```sql +ASCII(expr) +``` + +**功能说明**：返回字符串第一个字符的 ASCII 码。 + +**版本**：v3.3.3.0 + +**返回结果数据类型**：BIGINT。 + +**适用数据类型**：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 如果 `expr` 为 NULL，返回 NULL。 +- 如果 `expr` 的第一个字符为多字节字符，只会返回该字符第一个字节的值对应的 ASCII 码。 + +**举例**： +```sql +taos> select ascii('testascii'); + ascii('testascii') | +===================== + 116 | +``` + +#### CHAR + +```sql +CHAR(expr1 [, expr2] [, epxr3] ...) +``` + +**功能说明**：将输入参数当作整数，并返回这些整数在 ASCII 编码中对应的字符。 + +**版本**：v3.3.3.0 + +**返回结果类型**：VARCHAR。 + +**适用数据类型**：整数类型,VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 输入的值超过 255 会被转化成多字节的结果，如 `CHAR(256)` 等同于 `CHAR(1,0)`、`CHAR(256 * 256)` 等同于 `CHAR(1,0,0)`。 +- 输入参数的 NULL 值会被跳过。 +- 输入参数若为字符串类型，会将其转换为数值类型处理。 +- 若输入的参数对应的字符为不可打印字符，返回值中仍有该参数对应的字符，但是可能无法显示出来。 +- 输入参数的个数上限为 2^31 - 1 个。 + +**举例**： +```sql +taos> select char(77); + char(77) | +=========== + M | + +taos> select char(77,77); + char(77,77) | +============== + MM | + +taos> select char(77 * 256 + 77); + char(77 * 256 + 77) | +====================== + MM | + +taos> select char(77,NULL,77); + char(77,null,77) | +=================== + MM | +``` + #### CHAR_LENGTH ```sql @@ -628,7 +713,6 @@ CONCAT(expr1, expr2 [, expr] ... ) **适用于**：表和超级表。 - #### CONCAT_WS ```sql @@ -645,7 +729,6 @@ CONCAT_WS(separator_expr, expr1, expr2 [, expr] ...) **适用于**：表和超级表。 - #### LENGTH ```sql @@ -662,7 +745,6 @@ LENGTH(expr) **适用于**：表和超级表。 - #### LOWER ```sql @@ -696,6 +778,120 @@ LTRIM(expr) **适用于**：表和超级表。 +#### POSITION + +```sql +POSITION(expr1 IN expr2) +``` + +**功能说明**：计算字符串 `expr1` 在字符串 `expr2` 中的位置。 + +**版本**：v3.3.3.0 + +**返回结果类型**：BIGINT。 + +**适用数据类型**： +- `expr1`：VARCHAR、NCHAR。 +- `expr2`：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 若 `expr1` 或 `expr2` 为 NULL，返回 NULL。 +- 若 `expr1` 在 `expr2` 中不存在，返回 0。 +- 若 `expr1` 为空串，认为 `expr1` 在 `expr2` 中总能匹配成功，返回 1。 +- 返回的位置是 1-base 的。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select position('a' in 'cba'); + position('a' in 'cba') | +========================= + 3 | + + +taos> select position('' in 'cba'); + position('' in 'cba') | +======================== + 1 | + +taos> select position('d' in 'cba'); + position('d' in 'cba') | +========================= + 0 | +``` + +#### REPEAT + +```sql +REPEAT(expr, count) +``` +**功能说明**：返回将字符串重复指定次数得到的字符串。 + +**版本**：v3.3.3.0 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR、NCHAR。 +- `count`：INTEGER。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 若 `count < 1`，返回空串。 +- 若 `expr` 或 `count` 为 NULL，返回 NULL。 + +**举例**： +```sql +taos> select repeat('abc',5); + repeat('abc',5) | +============================ + abcabcabcabcabc | + +taos> select repeat('abc',-1); + repeat('abc',-1) | +=================== + | +``` + +#### REPLACE + +```sql +REPLACE(expr, from_str, to_str) +``` +**功能说明**：将字符串中的 `from_str` 全部替换为 `to_str`。 + +**版本**：v3.3.3.0 + +**返回结果类型**：与输入字段 `expr` 的原始类型相同。 + +**适用数据类型**： +- `expr`：VARCHAR、NCHAR。 +- `from_str`：VARCHAR、NCHAR。 +- `to_str`：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 该函数是大小写敏感的。 +- 任意参数为 NULL，返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select replace('aabbccAABBCC', 'AA', 'DD'); + replace('aabbccAABBCC', 'AA', 'DD') | +====================================== + aabbccDDBBCC | +``` #### RTRIM @@ -713,62 +909,8 @@ RTRIM(expr) **适用于**：表和超级表。 -#### TRIM -```sql -TRIM([{LEADING | TRAILING | BOTH} [remstr] FROM] expr) -TRIM([remstr FROM] expr) -``` - -**功能说明**：返回去掉了所有 remstr 前缀或后缀的字符串 epxr。 - -**版本**：v3.3.3.0 - -**返回结果类型**：与输入字段 epxr 的原始类型相同。 - -**适用数据类型**： -- remstr：VARCHAR、NCHAR。 -- epxr：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 第一个可选变量 [LEADING | BOTH | TRAILING] 指定要剪裁字符串的哪一侧： - - LEADING 将移除字符串开头的指定字符。 - - TRAILING 将移除字符串末尾的指定字符。 - - BOTH（默认值）将移除字符串开头和末尾的指定字符。 -- 第二个可选变量[remstr]指定要裁剪掉的字符串： - - 如果不指定 remstr，默认裁剪空格。 - - remstr 可以指定多个字符，如 trim('ab' from 'abacd')，此时会将 'ab' 看做一个整体来裁剪，得到裁剪结果 'acd'。 -- 若 expr 为 NULL，返回 NULL。 -- 该函数是多字节安全的。 - -**举例**： -```sql -taos> select trim(' a '); - trim(' a ') | -============================= - a | - -taos> select trim(leading from ' a '); - trim(leading from ' a ') | -========================================== - a | - - -taos> select trim(leading 'b' from 'bbbbbbbba '); - trim(leading 'b' from 'bbbbbbbba ') | -============================================== - a | - -taos> select trim(both 'b' from 'bbbbbabbbbbb'); - trim(both 'b' from 'bbbbbabbbbbb') | -===================================== - a | -``` - #### SUBSTRING/SUBSTR + ```sql SUBSTRING/SUBSTR(expr, pos [, len]) SUBSTRING/SUBSTR(expr FROM pos [FOR len]) @@ -826,6 +968,7 @@ taos> select substring('tdengine', -3,-3); ``` #### SUBSTRING_INDEX + ```sql SUBSTRING_INDEX(expr, delim, count) ``` @@ -864,6 +1007,62 @@ taos> select substring_index('www.taosdata.com','.',-2); taosdata.com | ``` +#### TRIM + +```sql +TRIM([{LEADING | TRAILING | BOTH} [remstr] FROM] expr) +TRIM([remstr FROM] expr) +``` + +**功能说明**：返回去掉了所有 remstr 前缀或后缀的字符串 epxr。 + +**版本**：v3.3.3.0 + +**返回结果类型**：与输入字段 epxr 的原始类型相同。 + +**适用数据类型**： +- remstr：VARCHAR、NCHAR。 +- epxr：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 第一个可选变量[LEADING | BOTH | TRAILING]指定要剪裁字符串的哪一侧： + - LEADING 将移除字符串开头的指定字符。 + - TRAILING 将移除字符串末尾的指定字符。 + - BOTH（默认值）将移除字符串开头和末尾的指定字符。 +- 第二个可选变量[remstr]指定要裁剪掉的字符串： + - 如果不指定 remstr，默认裁剪空格。 + - remstr 可以指定多个字符，如trim('ab' from 'abacd')，此时会将 'ab' 看做一个整体来裁剪，得到裁剪结果 'acd'。 +- 若 expr 为 NULL，返回 NULL。 +- 该函数是多字节安全的。 + +**举例**： +```sql +taos> select trim(' a '); + trim(' a ') | +============================= + a | + +taos> select trim(leading from ' a '); + trim(leading from ' a ') | +========================================== + a | + + +taos> select trim(leading 'b' from 'bbbbbbbba '); + trim(leading 'b' from 'bbbbbbbba ') | +============================================== + a | + +taos> select trim(both 'b' from 'bbbbbabbbbbb'); + trim(both 'b' from 'bbbbbabbbbbb') | +===================================== + a | +``` + #### UPPER ```sql @@ -880,193 +1079,6 @@ UPPER(expr) **适用于**：表和超级表。 -#### CHAR -```sql -CHAR(expr1 [, expr2] [, epxr3] ...) -``` - -**功能说明**：将输入参数当作整数，并返回这些整数在 ASCII 编码中对应的字符。 - -**版本**：v3.3.3.0 - -**返回结果类型**：VARCHAR。 - -**适用数据类型**：整数类型,VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 输入的值超过 255 会被转化成多字节的结果，如 `CHAR(256)` 等同于 `CHAR(1,0)`、`CHAR(256 * 256)` 等同于 `CHAR(1,0,0)`。 -- 输入参数的 NULL 值会被跳过。 -- 输入参数若为字符串类型，会将其转换为数值类型处理。 -- 若输入的参数对应的字符为不可打印字符，返回值中仍有该参数对应的字符，但是可能无法显示出来。 -- 输入参数的个数上限为 2^31 - 1 个。 - -**举例**： -```sql -taos> select char(77); - char(77) | -=========== - M | - -taos> select char(77,77); - char(77,77) | -============== - MM | - -taos> select char(77 * 256 + 77); - char(77 * 256 + 77) | -====================== - MM | - -taos> select char(77,NULL,77); - char(77,null,77) | -=================== - MM | -``` - -#### ASCII -```sql -ASCII(expr) -``` - -**功能说明**：返回字符串第一个字符的 ASCII 码。 - -**版本**：v3.3.3.0 - -**返回结果数据类型**：BIGINT。 - -**适用数据类型**：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 如果 `expr` 为 NULL，返回 NULL。 -- 如果 `expr` 的第一个字符为多字节字符，只会返回该字符第一个字节的值对应的 ASCII 码。 - -**举例**： -```sql -taos> select ascii('testascii'); - ascii('testascii') | -===================== - 116 | -``` - -#### POSITION -```sql -POSITION(expr1 IN expr2) -``` - -**功能说明**：计算字符串 `expr1` 在字符串 `expr2` 中的位置。 - -**版本**：v3.3.3.0 - -**返回结果类型**：BIGINT。 - -**适用数据类型**： -- `expr1`：VARCHAR、NCHAR。 -- `expr2`：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 若 `expr1` 或 `expr2` 为 NULL，返回 NULL。 -- 若 `expr1` 在 `expr2` 中不存在，返回 0。 -- 若 `expr1` 为空串，认为 `expr1` 在 `expr2` 中总能匹配成功，返回 1。 -- 返回的位置是 1-base 的。 -- 该函数是多字节安全的。 - -**举例**： -```sql -taos> select position('a' in 'cba'); - position('a' in 'cba') | -========================= - 3 | - - -taos> select position('' in 'cba'); - position('' in 'cba') | -======================== - 1 | - -taos> select position('d' in 'cba'); - position('d' in 'cba') | -========================= - 0 | -``` - -#### REPLACE -```sql -REPLACE(expr, from_str, to_str) -``` -**功能说明**：将字符串中的 `from_str` 全部替换为 `to_str`。 - -**版本**：v3.3.3.0 - -**返回结果类型**：与输入字段 `expr` 的原始类型相同。 - -**适用数据类型**： -- `expr`：VARCHAR、NCHAR。 -- `from_str`：VARCHAR、NCHAR。 -- `to_str`：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 该函数是大小写敏感的。 -- 任意参数为 NULL，返回 NULL。 -- 该函数是多字节安全的。 - -**举例**： -```sql -taos> select replace('aabbccAABBCC', 'AA', 'DD'); - replace('aabbccAABBCC', 'AA', 'DD') | -====================================== - aabbccDDBBCC | -``` - -#### REPEAT -```sql -REPEAT(expr, count) -``` -**功能说明**：返回将字符串重复指定次数得到的字符串。 - -**版本**：v3.3.3.0 - -**返回结果类型**：与输入字段 `expr` 的原始类型相同。 - -**适用数据类型**： -- `expr`：VARCHAR、NCHAR。 -- `count`：INTEGER。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 若 `count < 1`，返回空串。 -- 若 `expr` 或 `count` 为 NULL，返回 NULL。 - -**举例**： -```sql -taos> select repeat('abc',5); - repeat('abc',5) | -============================ - abcabcabcabcabc | - -taos> select repeat('abc',-1); - repeat('abc',-1) | -=================== - | -``` ### 转换函数 转换函数将值从一种数据类型转换为另一种数据类型。 @@ -1095,72 +1107,6 @@ CAST(expr AS type_name) - 转换到数值类型时，数值大于 type_name 可表示的范围时，则会溢出，但不会报错。 - 转换到字符串类型时，如果转换后长度超过 type_name 中指定的长度，则会截断，但不会报错。 -#### TO_ISO8601 - -```sql -TO_ISO8601(expr [, timezone]) -``` - -**功能说明**：将时间戳转换成为 ISO8601 标准的日期时间格式，并附加时区信息。timezone 参数允许用户为输出结果指定附带任意时区信息。如果 timezone 参数省略，输出结果则附带当前客户端的系统时区信息。 - -**返回结果数据类型**：VARCHAR 类型。 - -**适用数据类型**：INTEGER、TIMESTAMP。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： - -- timezone 参数允许输入的时区格式为：[z/Z, +/-hhmm, +/-hh, +/-hh:mm]。例如，TO_ISO8601(1, "+00:00")。 -- 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒. - - -#### TO_JSON - -```sql -TO_JSON(str_literal) -``` - -**功能说明**：将字符串常量转换为 JSON 类型。 - -**返回结果数据类型**：JSON。 - -**适用数据类型**：JSON 字符串，形如 '\{ "literal" : literal }'。'\{}'表示空值。键必须为字符串字面量，值可以为数值字面量、字符串字面量、布尔字面量或空值字面量。str_literal 中不支持转义符。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - - -#### TO_UNIXTIMESTAMP - -```sql -TO_UNIXTIMESTAMP(expr [, return_timestamp]) - -return_timestamp: { - 0 - | 1 -} -``` - -**功能说明**：将日期时间格式的字符串转换成为时间戳。 - -**返回结果数据类型**：BIGINT、TIMESTAMP。 - -**应用字段**：VARCHAR、NCHAR。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： - -- 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 NULL。 -- 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 -- return_timestamp 指定函数返回值是否为时间戳类型，设置为 1 时返回 TIMESTAMP 类型，设置为 0 时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。 - #### TO_CHAR ```sql @@ -1222,6 +1168,43 @@ TO_CHAR(ts, format_str_literal) - 推荐在时间格式中带时区信息，如果不带则默认输出的时区为服务端或客户端所配置的时区。 - 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒。 +#### TO_ISO8601 + +```sql +TO_ISO8601(expr [, timezone]) +``` + +**功能说明**：将时间戳转换成为 ISO8601 标准的日期时间格式，并附加时区信息。timezone 参数允许用户为输出结果指定附带任意时区信息。如果 timezone 参数省略，输出结果则附带当前客户端的系统时区信息。 + +**返回结果数据类型**：VARCHAR 类型。 + +**适用数据类型**：INTEGER、TIMESTAMP。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： + +- timezone 参数允许输入的时区格式为：[z/Z, +/-hhmm, +/-hh, +/-hh:mm]。例如，TO_ISO8601(1, "+00:00")。 +- 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒. + +#### TO_JSON + +```sql +TO_JSON(str_literal) +``` + +**功能说明**：将字符串常量转换为 JSON 类型。 + +**返回结果数据类型**：JSON。 + +**适用数据类型**：JSON 字符串，形如 '\{ "literal" : literal }'。'\{}'表示空值。键必须为字符串字面量，值可以为数值字面量、字符串字面量、布尔字面量或空值字面量。str_literal 中不支持转义符。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + #### TO_TIMESTAMP ```sql @@ -1252,6 +1235,32 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal) - `to_timestamp` 转换具有一定的容错机制，在格式串和时间戳串不完全对应时，有时也可转换，如 `to_timestamp('200101/2', 'yyyyMM1/dd')`，格式串中多出来的1会被丢弃。格式串与时间戳串中多余的空格字符(空格、tab 等)也会被自动忽略，如 `to_timestamp(' 23 年 - 1 月 - 01 日 ', 'yy 年-MM月-dd日')` 可以被成功转换。虽然 `MM` 等字段需要两个数字对应(只有一位时前面补 0), 在 `to_timestamp` 时，一个数字也可以成功转换。 - 输出时间戳的精度与查询表的精度相同，若查询未指定表，则输出精度为毫秒，如 `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')` 的输出将会把微秒和纳秒进行截断、如果指定一张纳秒表，那么就不会发生截断，如 `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`。 +#### TO_UNIXTIMESTAMP + +```sql +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} +``` + +**功能说明**：将日期时间格式的字符串转换成为时间戳。 + +**返回结果数据类型**：BIGINT、TIMESTAMP。 + +**应用字段**：VARCHAR、NCHAR。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： + +- 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 NULL。 +- 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 +- return_timestamp 指定函数返回值是否为时间戳类型，设置为 1 时返回 TIMESTAMP 类型，设置为 0 时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。 ### 时间和日期函数 @@ -1259,6 +1268,35 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal) 所有返回当前时间的函数，如 NOW、TODAY 和 TIMEZONE，在一条 SQL 语句中不论出现多少次都只会被计算一次。 +#### DAYOFWEEK +```sql +DAYOFWEEK(expr) +``` +**功能说明**：返回输入日期是周几。 + +**版本**：v3.3.3.0 + +**返回结果类型**：BIGINT。 + +**适用数据类型**：表示时间戳的 BIGINT、TIMESTAMP 类型，或符合 ISO8601/RFC3339 标准的日期时间格式的 VARCHAR、NCHAR 类型。 + +**嵌套子查询支持**：适用于内层查询和外层查询。 + +**适用于**：表和超级表。 + +**使用说明**： +- 返回值 1 代表周日，2 代表周一 ... 7 代表周六 +- 若 `expr` 为 NULL，返回 NULL。 +- 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒。 + +**举例**： +```sql +taos> select dayofweek('2000-01-01'); + dayofweek('2000-01-01') | +========================== + 7 | +``` + #### NOW ```sql @@ -1281,7 +1319,6 @@ NOW() b(纳秒)、u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 - #### TIMEDIFF ```sql @@ -1355,7 +1392,6 @@ use_current_timezone: { - 当将时间值截断到一周（1w）时，timetruncate 的计算是基于 Unix 时间戳（1970年1月1日00:00:00 UTC）进行的。Unix 时间戳始于星期四， 因此所有截断后的日期都是星期四。 - #### TIMEZONE ```sql @@ -1370,7 +1406,6 @@ TIMEZONE() **适用于**：表和超级表。 - #### TODAY ```sql @@ -1392,6 +1427,7 @@ TODAY() - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 #### WEEK + ```sql WEEK(expr [, mode]) ``` @@ -1454,35 +1490,6 @@ taos> select week('2000-01-01',3); 52 | ``` -#### WEEKOFYEAR -```sql -WEEKOFYEAR(expr) -``` -**功能说明**：返回输入日期的周数。 - -**版本**：v3.3.3.0 - -**返回结果类型**：BIGINT。 - -**适用数据类型**：表示时间戳的 BIGINT、TIMESTAMP 类型，或符合 ISO8601/RFC3339 标准的日期时间格式的 VARCHAR、NCHAR 类型。 - -**嵌套子查询支持**：适用于内层查询和外层查询。 - -**适用于**：表和超级表。 - -**使用说明**： -- 等同于`WEEK(expr, 3)`，即在每周第一天是周一，返回值范围为 1-53，第一个包含四天及以上的周为第 1 周的条件下判断输入日期的周数。 -- 若 `expr` 为 NULL，返回 NULL。 -- 输入时间戳的精度由所查询表的精度确定，未未指定表，则精度为毫秒。 - -**举例**： -```sql -taos> select weekofyear('2000-01-01'); - weekofyear('2000-01-01') | -=========================== - 52 | -``` - #### WEEKDAY ```sql WEEKDAY(expr) @@ -1512,11 +1519,12 @@ taos> select weekday('2000-01-01'); 5 | ``` -#### DAYOFWEEK +#### WEEKOFYEAR + ```sql -DAYOFWEEK(expr) +WEEKOFYEAR(expr) ``` -**功能说明**：返回输入日期是周几。 +**功能说明**：返回输入日期的周数。 **版本**：v3.3.3.0 @@ -1529,19 +1537,18 @@ DAYOFWEEK(expr) **适用于**：表和超级表。 **使用说明**： -- 返回值 1 代表周日，2 代表周一 ... 7 代表周六。 +- 等同于`WEEK(expr, 3)`，即在每周第一天是周一，返回值范围为 1 - 53，第一个包含四天及以上的周为第 1 周的条件下判断输入日期的周数。 - 若 `expr` 为 NULL，返回 NULL。 -- 输入时间戳的精度由所查询表的精度确定，若未指定表，则精度为毫秒。 - +- 输入时间戳的精度由所查询表的精度确定，未指定表，则精度为毫秒。 + **举例**： ```sql -taos> select dayofweek('2000-01-01'); - dayofweek('2000-01-01') | -========================== - 7 | +taos> select weekofyear('2000-01-01'); + weekofyear('2000-01-01') | +=========================== + 52 | ``` - ## 聚合函数 聚合函数为查询结果集的每一个分组返回单个结果行。可以由 GROUP BY 或窗口切分子句指定分组，如果没有，则整个查询结果集视为一个分组。 @@ -1586,7 +1593,6 @@ AVG(expr) **适用于**：表和超级表。 - ### COUNT ```sql @@ -1606,7 +1612,6 @@ COUNT({* | expr}) - 可以使用星号 (\*) 来替代具体的字段，使用星号 (\*) 返回全部记录数量。 - 如果统计字段是具体的列，则返回该列中非 NULL 值的记录数量。 - ### ELAPSED ```sql @@ -1632,6 +1637,50 @@ ELAPSED(ts_primary_key [, time_unit]) - 对于嵌套查询，仅当内层查询会输出隐式时间戳列时有效。例如 `select elapsed(ts) from (select diff(value) from sub1)` 语句，diff 函数会让内层查询输出隐式时间戳列，此为主键列，可以用于 elapsed 函数的第一个参数。相反，例如 `select elapsed(ts) from (select * from sub1)` 语句，ts 列输出到外层时已经没有了主键列的含义，无法使用 elapsed 函数。此外，elapsed 函数作为一个与时间线强依赖的函数，形如 `select elapsed(ts) from (select diff(value) from st group by tbname)` 尽管会返回一条计算结果，但并无实际意义，这种用法后续也将被限制。 - 不支持与 leastsquares、diff、derivative、top、bottom、last_row、interp 等函数混合使用。 +### HISTOGRAM + +```sql +HISTOGRAM(expr，bin_type, bin_description, normalized) +``` + +**功能说明**：统计数据按照用户指定区间的分布。 + +**返回结果类型**：如归一化参数 normalized 设置为 1，返回结果为 DOUBLE 类型，否则为 BIGINT 类型。 + +**适用数据类型**：数值型字段。 + +**适用于**：表和超级表。 + +**详细说明**： +- bin_type 用户指定的分桶类型，有效输入类型为 "user_input"、"linear_bin"、"log_bin"。 +- bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)： + - "user_input": "[1, 3, 5, 7]" + 用户指定 bin 的具体数值。 + + - "linear_bin": "\{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" + "start" 表示数据起始点，"width" 表示每次 bin 偏移量，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， + 生成区间为[-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]。 + + - "log_bin": "\{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" + "start" 表示数据起始点，"factor" 表示按指数递增的因子，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， + 生成区间为[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]。 +- normalized 是否将返回结果归一化到 0~1 之间。有效输入为 0 和 1。 + +### HYPERLOGLOG + +```sql +HYPERLOGLOG(expr) +``` + +**功能说明**： + - 采用 hyperloglog 算法，返回某列的基数。该算法在数据量很大的情况下，可以明显降低内存的占用，求出来的基数是个估算值，标准误差（标准误差是多次实验，每次的平均数的标准差，不是与真实结果的误差）为 0.81%。 + - 在数据量较少的时候该算法不是很准确，可以使用 `select count(data) from (select unique(col) as data from table)` 的方法。 + +**返回结果类型**：INTEGER。 + +**适用数据类型**：任何类型。 + +**适用于**：表和超级表。 ### LEASTSQUARES @@ -1648,6 +1697,26 @@ LEASTSQUARES(expr, start_val, step_val) **适用于**：表。 +### PERCENTILE + +```sql +PERCENTILE(expr, p [, p1] ... ) +``` + +**功能说明**：统计表中某列的值百分比分位数。 + +**返回数据类型**：该函数最小参数个数为 2 个，最大参数个数为 11 个。可以最多同时返回 10 个百分比分位数。当参数个数为 2 时，返回一个分位数，类型为DOUBLE，当参数个数大于 2 时，返回类型为VARCHAR，格式为包含多个返回值的JSON数组。 + +**应用字段**：数值类型。 + +**适用于**：表。 + +**使用说明**： + +- *P* 值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX; +- 同时计算针对同一列的多个分位数时，建议使用一个 PERCENTILE 函数和多个参数的方式，能很大程度上降低查询的响应时间。 + 比如，使用查询 `SELECT percentile(col, 90, 95, 99) FROM table`，性能会优于 `SELECT percentile(col, 90), percentile(col, 95), percentile(col, 99) from table`。 + ### SPREAD ```sql @@ -1696,7 +1765,23 @@ taos> select stddev_pop(id) from test_stddev; ============================ 1.414213562373095 | ``` + +### SUM + +```sql +SUM(expr) +``` + +**功能说明**：统计表/超级表中某列的和。 + +**返回数据类型**：DOUBLE、BIGINT。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + ### VAR_POP + ```sql VAR_POP(expr) ``` @@ -1727,88 +1812,6 @@ taos> select var_pop(id) from test_var; ============================ 2.000000000000000 | ``` -### SUM - -```sql -SUM(expr) -``` - -**功能说明**：统计表/超级表中某列的和。 - -**返回数据类型**：DOUBLE、BIGINT。 - -**适用数据类型**：数值类型。 - -**适用于**：表和超级表。 - - -### HYPERLOGLOG - -```sql -HYPERLOGLOG(expr) -``` - -**功能说明**： - - 采用 hyperloglog 算法，返回某列的基数。该算法在数据量很大的情况下，可以明显降低内存的占用，求出来的基数是个估算值，标准误差（标准误差是多次实验，每次的平均数的标准差，不是与真实结果的误差）为 0.81%。 - - 在数据量较少的时候该算法不是很准确，可以使用 `select count(data) from (select unique(col) as data from table)` 的方法。 - -**返回结果类型**：INTEGER。 - -**适用数据类型**：任何类型。 - -**适用于**：表和超级表。 - - -### HISTOGRAM - -```sql -HISTOGRAM(expr，bin_type, bin_description, normalized) -``` - -**功能说明**：统计数据按照用户指定区间的分布。 - -**返回结果类型**：如归一化参数 normalized 设置为 1，返回结果为 DOUBLE 类型，否则为 BIGINT 类型。 - -**适用数据类型**：数值型字段。 - -**适用于**：表和超级表。 - -**详细说明**： -- bin_type 用户指定的分桶类型，有效输入类型为 "user_input"、"linear_bin"、"log_bin"。 -- bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)： - - "user_input": "[1, 3, 5, 7]" - 用户指定 bin 的具体数值。 - - - "linear_bin": "\{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" - "start" 表示数据起始点，"width" 表示每次 bin 偏移量，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， - 生成区间为[-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]。 - - - "log_bin": "\{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" - "start" 表示数据起始点，"factor" 表示按指数递增的因子，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， - 生成区间为[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]。 -- normalized 是否将返回结果归一化到 0~1 之间。有效输入为 0 和 1。 - - -### PERCENTILE - -```sql -PERCENTILE(expr, p [, p1] ... ) -``` - -**功能说明**：统计表中某列的值百分比分位数。 - -**返回数据类型**：该函数最小参数个数为 2 个，最大参数个数为 11 个。可以最多同时返回 10 个百分比分位数。当参数个数为 2 时，返回一个分位数，类型为DOUBLE，当参数个数大于 2 时，返回类型为VARCHAR，格式为包含多个返回值的JSON数组。 - -**应用字段**：数值类型。 - -**适用于**：表。 - -**使用说明**： - -- *P* 值取值范围 0≤*P*≤100，为 0 的时候等同于 MIN，为 100 的时候等同于 MAX; -- 同时计算针对同一列的多个分位数时，建议使用一个 PERCENTILE 函数和多个参数的方式，能很大程度上降低查询的响应时间。 - 比如，使用查询 `SELECT percentile(col, 90, 95, 99) FROM table`，性能会优于 `SELECT percentile(col, 90), percentile(col, 95), percentile(col, 99) from table`。 - ## 选择函数 @@ -1912,7 +1915,6 @@ LAST(expr) - 在用于超级表时，时间戳完全一样且同为最大的数据行可能有多个，那么会从中随机返回一条，而并不保证多次运行所挑选的数据行必然一致。 - 对于存在复合主键的表的查询，若最大时间戳的数据有多条，则只有对应的复合主键最大的数据被返回。 - ### LAST_ROW ```sql @@ -1981,7 +1983,6 @@ MODE(expr) **适用于**：表和超级表。 - ### SAMPLE ```sql @@ -1998,7 +1999,6 @@ SAMPLE(expr, k) **适用于**：表和超级表。 - ### TAIL ```sql @@ -2015,7 +2015,6 @@ TAIL(expr, k [, offset_rows]) **适用于**：表、超级表。 - ### TOP ```sql @@ -2093,11 +2092,9 @@ CSUM(expr) **适用于**：表和超级表。 **使用说明**： - - 不支持 +、-、*、/ 运算，如 csum(col1) + csum(col2)。 - 只能与聚合（Aggregation）函数一起使用。该函数可以应用在普通表和超级表上。 - ### DERIVATIVE ```sql @@ -2173,7 +2170,6 @@ IRATE(expr) **适用于**：表和超级表。 - ### MAVG ```sql @@ -2195,7 +2191,6 @@ MAVG(expr, k) - 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1); - 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用； - ### STATECOUNT ```sql @@ -2221,7 +2216,6 @@ STATECOUNT(expr, oper, val) - 不能和窗口操作一起使用，例如 `interval/state_window/session_window`。 - ### STATEDURATION ```sql @@ -2248,7 +2242,6 @@ STATEDURATION(expr, oper, val, unit) - 不能和窗口操作一起使用，例如 interval、state_window、session_window。 - ### TWA ```sql @@ -2263,18 +2256,8 @@ TWA(expr) **适用于**：表和超级表。 - ## 系统信息函数 -### DATABASE - -```sql -SELECT DATABASE(); -``` - -**说明**：返回当前登录的数据库。如果登录的时候没有指定默认数据库，且没有使用 USE 命令切换数据库，则返回 NULL。 - - ### CLIENT_VERSION ```sql @@ -2283,13 +2266,21 @@ SELECT CLIENT_VERSION(); **说明**：返回客户端版本。 -### SERVER_VERSION +### CURRENT_USER ```sql -SELECT SERVER_VERSION(); +SELECT CURRENT_USER(); ``` -**说明**：返回服务端版本。 +**说明**：获取当前用户。 + +### DATABASE + +```sql +SELECT DATABASE(); +``` + +**说明**：返回当前登录的数据库。如果登录的时候没有指定默认数据库，且没有使用 USE 命令切换数据库，则返回 NULL。 ### SERVER_STATUS @@ -2299,14 +2290,13 @@ SELECT SERVER_STATUS(); **说明**：检测服务端是否所有 dnode 都在线，如果是则返回成功，否则返回无法建立连接的错误。如果想要查询集群的状态，推荐使用 `SHOW CLUSTER ALIVE` 与 `SELECT SERVER_STATUS()` 不同，当集群中的部分节点不可用时，它不会返回错误，而是返回不同的状态码，详见：[SHOW CLUSTER ALIVE](https://docs.taosdata.com/reference/taos-sql/show/#show-cluster-alive) -### CURRENT_USER +### SERVER_VERSION ```sql -SELECT CURRENT_USER(); +SELECT SERVER_VERSION(); ``` -**说明**：获取当前用户。 - +**说明**：返回服务端版本。 ## Geometry 函数 @@ -2348,70 +2338,6 @@ ST_AsText(GEOMETRY geom) ### Geometry 关系函数 -#### ST_Intersects - -```sql -ST_Intersects(GEOMETRY geomA, GEOMETRY geomB) -``` - -##功能说明**：比较两个几何对象，并在它们相交时返回 true。 - -**返回值类型**：BOOL。 - -**适用数据类型**：GEOMETRY、GEOMETRY。 - -**适用表类型**：标准表和超表。 - -**使用说明**：如果两个几何对象有任何一个共享点，则它们相交。 - -#### ST_Equals - -```sql -ST_Equals(GEOMETRY geomA, GEOMETRY geomB) -``` - -**功能说明**：如果给定的几何对象是“空间相等”的，则返回 TRUE。 - -**返回值类型**：BOOL。 - -**适用数据类型**：GEOMETRY、GEOMETRY。 - -**适用表类型**：标准表和超表。 - -**使用说明**："空间相等"意味着 ST_Contains(A,B) = true 和 ST_Contains(B,A) = true，并且点的顺序可能不同，但表示相同的几何结构。 - -#### ST_Touches - -```sql -ST_Touches(GEOMETRY geomA, GEOMETRY geomB) -``` - -**功能说明**：如果 A 和 B 相交，但它们的内部不相交，则返回 TRUE。 - -**返回值类型**：BOOL。 - -**适用数据类型**：GEOMETRY、GEOMETRY。 - -**适用表类型**：标准表和超表。 - -**使用说明**：A 和 B 至少有一个公共点，并且这些公共点位于至少一个边界中。对于点/点输入，关系始终为 FALSE，因为点没有边界。 - -#### ST_Covers - -```sql -ST_Covers(GEOMETRY geomA, GEOMETRY geomB) -``` - -**功能说明**：如果 B 中的每个点都位于几何形状 A 内部（与内部或边界相交），则返回 TRUE。 - -**返回值类型**：BOOL。 - -**适用数据类型**：GEOMETRY、GEOMETRY。 - -**适用表类型**：标准表和超表。 - -**使用说明**：A 包含 B 意味着 B 中的没有点位于 A 的外部（在外部）。 - #### ST_Contains ```sql @@ -2443,3 +2369,68 @@ ST_ContainsProperly(GEOMETRY geomA, GEOMETRY geomB) **适用表类型**：标准表和超表。 **使用说明**：B 的没有点位于 A 的边界或外部。 + +#### ST_Covers + +```sql +ST_Covers(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果 B 中的每个点都位于几何形状 A 内部（与内部或边界相交），则返回 TRUE。 + +**返回值类型**：BOOL。 + +**适用数据类型**：GEOMETRY、GEOMETRY。 + +**适用表类型**：标准表和超表。 + +**使用说明**：A 包含 B 意味着 B 中的没有点位于 A 的外部（在外部）。 + +#### ST_Equals + +```sql +ST_Equals(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果给定的几何对象是"空间相等"的，则返回 TRUE。 + +**返回值类型**：BOOL。 + +**适用数据类型**：GEOMETRY、GEOMETRY。 + +**适用表类型**：标准表和超表。 + +**使用说明**："空间相等"意味着 ST_Contains(A,B) = true 和 ST_Contains(B,A) = true，并且点的顺序可能不同，但表示相同的几何结构。 + + +#### ST_Intersects + +```sql +ST_Intersects(GEOMETRY geomA, GEOMETRY geomB) +``` + +##功能说明**：比较两个几何对象，并在它们相交时返回 true。 + +**返回值类型**：BOOL。 + +**适用数据类型**：GEOMETRY、GEOMETRY。 + +**适用表类型**：标准表和超表。 + +**使用说明**：如果两个几何对象有任何一个共享点，则它们相交。 + +#### ST_Touches + +```sql +ST_Touches(GEOMETRY geomA, GEOMETRY geomB) +``` + +**功能说明**：如果 A 和 B 相交，但它们的内部不相交，则返回 TRUE。 + +**返回值类型**：BOOL。 + +**适用数据类型**：GEOMETRY、GEOMETRY。 + +**适用表类型**：标准表和超表。 + +**使用说明**：A 和 B 至少有一个公共点，并且这些公共点位于至少一个边界中。对于点/点输入，关系始终为 FALSE，因为点没有边界。 diff --git a/docs/zh/27-train-faq/02-dst.md b/docs/zh/27-train-faq/02-dst.md new file mode 100644 index 0000000000..5c430fd42f --- /dev/null +++ b/docs/zh/27-train-faq/02-dst.md @@ -0,0 +1,291 @@ +--- +title: 夏令时使用指南 +description: TDengine 中关于夏令时使用问题的解释和建议 +--- + +## 背景 + +在时序数据库的使用中，有时会遇到使用夏令时的情况。我们将 TDengine 中使用夏令时的情况和问题进行分析说明，以便您在 TDengine 的使用中更加顺利。 + +## 定义 + +### 时区 + +时区是地球上使用相同标准时间的区域。由于地球的自转，为了保证各地的时间与当地的日出日落相协调，全球划分为多个时区。 + +### IANA 时区 + +IANA（Internet Assigned Numbers Authority）时区数据库，也称为 tz database，提供全球时区信息的标准参考。它是现代各类系统和软件处理时区相关操作的基础。 + +IANA 使用“区域/城市”格式（如 Europe/Berlin）来明确标识时区。 + +TDengine 在不同组件中均支持使用 IANA 时区（除 Windows taos.cfg 时区设置外）。 + +### 标准时间与当地时间 + +标准时间是根据地球上某个固定经线确定的时间。它为各个时区提供了一个统一的参考点。 + +- 格林尼治标准时间（GMT）：历史上使用的参考时间，位于 0° 经线。 +- 协调世界时（UTC）：现代的时间标准，类似于GMT，但更加精确。 + +标准时间与时区的关系如下： + +- 基准：标准时间（如 UTC）是时区设定的基准点。 +- 偏移量：不同时区通过相对于标准时间的偏移量来定义。例如，UTC+1 表示比 UTC 快 1 小时。 +- 区域划分：全球被划分为多个时区，每个时区使用一个或多个标准时间。 + +相对于标准时间，每个地区根据其所在时区设定其当地时间： + +- 时区偏移：当地时间等于标准时间加上该时区的偏移量。例如，UTC+2 表示比 UTC 时间快 2 小时。 +- 夏令时（DST）：某些地区在特定时间段调整当地时间，例如将时钟拨快一小时。详见下节。 + +### 夏令时 + +夏令时（Daylight Saving Time，DST）是一种通过将时间提前一小时，以充分利用日光、节约能源的制度。通常在春季开始，秋季结束。夏令时的具体开始和结束时间因地区而异。以下均以柏林时间为例，对夏令时和夏令时的影响做说明。 + +按照这个规则，可以看到： + +- 柏林当地时间 2024 年 03 月 31 日 02:00:00 到 03:00:00 （不含 03:00:00）之间的时间不存在（跳变）。 +- 柏林当地时间 2024 年 10 月 27 日 02:00:00 到 03:00:00 （不含 03:00:00）之间的时间出现了两次。 + +#### 夏令时与 IANA 时区数据库 + +- 记录规则：IANA 时区数据库详细记录了各地的夏令时规则，包括开始和结束的日期与时间。 +- 自动调整：许多操作系统和软件利用 IANA 数据库来自动处理夏令时的调整。 +- 历史变更：IANA 数据库还追踪历史上的夏令时变化，以确保准确性。 + +#### 夏令时与时间戳转换 + +- 时间戳转为当地时间是确定的。例如，1729990654 为柏林时间**夏令时** `2024-10-27 02:57:34`，1729994254 为柏林时间**冬令时** `2024-10-27 02:57:34`（这两个本地时间除时间偏移量外是一样的）。 +- 不指定时间偏移量时，当地时间转为时间戳是不确定的。夏令时跳过的时间不存在会造成无法转换成时间戳，如 **柏林时间** `2024-03-31 02:34:56` 不存在，所以无法转换为时间戳。夏令时结束时重复导致无法确定是哪个时间戳，如 `2024-10-27 02:57:34` 不指定时间偏移量无法确定 是 1729990654 还是 1729994254。指定时间偏移量才能确定时间戳，如 `2024-10-27 02:57:34 CEST(+02:00) `，指定了夏令时 `2024-10-27 02:57:34` 时间戳 1729990654 。 + +### RFC3339 时间格式 + +RFC 3339 是一种互联网时间格式标准，用于表示日期和时间。它基于 ISO 8601 标准，但更具体地规定了一些格式细节。 + +其格式如下： + +- 基本格式：`YYYY-MM-DDTHH:MM:SSZ` +- 时区表示： + - Z 表示协调世界时（UTC）。 + - 偏移量格式，例如 +02:00，表示与 UTC 的时差。 + +通过明确的时区偏移，RFC 3339 格式可以在全球范围内准确地解析和比较时间。 + +RFC 3339 的优势包括： + +- 标准化：提供统一的格式，方便跨系统数据交换。 +- 清晰性：明确时区信息，避免时间误解。 + +TDengine 在 REST API 和 Explorer UI 中，均使用 RFC3339 格式进行展示。在 SQL 语句中，可使用 RFC3339 格式写入时间戳数据： + +```sql +insert into t1 values('2024-10-27T01:59:59.000Z', 0); +select * from t1 where ts >= '2024-10-27T01:59:59.000Z'; +``` + +### 未定义行为 + +未定义行为（Undefined Behavior）是指特定代码或操作没有明确规定的结果，也不会对该结果作出兼容性的保证，TDengine 可能在某个版本后对当前的行为作出修改而不会通知用户。所以，在 TDengine 中，用户不可依赖当前未定义的行为进行判断或应用。 + +## 夏令时在 TDengine 中的写入与查询 + +我们使用下表来展示夏令时在写入和查询中的影响。 + +![DST Berlin](./02-dst/dst-berlin.png) + +### 表格说明 + +- **TIMESTAMP**：TDengine 中使用 64位整数来存储原始时间戳。 +- **UTC**：时间戳对应的 UTC 时间表示。 +- **Europe/Berlin**：表示时区 Europe/Berlin 对应的 RFC3339 格式时间。 +- **Local**：表示时区 Europe/Berlin 对应的当地时间（不含时区）。 + +### 表格分析 + +- 在**夏令时开始**（柏林时间 3 月 31 日 02:00）时，时间直接从 02:00 跳到 03:00（往后跳一小时）。 + - 浅绿色是夏令时开始前一小时的时间戳； + - 深绿色是夏令时开始后一小时的时间戳； + - 红色为 TDengine 数据库中插入了不存在的当地时间： + - 使用 SQL `INSERT INTO t1 values('2024-03-31 02:59:59',..)` 插入 `2024-03-31 02:00:00` 到 `2024-03-31 02:59:59` 的数据会被自动调整为 -1000（在 TDengine 中属于未定义行为，当前该值与数据库精度 precision 有关，毫秒数据库为 -1000，微秒数据库为 -1000000，纳秒数据库为 -1000000000），因为那一时刻在本地时间中不存在； +- 在**夏令时结束**（柏林时间 10 月 27 日 03:00）时，时间从 03:00 跳到 02:00 （往前跳一小时）。 + - 浅蓝色表示时钟跳变前一小时的时间戳； + - 深蓝色表示时钟跳变后一小时内的时间戳，其无时区的当地时间与上一小时一致。 + - 紫色表示时钟跳变一小时后的时间戳； +- **当地时间变化**：可见，由于夏令时的调整而导致了当地时间的变化，可能导致某些时间段出现重复或缺失。 +- **UTC 时间不变**：UTC 时间保持不变，确保了时间的一致性和顺序性。 +- **RFC3339**：RFC3339 格式时间显示了时间偏移量的变化，在夏令时开始后变为 +02:00，结束后变为 +01:00 。 +- **条件查询**： + - **夏令时开始**时，跳过的时间（`[03-31 02:00:00,03-31 03:00:00)`）不存在，所以在使用该时间进行查询时，行为不确定：`SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'`（不存在的本地时间戳被转换为 `-1000`）: + + ```sql + taos> SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'; + ts | + ================= + -1000 | + Query OK, 1 row(s) in set (0.003635s) + ``` + + 当不存在的时间戳与存在的时间戳共同使用时，其结果同样不符合预期，以下为起始本地时间不存在： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 03:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + -1000 | 1969-12-31T23:59:59.000Z | + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + 1711846800000 | 2024-03-31T01:00:00.000Z | + 1711846801000 | 2024-03-31T01:00:01.000Z | + Query OK, 5 row(s) in set (0.003339s) + ``` + + 以下语句中第一个 SQL 查询截止时间不存在，第二个截止时间存在，第一个 SQL 查询结果不符合预期： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 02:00:00'; + Query OK, 0 row(s) in set (0.000930s) + + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 01:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + Query OK, 2 row(s) in set (0.001227s) + ``` + + - 夏令时结束时，跳变的时间（`[10-27 02:00:00,10-27 03:00:00)` 不包含 `10-27 03:00:00`）重复了两次，TDengine 在使用该区间内的时间戳进行查询时，也属于未定义行为。 + - 查询 `[2024-10-27 02:00:00, 2024-10-27 03:00:00]` 之间的数据结果，包含了两次重复的时间戳和 `2024-10-27 03:00:00` 这个时间点的数据： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts BETWEEN '2024-10-27 02:00:00' AND '2024-10-27 03:00:00'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + 1729994400000 | 2024-10-27T02:00:00.000Z | 2024-10-27 03:00:00 | + Query OK, 5 row(s) in set (0.001370s) + ```` + + - 但以下查询 [2024-10-27 02:00:00.000,2024-10-27 02:57:34.999] 区间只能查询到第一个2024-10-27 02:00:00 时间点的数据： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:00.999'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 1 row(s) in set (0.004480s) + ``` + + - 以下查询 `[2024-10-27 02:00:01,2024-10-27 02:57:35]` 却能查到 3 条数据（包含一条 02:59:59 的当地时间数据）： + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:35';; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ================================================================================================ + 2024-10-27 02:00:00.000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 2024-10-27 02:59:59.000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 2024-10-27 02:00:00.000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 3 row(s) in set (0.004428s) + ``` + +## 总结与建议 + +### 总结 + +仅针对使用当地时间带来的影响作说明，使用 UNIX 时间戳或 RFC3339 无影响。 + +- 写入： + - 无法写入夏令时跳变时不存在的时间数据。 + - 写入夏令时跳变时重复的时间是未定义行为。 +- 查询： + - 查询条件指定夏令时开始时跳变的时间，其查询结果为未定义行为。 + - 查询条件指定夏令时结束时重复的时间，其查询结果为未定义行为。 +- 显示： + - 带时区显示不受影响。 + - 显示当地时间是准确的，但夏令时结束时重复的时间会无法区分。 + - 用户应谨慎使用不带时区的时间进行展示和应用。 + +### 建议 + +为避免夏令时给查询和写入造成不必要的影响，在 TDengine 中，建议使用明确的时间偏移量进行写入和查询。 + +- 使用 UNIX 时间戳：使用 UNIX 时间戳可避免时区问题。 + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1711846799000 | 2024-03-31T00:59:59.000Z | 2024-03-31T01:59:59.000+01:00 | 2024-03-31 01:59:59 | + | 1711846800000 | 2024-03-31T01:00:00.000Z | 2024-03-31T03:00:00.000+02:00 | 2024-03-31 03:00:00 | + + ```sql + taos> insert into t1 values(1711846799000, 1)(1711846800000, 2); + Insert OK, 2 row(s) affected (0.001434s) + + taos> select * from t1 where ts between 1711846799000 and 1711846800000; + ts | v1 | + =============================== + 1711846799000 | 1 | + 1711846800000 | 2 | + Query OK, 2 row(s) in set (0.003503s) + ``` + +- 使用 RFC3339 时间格式：带时区偏移量的 RFC3339 时间格式可以有效避免夏令时的不确定性。 + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27T02:00:00.000+02:00 | 2024-10-27 02:00:00 | + | 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27T02:59:59.000+02:00 | 2024-10-27 02:59:59 | + | 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27T02:00:00.000+01:00 | 2024-10-27 02:00:00 | + | 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27T02:59:59.000+01:00 | 2024-10-27 02:59:59 | + + ```sql + taos> insert into t1 values ('2024-10-27T02:00:00.000+02:00', 1) + ('2024-10-27T02:59:59.000+02:00', 2) + ('2024-10-27T02:00:00.000+01:00', 3) + ('2024-10-27T02:59:59.000+01:00', 4); + Insert OK, 4 row(s) affected (0.001514s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+01:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 3 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 4 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 4 row(s) in set (0.004275s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+02:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 2 row(s) in set (0.004275s) + ``` + +- 查询时注意时区设定：在查询和显示时，如果需要本地时间，务必考虑夏令时的影响。 + - taosAdapter：使用 REST API 时，支持设置 IANA 时区，结果使用 RFC3339 格式返回。 + + ```shell + $ curl -uroot:taosdata 'localhost:6041/rest/sql?tz=Europe/Berlin'\ + -d "select ts from tz1.t1" + {"code":0,"column_meta":[["ts","TIMESTAMP",8]],"data":[["1970-01-01T00:59:59.000+01:00"],["2024-03-31T01:00:00.000+01:00"],["2024-03-31T01:59:59.000+01:00"],["2024-03-31T03:00:00.000+02:00"],["2024-03-31T03:00:01.000+02:00"],["2024-10-27T02:00:00.000+02:00"],["2024-10-27T02:59:59.000+02:00"],["2024-10-27T02:00:00.000+01:00"],["2024-10-27T02:59:59.000+01:00"],["2024-10-27T03:00:00.000+01:00"]],"rows":10} + ``` + + - Explorer：使用 Explorer 页面进行 SQL 查询时，用户可配置客户端时区，以 RFC3339 格式显示。 + + ![Explorer DST](./02-dst/explorer-with-tz.png) + +## 参考文档 + +- IANA Time Zone Database: [https://www.iana.org/time-zones](https://www.iana.org/time-zones) +- RFC3339: [https://datatracker.ietf.org/doc/html/rfc3339](https://datatracker.ietf.org/doc/html/rfc3339) diff --git a/docs/zh/27-train-faq/02-dst/dst-berlin.png b/docs/zh/27-train-faq/02-dst/dst-berlin.png new file mode 100644 index 0000000000..8c64626dd0 Binary files /dev/null and b/docs/zh/27-train-faq/02-dst/dst-berlin.png differ diff --git a/docs/zh/27-train-faq/02-dst/dst-table.png b/docs/zh/27-train-faq/02-dst/dst-table.png new file mode 100644 index 0000000000..f5eddbe010 Binary files /dev/null and b/docs/zh/27-train-faq/02-dst/dst-table.png differ diff --git a/docs/zh/27-train-faq/02-dst/explorer-with-tz.png b/docs/zh/27-train-faq/02-dst/explorer-with-tz.png new file mode 100644 index 0000000000..95758f117a Binary files /dev/null and b/docs/zh/27-train-faq/02-dst/explorer-with-tz.png differ diff --git a/include/common/tanalytics.h b/include/common/tanalytics.h index 42c3ce9391..0fb1d543f7 100644 --- a/include/common/tanalytics.h +++ b/include/common/tanalytics.h @@ -25,12 +25,13 @@ extern "C" { #endif -#define ANALY_FORECAST_DEFAULT_ROWS 10 -#define ANALY_FORECAST_DEFAULT_CONF 95 -#define ANALY_FORECAST_DEFAULT_WNCHECK 1 -#define ANALY_FORECAST_MAX_HISTORY_ROWS 40000 -#define ANALY_MAX_FC_ROWS 1024 -#define ANALY_ANOMALY_WINDOW_MAX_ROWS 40000 +#define ANALY_FORECAST_DEFAULT_ROWS 10 +#define ANALY_FORECAST_DEFAULT_CONF 95 +#define ANALY_FORECAST_DEFAULT_WNCHECK 1 +#define ANALY_FORECAST_MAX_ROWS 40000 +#define ANALY_ANOMALY_WINDOW_MAX_ROWS 40000 +#define ANALY_DEFAULT_TIMEOUT 60 +#define ANALY_MAX_TIMEOUT 600 typedef struct { EAnalAlgoType type; @@ -48,7 +49,7 @@ typedef enum { typedef enum { ANALYTICS_HTTP_TYPE_GET = 0, ANALYTICS_HTTP_TYPE_POST, -} EAnalHttpType; +} EAnalyHttpType; typedef struct { TdFilePtr filePtr; @@ -66,7 +67,7 @@ typedef struct { int32_t taosAnalyticsInit(); void taosAnalyticsCleanup(); -SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf *pBuf); +SJson *taosAnalySendReqRetJson(const char *url, EAnalyHttpType type, SAnalyticBuf *pBuf, int64_t timeout); int32_t taosAnalyGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen); bool taosAnalyGetOptStr(const char *option, const char *optName, char *optValue, int32_t optMaxLen); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 266bd4a618..986874c5bb 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -27,6 +27,7 @@ #include "scheduler.h" #include "tcache.h" #include "tcompare.h" +#include "tconv.h" #include "tglobal.h" #include "thttp.h" #include "tmsg.h" @@ -36,7 +37,6 @@ #include "tsched.h" #include "ttime.h" #include "tversion.h" -#include "tconv.h" #include "cus_name.h" @@ -63,13 +63,13 @@ } \ } while (0) -STscDbg tscDbg = {0}; -SAppInfo appInfo; -int64_t lastClusterId = 0; -int32_t clientReqRefPool = -1; -int32_t clientConnRefPool = -1; -int32_t clientStop = -1; -SHashObj* pTimezoneMap = NULL; +STscDbg tscDbg = {0}; +SAppInfo appInfo; +int64_t lastClusterId = 0; +int32_t clientReqRefPool = -1; +int32_t clientConnRefPool = -1; +int32_t clientStop = -1; +SHashObj *pTimezoneMap = NULL; int32_t timestampDeltaLimit = 900; // s @@ -964,7 +964,7 @@ void taos_init_imp(void) { ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg"); initQueryModuleMsgHandle(); - if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL){ + if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) { tscInitRes = terrno; tscError("failed to init conv"); return; diff --git a/source/common/src/tanalytics.c b/source/common/src/tanalytics.c index deb68af3ea..397accc0b1 100644 --- a/source/common/src/tanalytics.c +++ b/source/common/src/tanalytics.c @@ -276,7 +276,7 @@ _OVER: return code; } -static int32_t taosCurlPostRequest(const char *url, SCurlResp *pRsp, const char *buf, int32_t bufLen) { +static int32_t taosCurlPostRequest(const char *url, SCurlResp *pRsp, const char *buf, int32_t bufLen, int32_t timeout) { struct curl_slist *headers = NULL; CURL *curl = NULL; CURLcode code = 0; @@ -292,7 +292,7 @@ static int32_t taosCurlPostRequest(const char *url, SCurlResp *pRsp, const char if (curl_easy_setopt(curl, CURLOPT_URL, url) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, taosCurlWriteData) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, pRsp) != 0) goto _OVER; - if (curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 60000) != 0) goto _OVER; + if (curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_POST, 1) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, bufLen) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buf) != 0) goto _OVER; @@ -311,7 +311,7 @@ _OVER: return code; } -SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf *pBuf) { +SJson *taosAnalySendReqRetJson(const char *url, EAnalyHttpType type, SAnalyticBuf *pBuf, int64_t timeout) { int32_t code = -1; char *pCont = NULL; int64_t contentLen; @@ -329,7 +329,7 @@ SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf terrno = code; goto _OVER; } - if (taosCurlPostRequest(url, &curlRsp, pCont, contentLen) != 0) { + if (taosCurlPostRequest(url, &curlRsp, pCont, contentLen, timeout) != 0) { terrno = TSDB_CODE_ANA_URL_CANT_ACCESS; goto _OVER; } @@ -767,7 +767,7 @@ static int32_t taosAnalyBufGetCont(SAnalyticBuf *pBuf, char **ppCont, int64_t *p int32_t taosAnalyticsInit() { return 0; } void taosAnalyticsCleanup() {} -SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf *pBuf) { return NULL; } +SJson *taosAnalySendReqRetJson(const char *url, EAnalyHttpType type, SAnalyticBuf *pBuf, int64_t timeout) { return NULL; } int32_t taosAnalyGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen) { return 0; } bool taosAnalyGetOptStr(const char *option, const char *optName, char *optValue, int32_t optMaxLen) { return true; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 00f9504bc9..299e947d35 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "tglobal.h" #include "cJSON.h" #include "defines.h" #include "os.h" #include "osString.h" #include "tconfig.h" -#include "tglobal.h" #include "tgrant.h" #include "tjson.h" #include "tlog.h" @@ -28,7 +28,6 @@ #include "tutil.h" - #define CONFIG_PATH_LEN (TSDB_FILENAME_LEN + 12) #define CONFIG_FILE_LEN (CONFIG_PATH_LEN + 32) @@ -501,9 +500,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -struct SConfig *taosGetCfg() { - return tsCfg; -} +struct SConfig *taosGetCfg() { return tsCfg; } static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -692,7 +689,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL) != 0); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_SERVER, + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 3000, 86400000, CFG_SCOPE_SERVER, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( diff --git a/source/dnode/mnode/impl/src/mndAnode.c b/source/dnode/mnode/impl/src/mndAnode.c index c08d4aead4..bd0a4f3138 100644 --- a/source/dnode/mnode/impl/src/mndAnode.c +++ b/source/dnode/mnode/impl/src/mndAnode.c @@ -789,7 +789,7 @@ static int32_t mndGetAnodeAlgoList(const char *url, SAnodeObj *pObj) { char anodeUrl[TSDB_ANALYTIC_ANODE_URL_LEN + 1] = {0}; snprintf(anodeUrl, TSDB_ANALYTIC_ANODE_URL_LEN, "%s/%s", url, "list"); - SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL); + SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL, 0); if (pJson == NULL) return terrno; int32_t code = mndDecodeAlgoList(pJson, pObj); @@ -805,7 +805,7 @@ static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status, int32_t statusLe char anodeUrl[TSDB_ANALYTIC_ANODE_URL_LEN + 1] = {0}; snprintf(anodeUrl, TSDB_ANALYTIC_ANODE_URL_LEN, "%s/%s", pObj->url, "status"); - SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL); + SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL, 0); if (pJson == NULL) return terrno; code = tjsonGetDoubleValue(pJson, "protocol", &tmp); diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index 6ed4991679..3b9857a6a2 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -14,10 +14,19 @@ */ #include "meta.h" +#include "vnd.h" static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { + SVBufPool *pool = (SVBufPool *)pPool; + SVnode *pVnode = pool->pVnode; + + if (pVnode->inUse && pVnode->inUse->size > pVnode->inUse->node.size) { + return NULL; + } + return vnodeBufPoolMallocAligned((SVBufPool *)pPool, size); } + static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } // begin a meta txn diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7d65673226..c80edd070d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -928,12 +928,6 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask // now the fill-history task starts to scan data from wal files. code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); -// if (code == TSDB_CODE_SUCCESS) { -// code = tqScanWalAsync(pTq, false); -// if (code) { -// tqError("vgId:%d failed to start scan wal file, code:%s", vgId, tstrerror(code)); -// } -// } } } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 08ca02e20e..b9cbb33746 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -148,6 +148,7 @@ static void doStartScanWal(void* param, void* tmrId) { return; } + // failed to lock, try 500ms later code = streamMetaTryRlock(pMeta); if (code == 0) { numOfTasks = taosArrayGetSize(pMeta->pTaskList); @@ -156,25 +157,23 @@ static void doStartScanWal(void* param, void* tmrId) { numOfTasks = 0; } - if (numOfTasks == 0) { - goto _end; - } + if (numOfTasks > 0) { + tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d", vgId, numOfTasks); - tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d", vgId, numOfTasks); - - #if 0 +#if 0 // wait for the vnode is freed, and invalid read may occur. taosMsleep(10000); - #endif +#endif - code = streamTaskSchedTask(&pParam->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); - if (code) { - tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code)); + code = streamTaskSchedTask(&pParam->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); + if (code) { + tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code)); + } } _end: streamTmrStart(doStartScanWal, SCAN_WAL_IDLE_DURATION, pParam, pTimer, &pMeta->scanInfo.scanTimer, vgId, "scan-wal"); - tqDebug("vgId:%d scan-wal will start in %dms", vgId, SCAN_WAL_IDLE_DURATION*SCAN_WAL_WAIT_COUNT); + tqDebug("vgId:%d try scan-wal will start in %dms", vgId, SCAN_WAL_IDLE_DURATION*SCAN_WAL_WAIT_COUNT); code = taosReleaseRef(streamMetaRefPool, pParam->metaId); if (code) { @@ -192,7 +191,7 @@ void tqScanWalAsync(STQ* pTq) { // 1. the vnode should be the leader. // 2. the stream isn't disabled - if ((pMeta->role == NODE_ROLE_FOLLOWER) || tsDisableStream) { + if ((pMeta->role != NODE_ROLE_LEADER) || tsDisableStream) { tqInfo("vgId:%d follower node or stream disabled, not scan wal", vgId); return; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 1afccd3d01..4295969c85 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -960,11 +960,6 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { streamMetaWUnLock(pMeta); -// if (scanWal && (vgId != SNODE_HANDLE)) { -// tqDebug("vgId:%d start scan wal for executing tasks", vgId); -// code = tqScanWalAsync(pMeta->ahandle, true); -// } - return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 8baf08ef94..ca3b82239b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -610,11 +610,12 @@ static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { code = taosThreadMutexTryLock(&pReader->readerMutex); if (code != TSDB_CODE_SUCCESS) { - tsdbError("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + // Failing to acquire the lock is reasonable, not an error + tsdbWarn("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); } else { tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex); } - TSDB_CHECK_CODE(code, lino, _end); + return code; _end: if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index 46aae38ad4..379177bb06 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -47,6 +47,7 @@ typedef struct { char algoName[TSDB_ANALYTIC_ALGO_NAME_LEN]; char algoUrl[TSDB_ANALYTIC_ALGO_URL_LEN]; char anomalyOpt[TSDB_ANALYTIC_ALGO_OPTION_LEN]; + int64_t timeout; SAnomalyWindowSupp anomalySup; SWindowRowsSup anomalyWinRowSup; SColumn anomalyCol; @@ -89,6 +90,20 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p goto _error; } + bool hasTimeout = taosAnalyGetOptInt(pAnomalyNode->anomalyOpt, "timeout", &pInfo->timeout); + if (!hasTimeout) { + qDebug("not set the timeout val, set default:%d", ANALY_DEFAULT_TIMEOUT); + pInfo->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + if (pInfo->timeout <= 0 || pInfo->timeout > ANALY_MAX_TIMEOUT) { + qDebug("timeout val:%" PRId64 "s is invalid (greater than 10min or less than 1s), use default:%dms", + pInfo->timeout, ANALY_DEFAULT_TIMEOUT); + pInfo->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + qDebug("timeout val is set to: %" PRId64 "s", pInfo->timeout); + } + } + pOperator->exprSupp.hasWindowOrGroup = true; pInfo->tsSlotId = ((SColumnNode*)pAnomalyNode->window.pTspk)->slotId; tstrncpy(pInfo->anomalyOpt, pAnomalyNode->anomalyOpt, sizeof(pInfo->anomalyOpt)); @@ -451,7 +466,7 @@ static int32_t anomalyAnalysisWindow(SOperatorInfo* pOperator) { code = taosAnalyBufClose(&analyBuf); QUERY_CHECK_CODE(code, lino, _OVER); - pJson = taosAnalySendReqRetJson(pInfo->algoUrl, ANALYTICS_HTTP_TYPE_POST, &analyBuf); + pJson = taosAnalySendReqRetJson(pInfo->algoUrl, ANALYTICS_HTTP_TYPE_POST, &analyBuf, pInfo->timeout * 1000); if (pJson == NULL) { code = terrno; goto _OVER; diff --git a/source/libs/executor/src/forecastoperator.c b/source/libs/executor/src/forecastoperator.c index e318530352..e9185824a3 100644 --- a/source/libs/executor/src/forecastoperator.c +++ b/source/libs/executor/src/forecastoperator.c @@ -38,6 +38,7 @@ typedef struct { int64_t optRows; int64_t cachedRows; int32_t numOfBlocks; + int64_t timeout; int16_t resTsSlot; int16_t resValSlot; int16_t resLowSlot; @@ -76,10 +77,10 @@ static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock, con int32_t lino = 0; SAnalyticBuf* pBuf = &pSupp->analyBuf; - if (pSupp->cachedRows > ANALY_FORECAST_MAX_HISTORY_ROWS) { + if (pSupp->cachedRows > ANALY_FORECAST_MAX_ROWS) { code = TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; qError("%s rows:%" PRId64 " for forecast cache, error happens, code:%s, upper limit:%d", id, pSupp->cachedRows, - tstrerror(code), ANALY_FORECAST_MAX_HISTORY_ROWS); + tstrerror(code), ANALY_FORECAST_MAX_ROWS); return code; } @@ -157,8 +158,8 @@ static int32_t forecastCloseBuf(SForecastSupp* pSupp, const char* id) { qDebug("%s forecast rows not found from %s, use default:%" PRId64, id, pSupp->algoOpt, pSupp->optRows); } - if (pSupp->optRows > ANALY_MAX_FC_ROWS) { - qError("%s required too many forecast rows, max allowed:%d, required:%" PRId64, id, ANALY_MAX_FC_ROWS, + if (pSupp->optRows > ANALY_FORECAST_MAX_ROWS) { + qError("%s required too many forecast rows, max allowed:%d, required:%" PRId64, id, ANALY_FORECAST_MAX_ROWS, pSupp->optRows); return TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; } @@ -198,12 +199,12 @@ static int32_t forecastCloseBuf(SForecastSupp* pSupp, const char* id) { static int32_t forecastAnalysis(SForecastSupp* pSupp, SSDataBlock* pBlock, const char* pId) { SAnalyticBuf* pBuf = &pSupp->analyBuf; int32_t resCurRow = pBlock->info.rows; - int8_t tmpI8; - int16_t tmpI16; - int32_t tmpI32; - int64_t tmpI64; - float tmpFloat; - double tmpDouble; + int8_t tmpI8 = 0; + int16_t tmpI16 = 0; + int32_t tmpI32 = 0; + int64_t tmpI64 = 0; + float tmpFloat = 0; + double tmpDouble = 0; int32_t code = 0; SColumnInfoData* pResValCol = taosArrayGet(pBlock->pDataBlock, pSupp->resValSlot); @@ -211,12 +212,13 @@ static int32_t forecastAnalysis(SForecastSupp* pSupp, SSDataBlock* pBlock, const return terrno; } - SColumnInfoData* pResTsCol = (pSupp->resTsSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resTsSlot) : NULL); - SColumnInfoData* pResLowCol = (pSupp->resLowSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resLowSlot) : NULL); + SColumnInfoData* pResTsCol = ((pSupp->resTsSlot != -1) ? taosArrayGet(pBlock->pDataBlock, pSupp->resTsSlot) : NULL); + SColumnInfoData* pResLowCol = + ((pSupp->resLowSlot != -1) ? taosArrayGet(pBlock->pDataBlock, pSupp->resLowSlot) : NULL); SColumnInfoData* pResHighCol = (pSupp->resHighSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resHighSlot) : NULL); - SJson* pJson = taosAnalySendReqRetJson(pSupp->algoUrl, ANALYTICS_HTTP_TYPE_POST, pBuf); + SJson* pJson = taosAnalySendReqRetJson(pSupp->algoUrl, ANALYTICS_HTTP_TYPE_POST, pBuf, pSupp->timeout * 1000); if (pJson == NULL) { return terrno; } @@ -527,18 +529,32 @@ static int32_t forecastParseInput(SForecastSupp* pSupp, SNodeList* pFuncs) { return 0; } -static int32_t forecastParseAlgo(SForecastSupp* pSupp) { +static int32_t forecastParseAlgo(SForecastSupp* pSupp, const char* id) { pSupp->maxTs = 0; pSupp->minTs = INT64_MAX; pSupp->numOfRows = 0; if (!taosAnalyGetOptStr(pSupp->algoOpt, "algo", pSupp->algoName, sizeof(pSupp->algoName))) { - qError("failed to get forecast algorithm name from %s", pSupp->algoOpt); + qError("%s failed to get forecast algorithm name from %s", id, pSupp->algoOpt); return TSDB_CODE_ANA_ALGO_NOT_FOUND; } + bool hasTimeout = taosAnalyGetOptInt(pSupp->algoOpt, "timeout", &pSupp->timeout); + if (!hasTimeout) { + qDebug("%s not set the timeout val, set default:%d", id, ANALY_DEFAULT_TIMEOUT); + pSupp->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + if (pSupp->timeout <= 0 || pSupp->timeout > ANALY_MAX_TIMEOUT) { + qDebug("%s timeout val:%" PRId64 "s is invalid (greater than 10min or less than 1s), use default:%dms", + id, pSupp->timeout, ANALY_DEFAULT_TIMEOUT); + pSupp->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + qDebug("%s timeout val is set to: %" PRId64 "s", id, pSupp->timeout); + } + } + if (taosAnalyGetAlgoUrl(pSupp->algoName, ANALY_ALGO_TYPE_FORECAST, pSupp->algoUrl, sizeof(pSupp->algoUrl)) != 0) { - qError("failed to get forecast algorithm url from %s", pSupp->algoName); + qError("%s failed to get forecast algorithm url from %s", id, pSupp->algoName); return TSDB_CODE_ANA_ALGO_NOT_LOAD; } @@ -589,6 +605,7 @@ int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNo goto _error; } + const char* pId = pTaskInfo->id.str; SForecastSupp* pSupp = &pInfo->forecastSupp; SForecastFuncPhysiNode* pForecastPhyNode = (SForecastFuncPhysiNode*)pPhyNode; SExprSupp* pExprSup = &pOperator->exprSupp; @@ -620,7 +637,7 @@ int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNo code = forecastParseOutput(pSupp, pExprSup); QUERY_CHECK_CODE(code, lino, _error); - code = forecastParseAlgo(pSupp); + code = forecastParseAlgo(pSupp, pId); QUERY_CHECK_CODE(code, lino, _error); code = forecastCreateBuf(pSupp); @@ -644,7 +661,7 @@ int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNo *pOptrInfo = pOperator; - qDebug("forecast env is initialized, option:%s", pSupp->algoOpt); + qDebug("%s forecast env is initialized, option:%s", pId, pSupp->algoOpt); return TSDB_CODE_SUCCESS; _error: diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 91be791dff..876b05d55f 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -797,7 +797,7 @@ static bool funcNotSupportStringSma(SFunctionNode* pFunc) { } EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { - if(funcNotSupportStringSma(pFunc)) { + if (funcNotSupportStringSma(pFunc)) { return FUNC_DATA_REQUIRED_DATA_LOAD; } return FUNC_DATA_REQUIRED_SMA_LOAD; @@ -6611,7 +6611,7 @@ int32_t blockDBUsageFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { double compressRadio = 0; if (rawDataSize != 0) { compressRadio = totalDiskSize * 100 / (double)rawDataSize; - len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f]", compressRadio); + len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f%]", compressRadio); } else { len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[NULL]"); } diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 1d1bc66414..257ad3d8ea 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -532,9 +532,17 @@ int32_t sifStr2Num(char *buf, int32_t len, int8_t type, void *val) { static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { int32_t code = 0; int8_t ltype = left->colValType, rtype = right->colValType; - if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) { - return TSDB_CODE_INVALID_PARA; + // if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) { + // return TSDB_CODE_INVALID_PARA; + // } + if (IS_VAR_DATA_TYPE(ltype)) { + if (ltype == TSDB_DATA_TYPE_VARCHAR || ltype == TSDB_DATA_TYPE_BINARY || ltype == TSDB_DATA_TYPE_VARBINARY) { + return 0; + } else { + return TSDB_CODE_INVALID_PARA; + } } + if (ltype == TSDB_DATA_TYPE_FLOAT) { float f = 0; if (IS_NUMERIC_TYPE(rtype)) { diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index eb8f2c741a..f07d6f4cc1 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -814,15 +814,17 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l } int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t dbRefId, ECHECKPOINT_BACKUP_TYPE type) { - int32_t code = 0; - char* path = NULL; - + int32_t code = 0; + char* path = NULL; + int64_t chkptSize = 0; SStreamMeta* pMeta = pTask->pMeta; const char* idStr = pTask->id.idStr; int64_t now = taosGetTimestampMs(); SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES); if (toDelFiles == NULL) { + stError("s-task:%s failed to prepare array list during upload checkpoint, code:%s", pTask->id.idStr, + tstrerror(terrno)); return terrno; } @@ -848,11 +850,11 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d } } - if (code == TSDB_CODE_SUCCESS) { - int32_t size = taosArrayGetSize(toDelFiles); - stDebug("s-task:%s remove redundant %d files", idStr, size); + int32_t num = taosArrayGetSize(toDelFiles); + if (code == TSDB_CODE_SUCCESS && num > 0) { + stDebug("s-task:%s remove redundant %d files", idStr, num); - for (int i = 0; i < size; i++) { + for (int i = 0; i < num; i++) { char* pName = taosArrayGetP(toDelFiles, i); code = deleteCheckpointFile(idStr, pName); if (code != 0) { @@ -868,12 +870,13 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d double el = (taosGetTimestampMs() - now) / 1000.0; if (code == TSDB_CODE_SUCCESS) { - stDebug("s-task:%s complete update checkpointId:%" PRId64 ", elapsed time:%.2fs remove local checkpoint data %s", - idStr, checkpointId, el, path); - taosRemoveDir(path); + code = taosGetDirSize(path, &chkptSize); + stDebug("s-task:%s complete upload checkpointId:%" PRId64 + ", elapsed time:%.2fs, checkpointSize:%.2fKiB local dir:%s", + idStr, checkpointId, el, SIZE_IN_KiB(chkptSize), path); } else { - stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " keep local checkpoint data, elapsed time:%.2fs", idStr, - checkpointId, el); + stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " elapsed time:%.2fs, checkpointSize:%.2fKiB", idStr, + checkpointId, el, SIZE_IN_KiB(chkptSize)); } taosMemoryFree(path); @@ -883,7 +886,7 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t checkpointId) { ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType(); if (type == DATA_UPLOAD_DISABLE) { - stDebug("s-task:%s not allowed to upload checkpoint data", pTask->id.idStr); + stDebug("s-task:%s not config to backup checkpoint data at snode, checkpointId:%"PRId64, pTask->id.idStr, checkpointId); return 0; } @@ -925,6 +928,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s gen checkpoint:%" PRId64 " failed, code:%s", id, ckId, tstrerror(terrno)); } + + int64_t et = taosGetTimestampMs(); + stDebug("s-task:%s gen local checkpoint completed, elapsed time:%.2fs", id, (et - startTs) / 1000.0); } // TODO: monitoring the checkpoint-source msg diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index ca5b6630fd..a6d0142010 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -243,6 +243,8 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { continue; } + // todo: this lock may blocked by lock in streamMetaStartOneTask function, which may lock a very long time when + // trying to load remote checkpoint data streamMutexLock(&pTask->lock); STaskStatusEntry entry = streamTaskGetStatusEntry(pTask); streamMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 605cf3fe21..350c71204a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1364,6 +1364,10 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) } pMeta->role = (isLeader) ? NODE_ROLE_LEADER : NODE_ROLE_FOLLOWER; + if (!isLeader) { + streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId); + } + streamMetaWUnLock(pMeta); if (isLeader) { diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index b0bcbd1a4c..23e6e5414e 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -345,7 +345,9 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (!pPage && pTxn->xMalloc != NULL) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { - tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + // when allocating from bufpool failed, it's time to flush cache. + // tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + terrno = ret; return NULL; } @@ -551,5 +553,4 @@ static void tdbPCacheCloseImpl(SPCache *pCache) { tdbOsFree(pCache->pgHash); tdbPCacheDestroyLock(pCache); - return ; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 6dc6aa0940..4c73d6470e 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -104,7 +104,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF } #endif - + /* if (rollback) { ret = tdbPagerRestoreJournals(pPager); if (ret < 0) { @@ -118,6 +118,13 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF return ret; } } + */ + // Always restore journal files with page flushing + ret = tdbPagerRestoreJournals(pPager); + if (ret < 0) { + tdbOsFree(pTb); + return ret; + } // pTb->pBt ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index 4715ccbd41..6319fce86b 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -18,3 +18,6 @@ target_link_libraries(tdbPageDefragmentTest tdb gtest gtest_main) add_executable(tdbPageRecycleTest "tdbPageRecycleTest.cpp") target_link_libraries(tdbPageRecycleTest tdb gtest gtest_main) +# page flush testing +add_executable(tdbPageFlushTest "tdbPageFlushTest.cpp") +target_link_libraries(tdbPageFlushTest tdb gtest gtest_main) diff --git a/source/libs/tdb/test/tdbPageFlushTest.cpp b/source/libs/tdb/test/tdbPageFlushTest.cpp new file mode 100644 index 0000000000..170a28af76 --- /dev/null +++ b/source/libs/tdb/test/tdbPageFlushTest.cpp @@ -0,0 +1,582 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#define ALLOW_FORBID_FUNC +#include "os.h" +#include "tdb.h" + +#include +#include +#include +#include +#include "tlog.h" + +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + clearPool(pPool); + taosMemoryFree(pPool); +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void *poolMallocRestricted(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + if (pPool->size > 1024 * 1024 * 10) { + return NULL; + } + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { + TDB *pEnv = NULL; + + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL); + if (ret) { + pEnv = NULL; + } + + return pEnv; +} + +static void clearDb(char const *db) { taosRemoveDir(db); } + +static void generateBigVal(char *val, int valLen) { + for (int i = 0; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } +} + +static void insertOfp(void) { + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + char const *key = "key123456789"; + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +static void insertMultipleOfp(void) { + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest) { +TEST(TdbPageFlushTest, TbRestoreTest) { + clearDb("tdb"); + + insertMultipleOfp(); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest2) { +TEST(TdbPageFlushTest, TbRestoreTest2) { + clearDb("tdb"); + + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest3) { +TEST(TdbPageFlushTest, TbRestoreTest3) { + clearDb("tdb"); + + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbAbort(pEnv, txn); + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest4) { +TEST(TdbPageFlushTest, TbRestoreTest4) { + clearDb("tdb"); + + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbAbort(pEnv, txn); + closePool(pPool); + tdbTbClose(pDb); + tdbClose(pEnv); + + pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + pPool = openPool(); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest5) { +TEST(TdbPageFlushTest, TbRestoreTest5) { + clearDb("tdb"); + + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + closePool(pPool); + tdbTbClose(pDb); + tdbClose(pEnv); + + pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + pPool = openPool(); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} diff --git a/source/util/src/mpChunk.c b/source/util/src/mpChunk.c index 2c1c415c04..c7bdcd809f 100755 --- a/source/util/src/mpChunk.c +++ b/source/util/src/mpChunk.c @@ -190,6 +190,8 @@ int32_t mpChunkNSAllocMem(SMemPool* pPool, SMPSession* pSession, int64_t size, u void* pRes = NULL; int64_t totalSize = size + sizeof(SMPMemHeader) + sizeof(SMPMemTailer) + alignment; + + MP_ERR_JRET(mpChunkNewNS(pPool, &pChunk, totalSize)); SMPMemHeader* pHeader = (SMPMemHeader*)pChunk->pMemStart; MP_INIT_MEM_HEADER(pHeader, size, false); diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 4427ddcea5..98007c6d8d 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -19,6 +19,7 @@ import random import copy import json +import frame.eos import frame.etool import frame.eutil from frame.log import * @@ -427,6 +428,42 @@ class TBase: return db, stb, child_count, insert_rows + # insert & check + def benchInsert(self, jsonFile, options = "", results = None): + # exe insert + benchmark = frame.etool.benchMarkFile() + cmd = f"{benchmark} {options} -f {jsonFile}" + rlist = frame.eos.runRetList(cmd, True, True, True) + if results != None: + for result in results: + self.checkListString(rlist, result) + + # open json + with open(jsonFile, "r") as file: + data = json.load(file) + + # read json + dbs = data["databases"] + for db in dbs: + dbName = db["dbinfo"]["name"] + stbs = db["super_tables"] + for stb in stbs: + stbName = stb["name"] + child_count = stb["childtable_count"] + insert_rows = stb["insert_rows"] + timestamp_step = stb["timestamp_step"] + + # check result + + # count + sql = f"select count(*) from {dbName}.{stbName}" + tdSql.checkAgg(sql, child_count * insert_rows) + # diff + sql = f"select * from (select diff(ts) as dif from {dbName}.{stbName} partition by tbname) where dif != {timestamp_step};" + tdSql.query(sql) + tdSql.checkRows(0) + # show + tdLog.info(f"insert check passed. db:{dbName} stb:{stbName} child_count:{child_count} insert_rows:{insert_rows}\n") # tmq def tmqBenchJson(self, jsonFile, options="", checkStep=False): diff --git a/tests/army/tools/benchmark/basic/insertBasic.py b/tests/army/tools/benchmark/basic/insertBasic.py index f1d3b81732..dfc5e3aff1 100644 --- a/tests/army/tools/benchmark/basic/insertBasic.py +++ b/tests/army/tools/benchmark/basic/insertBasic.py @@ -13,6 +13,7 @@ import os import json import frame +import frame.eos import frame.etool from frame.log import * from frame.cases import * @@ -109,7 +110,6 @@ class TDTestCase(TBase): tdLog.info(f" vgroups real={tdSql.getData(0,0)} expect={vgroups}") tdSql.checkData(0, 0, vgroups, True) - # bugs ts def checkVGroups(self, benchmark): # vgroups with command line set @@ -117,12 +117,19 @@ class TDTestCase(TBase): # vgroups with json file self.testBenchmarkJson(benchmark, "./tools/benchmark/basic/json/insertBasic.json", "", True) + + def checkInsertManyStb(self): + # many stb + self.benchInsert("./tools/benchmark/basic/json/insertManyStb.json") + def run(self): - benchmark = etool.benchMarkFile() + benchmark = frame.etool.benchMarkFile() # vgroups self.checkVGroups(benchmark) + # check many stable + self.checkInsertManyStb() def stop(self): tdSql.close() diff --git a/tests/army/tools/benchmark/basic/json/insertManyStb.json b/tests/army/tools/benchmark/basic/json/insertManyStb.json new file mode 100644 index 0000000000..caa418c858 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/insertManyStb.json @@ -0,0 +1,151 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 3000, + "thread_count": 2, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "test1", + "drop": "yes", + "precision": "us", + "vgroups": 1 + }, + "super_tables": [ + { + "name": "meters1", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 1000, + "childtable_prefix": "da", + "insert_mode": "stmt", + "timestamp_step": 15, + "start_timestamp":1700000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters2", + "child_table_exists": "no", + "childtable_count": 3, + "insert_rows": 100, + "childtable_prefix": "db", + "insert_mode": "stmt", + "timestamp_step": 20, + "interlace_rows": 1, + "start_timestamp":1700000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters3", + "child_table_exists": "no", + "childtable_count": 5, + "insert_rows": 100, + "childtable_prefix": "dc", + "insert_mode": "stmt2", + "timestamp_step": 20, + "interlace_rows": 1, + "start_timestamp":1700000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters4", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 70, + "childtable_prefix": "dd", + "insert_mode": "stmt2", + "timestamp_step": 50, + "interlace_rows": 0, + "start_timestamp":1700000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + } + ] + }, + { + "dbinfo": { + "name": "test2", + "drop": "yes", + "precision": "ns", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters1", + "child_table_exists": "no", + "childtable_count": 3, + "insert_rows": 120, + "childtable_prefix": "de", + "insert_mode": "taosc", + "timestamp_step": 15, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + }, + { + "name": "meters2", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 200, + "childtable_prefix": "df", + "insert_mode": "taosc", + "timestamp_step": 2, + "interlace_rows": 4, + "start_timestamp":1700000000000000000, + "columns": [ + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "binary", "name": "bin", "len": 4} + ], + "tags": [ + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/script/tsim/analytics/basic0.sim b/tests/script/tsim/analytics/basic0.sim index 999b2fff37..0d9a29a19b 100644 --- a/tests/script/tsim/analytics/basic0.sim +++ b/tests/script/tsim/analytics/basic0.sim @@ -91,6 +91,14 @@ sql_error select count(*) from ct1 anomaly_window(c6, 'algo=ksigma,k=2'); sql_error select forecast(c6, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 +print ==================== invalid timeout parameter, will reset the parameters. +sql select forecast(c1, 'algo=holtwinters, timeout=6000') from ct1; +sql select forecast(c1, 'algo=holtwinters, timeout=0') from ct1; + +print =========================== valid timeout +sql select forecast(c1, 'algo=holtwinters, timeout=120') from ct1; + + sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1 diff --git a/tests/system-test/0-others/test_show_disk_usage.py b/tests/system-test/0-others/test_show_disk_usage.py index eb5bdf1aa7..1c863d7cdf 100644 --- a/tests/system-test/0-others/test_show_disk_usage.py +++ b/tests/system-test/0-others/test_show_disk_usage.py @@ -102,7 +102,8 @@ class TDTestCase: elif "Compress_radio=" in item[0]: value = item[0].split("=")[1].split(" ")[0].replace("[", "").replace("]", "") if value != 'NULL': - compress_radio = float(value) + tValue = value[0:len(value) - 1] + compress_radio = float(tValue) #tdLog.debug("compress_occupied: %s" % compress_radio) return disk_occupied, compress_radio diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 110a644e90..1ee2bc4ce6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -19,7 +19,8 @@ IF(TD_WEBSOCKET) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND RUSTFLAGS=-Ctarget-feature=-crt-static cargo build --release --locked -p taos-ws-sys --features rustls + COMMAND cargo update + COMMAND RUSTFLAGS=-Ctarget-feature=-crt-static cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include @@ -37,7 +38,8 @@ IF(TD_WEBSOCKET) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND cargo build --release --locked -p taos-ws-sys --features rustls + COMMAND cargo update + COMMAND cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/taosws.dll ${CMAKE_BINARY_DIR}/build/lib COMMAND cp target/release/taosws.dll.lib ${CMAKE_BINARY_DIR}/build/lib/taosws.lib @@ -56,7 +58,8 @@ IF(TD_WEBSOCKET) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND cargo build --release --locked -p taos-ws-sys --features rustls + COMMAND cargo update + COMMAND cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include diff --git a/tools/taos-tools/src/benchInsert.c b/tools/taos-tools/src/benchInsert.c index 885a6e529f..894e4621a0 100644 --- a/tools/taos-tools/src/benchInsert.c +++ b/tools/taos-tools/src/benchInsert.c @@ -1928,7 +1928,7 @@ static void *syncWriteInterlace(void *sarg) { bindv = createBindV(nBatchTable, tagCnt, stbInfo->cols->size + 1); } - bool oldInitStmt = stbInfo->autoTblCreating || database->superTbls->size > 1; + bool oldInitStmt = stbInfo->autoTblCreating; // not auto create table call once if(stbInfo->iface == STMT_IFACE && !oldInitStmt) { debugPrint("call prepareStmt for stable:%s\n", stbInfo->stbName); @@ -2900,7 +2900,7 @@ void *syncWriteProgressive(void *sarg) { tagData = benchCalloc(TAG_BATCH_COUNT, stbInfo->lenOfTags, false); } - bool oldInitStmt = stbInfo->autoTblCreating || database->superTbls->size > 1; + bool oldInitStmt = stbInfo->autoTblCreating; // stmt. not auto table create call on stmt if (stbInfo->iface == STMT_IFACE && !oldInitStmt) { if (prepareStmt(pThreadInfo->conn->stmt, stbInfo, tagData, w)) { @@ -3909,11 +3909,8 @@ int32_t initInsertThread(SDataBase* database, SSuperTable* stbInfo, int32_t nthr if (NULL == pThreadInfo->conn) { goto END; } + // single always true for benchmark bool single = true; - if (database->superTbls->size > 1) { - single = false; - } - if (stbInfo->iface == STMT2_IFACE) { // stmt2 init if (pThreadInfo->conn->stmt2) diff --git a/tools/tdgpt/requirements.txt b/tools/tdgpt/requirements.txt index ad4c42d7cf..b123e186e6 100644 --- a/tools/tdgpt/requirements.txt +++ b/tools/tdgpt/requirements.txt @@ -51,4 +51,4 @@ uWSGI==2.0.27 Werkzeug==3.0.4 Flask-Testing==0.8.1 xlsxwriter==3.2.1 -taospy==2.7.16 \ No newline at end of file +taospy==2.7.16 diff --git a/tools/tdgpt/taosanalytics/algo/ad/autoencoder.py b/tools/tdgpt/taosanalytics/algo/ad/autoencoder.py index 0d3bb21faa..e58db3f54b 100644 --- a/tools/tdgpt/taosanalytics/algo/ad/autoencoder.py +++ b/tools/tdgpt/taosanalytics/algo/ad/autoencoder.py @@ -8,8 +8,8 @@ import numpy as np import pandas as pd from taosanalytics.conf import app_logger, conf -from taosanalytics.misc.train_model import create_sequences from taosanalytics.service import AbstractAnomalyDetectionService +from taosanalytics.util import create_sequences class _AutoEncoderDetectionService(AbstractAnomalyDetectionService):