diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 7ce0a5ce48..b7b97cee8e 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -9,17 +9,12 @@ on: paths-ignore: - 'packaging/**' - 'docs/**' - repository_dispatch: - types: [trigger-tests-from-tdinternal] concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.event.client_payload.ref}}-${{ github.event_name == 'repository_dispatch' && 'dispatch' || ''}} + group: ${{ github.workflow }}-${{ github.ref }}-TDengine cancel-in-progress: true env: - CONTAINER_NAME: 'taosd-test' - WKDIR: '/var/lib/jenkins/workspace' - WK: '/var/lib/jenkins/workspace/TDinternal' WKC: '/var/lib/jenkins/workspace/TDinternal/community' jobs: @@ -28,439 +23,80 @@ jobs: group: CI labels: [self-hosted, Linux, X64, testing] outputs: - tdinternal: ${{ steps.parameters.outputs.tdinternal }} run_function_test: ${{ steps.parameters.outputs.run_function_test }} run_tdgpt_test: ${{ steps.parameters.outputs.run_tdgpt_test }} - source_branch: ${{ steps.parameters.outputs.source_branch }} - target_branch: ${{ steps.parameters.outputs.target_branch }} - pr_number: ${{ steps.parameters.outputs.pr_number }} steps: - name: Determine trigger source and fetch parameters id: parameters run: | set -euo pipefail - # check the trigger source and get branch information - if [ "${{ github.event_name }}" == "repository_dispatch" ]; then - tdinternal="true" - source_branch=${{ github.event.client_payload.tdinternal_source_branch }} - target_branch=${{ github.event.client_payload.tdinternal_target_branch }} - pr_number=${{ github.event.client_payload.tdinternal_pr_number }} + target_branch=${{ github.event.pull_request.base.ref }} + + # Check whether to run tdgpt test cases + cd ${{ env.WKC }} + changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) + echo "changed files exclude doc: ${changed_files_non_doc}" + + if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then run_tdgpt_test="true" + else + run_tdgpt_test="false" + fi + echo "run tdgpt test: ${run_tdgpt_test}" + + # Check whether to run function test cases + changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ + grep -v "^docs/en/" | \ + grep -v "^docs/zh/" | \ + grep -v ".md$" | \ + grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ + tr '\n' ' ' || :) + echo "changed files exclude tdgpt: ${changed_files_non_tdgpt}" + + if [ -n "$changed_files_non_tdgpt" ]; then run_function_test="true" else - tdinternal="false" - source_branch=${{ github.event.pull_request.head.ref }} - target_branch=${{ github.event.pull_request.base.ref }} - pr_number=${{ github.event.pull_request.number }} - - # check whether to run tdgpt test cases - cd ${{ env.WKC }} - changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) - echo “changed files exclude doc, ${changed_files_non_doc}" - if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then - run_tdgpt_test="true" - else - run_tdgpt_test="false" - fi - echo "run tdgpt test: ${run_tdgpt_test}" - - # check whether to run function test cases - changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ - grep -v "^docs/en/" | \ - grep -v "^docs/zh/" | \ - grep -v ".md$" | \ - grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ - tr '\n' ' ' || :) - echo "changed file exclude tdgpt: ${changed_files_non_tdgpt}" - if [ -n "$changed_files_non_tdgpt" ]; then - run_function_test="true" - else - run_function_test="false" - fi - echo "run function test: ${run_function_test}" + run_function_test="false" fi - echo "tdinternal=$tdinternal" >> $GITHUB_OUTPUT + echo "run function test: ${run_function_test}" + + # Output the results for GitHub Actions echo "run_function_test=$run_function_test" >> $GITHUB_OUTPUT echo "run_tdgpt_test=$run_tdgpt_test" >> $GITHUB_OUTPUT - echo "source_branch=$source_branch" >> $GITHUB_OUTPUT - echo "target_branch=$target_branch" >> $GITHUB_OUTPUT - echo "pr_number=$pr_number" >> $GITHUB_OUTPUT + + echo ${{ github.event.pull_request.head.ref }} + echo ${{ github.event.pull_request.base.ref }} + echo ${{ github.event.pull_request.number }} run-tests-on-linux: + uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml needs: fetch-parameters - runs-on: - group: CI - labels: [self-hosted, Linux, X64, testing] - timeout-minutes: 200 - env: - IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }} - RUN_RUNCTION_TEST: ${{ needs.fetch-parameters.outputs.run_function_test }} - RUN_TDGPT_TEST: ${{ needs.fetch-parameters.outputs.run_tdgpt_test }} - SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }} - TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }} - PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }} - steps: - - name: Output the environment information - run: | - echo "::group::Environment Info" - date - hostname - env - echo "Runner: ${{ runner.name }}" - echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}" - echo "Workspace: ${{ env.WKDIR }}" - git --version - echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}" - echo "::endgroup::" - - - name: Prepare repositories - run: | - set -euo pipefail - prepare_environment() { - cd "$1" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "$2" - } - prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}" - prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}" - - - name: Get latest codes and logs for TDinternal PR - if: ${{ env.IS_TDINTERNAL == 'true' }} - run: | - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Get latest codes and logs for TDengine PR - if: ${{ env.IS_TDINTERNAL == 'false' }} - run: | - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Update submodule - run: | - cd ${{ env.WKC }} - git submodule update --init --recursive - - name: Output the 'file_no_doc_changed' information to the file - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - mkdir -p ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }} - cd ${{ env.WKC }} - changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ env.TARGET_BRANCH }}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :) - echo $changed_files_non_doc > ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt - - name: Check assert testing - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - ./run_check_assert_container.sh -d ${{ env.WKDIR }} - - name: Check void function testing - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - ./run_check_void_container.sh -d ${{ env.WKDIR }} - - name: Build docker container - if: ${{ env.RUN_RUNCTION_TEST == 'true' }} - run: | - date - rm -rf ${{ env.WKC }}/debug - cd ${{ env.WKC }}/tests/parallel_test - time ./container_build.sh -w ${{ env.WKDIR }} -e - - name: Get parameters for testing - id: get_param - run: | - log_server_file="/home/log_server.json" - timeout_cmd="" - extra_param="" - - if [ -f "$log_server_file" ]; then - log_server_enabled=$(jq '.enabled' "$log_server_file") - timeout_param=$(jq '.timeout' "$log_server_file") - if [ "$timeout_param" != "null" ] && [ "$timeout_param" != "0" ]; then - timeout_cmd="timeout $timeout_param" - fi - - if [ "$log_server_enabled" == "1" ]; then - log_server=$(jq '.server' "$log_server_file" | sed 's/\\\"//g') - if [ "$log_server" != "null" ] && [ "$log_server" != "" ]; then - extra_param="-w $log_server" - fi - fi - fi - echo "timeout_cmd=$timeout_cmd" >> $GITHUB_OUTPUT - echo "extra_param=$extra_param" >> $GITHUB_OUTPUT - - name: Run function returns with a null pointer scan testing - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - ./run_scan_container.sh -d ${{ env.WKDIR }} -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -f ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt ${{ steps.get_param.outputs.extra_param }} - - name: Run tdgpt test cases - if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' && env.RUN_TDGPT_TEST == 'true' }} - run: | - cd ${{ env.WKC }}/tests/parallel_test - export DEFAULT_RETRY_TIME=2 - date - timeout 600 time ./run.sh -e -m /home/m.json -t tdgpt_cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 300 ${{ steps.get_param.outputs.extra_param }} - - name: Run function test cases - if: ${{ env.RUN_RUNCTION_TEST == 'true'}} - run: | - cd ${{ env.WKC }}/tests/parallel_test - export DEFAULT_RETRY_TIME=2 - date - ${{ steps.get_param.outputs.timeout_cmd }} time ./run.sh -e -m /home/m.json -t cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 1200 ${{ steps.get_param.outputs.extra_param }} + if: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || needs.fetch-parameters.outputs.run_function_test == 'true' }} + with: + tdinternal: false + run_function_test: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' || false }} + run_tdgpt_test: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || false }} + source_branch: ${{ github.event.pull_request.head.ref }} + target_branch: ${{ github.event.pull_request.base.ref }} + pr_number: ${{ github.event.pull_request.number }} run-tests-on-mac: + uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} - runs-on: - group: CI - labels: [self-hosted, macOS, testing] - timeout-minutes: 60 - env: - IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }} - SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }} - TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }} - PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }} - steps: - - name: Output the environment information - run: | - echo "::group::Environment Info" - date - hostname - env - echo "Runner: ${{ runner.name }}" - echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}" - echo "Workspace: ${{ env.WKDIR }}" - git --version - echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}" - echo "::endgroup::" - - name: Prepare repositories - run: | - set -euo pipefail - prepare_environment() { - cd "$1" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "$2" - } - prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}" - prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}" - - name: Get latest codes and logs for TDinternal PR - if: ${{ env.IS_TDINTERNAL == 'true' }} - run: | - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Get latest codes and logs for TDengine PR - if: ${{ env.IS_TDINTERNAL == 'false' }} - run: | - cd ${{ env.WKC }} - git remote prune origin - git pull >/dev/null - git log -5 - echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log - echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge - git checkout -qf FETCH_HEAD - git log -5 - echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - cd ${{ env.WK }} - git pull >/dev/null - git log -5 - echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log - - name: Update submodule - run: | - cd ${{ env.WKC }} - git submodule update --init --recursive - - name: Run tests - run: | - date - cd ${{ env.WK }} - rm -rf debug - mkdir debug - cd ${{ env.WK }}/debug - echo $PATH - echo "PATH=/opt/homebrew/bin:$PATH" >> $GITHUB_ENV - cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false -DCMAKE_BUILD_TYPE=Release - make -j10 - ctest -j10 || exit 7 - date + with: + tdinternal: false + source_branch: ${{ github.event.pull_request.head.ref }} + target_branch: ${{ github.event.pull_request.base.ref }} + pr_number: ${{ github.event.pull_request.number }} run-tests-on-windows: + uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} - runs-on: - group: CI - labels: [self-hosted, Windows, X64, testing] - timeout-minutes: 126 - env: - IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }} - SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }} - TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }} - PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }} - WIN_INTERNAL_ROOT: "C:\\workspace\\0\\TDinternal" - WIN_COMMUNITY_ROOT: "C:\\workspace\\0\\TDinternal\\community" - WIN_SYSTEM_TEST_ROOT: "C:\\workspace\\0\\TDinternal\\community\\tests\\system-test" - WIN_VS_PATH: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" - WIN_CPU_TYPE: "x64" - steps: - - name: Output the environment information - run: | - hostname - taskkill /f /t /im python.exe - taskkill /f /t /im bash.exe - taskkill /f /t /im taosd.exe - ipconfig - set - date /t - time /t - rd /s /Q "%WIN_INTERNAL_ROOT%\debug" || exit 0 - shell: cmd - - name: Prepare repositories - run: | - :: Prepare internal repository - if exist "%WIN_INTERNAL_ROOT%" ( - cd /d "%WIN_INTERNAL_ROOT%" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "%TARGET_BRANCH%" - ) else ( - echo Directory does not exist: "%WIN_INTERNAL_ROOT%" - exit 1 - ) - - :: Prepare community repository - if exist "%WIN_COMMUNITY_ROOT%" ( - cd /d "%WIN_COMMUNITY_ROOT%" - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "%TARGET_BRANCH%" - ) else ( - echo Directory does not exist: "%WIN_COMMUNITY_ROOT%" - exit 1 - ) - shell: cmd - - name: Get latest codes and logs for TDinternal PR - if: ${{ env.IS_TDINTERNAL == 'true' }} - run: | - cd %WIN_INTERNAL_ROOT% - git pull origin %TARGET_BRANCH% - git fetch origin +refs/pull/%PR_NUMBER%/merge - git checkout -qf FETCH_HEAD - cd %WIN_COMMUNITY_ROOT% - git remote prune origin - git pull - shell: cmd - - name: Get latest codes and logs for TDengine PR - if: ${{ env.IS_TDINTERNAL == 'false' }} - run: | - cd %WIN_INTERNAL_ROOT% - git pull origin %TARGET_BRANCH% - cd %WIN_COMMUNITY_ROOT% - git remote prune origin - git pull origin %TARGET_BRANCH% - git fetch origin +refs/pull/%PR_NUMBER%/merge - git checkout -qf FETCH_HEAD - shell: cmd - - name: Output branch and log information - run: | - cd %WIN_INTERNAL_ROOT% - git branch - git log -5 - - cd %WIN_COMMUNITY_ROOT% - git branch - git log -5 - shell: cmd - - name: Update submodule - run: | - cd %WIN_COMMUNITY_ROOT% - git submodule update --init --recursive - shell: cmd - - name: Build on windows - run: | - echo "building ..." - time /t - cd %WIN_INTERNAL_ROOT% - mkdir debug - cd debug - time /t - call "%WIN_VS_PATH%" %WIN_CPU_TYPE% - set CL=/MP8 - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" - time /t - cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true || exit 7 - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" - time /t - jom -j 6 || exit 8 - time /t - - cd %WIN_COMMUNITY_ROOT%/tests/ci - pip3 install taospy==2.7.21 - pip3 install taos-ws-py==0.3.8 - xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 - shell: cmd - - name: Run ctest - run: | - echo "windows ctest ..." - time /t - cd %WIN_INTERNAL_ROOT%\\debug - ctest -j 1 || exit 7 - time /t - shell: cmd - - name: Run function test - run: | - echo "windows test ..." - xcopy /e/y/i/f "%WIN_INTERNAL_ROOT%\debug\build\lib\taos.dll" C:\Windows\System32 - ls -l "C:\Windows\System32\taos.dll" - time /t - cd %WIN_SYSTEM_TEST_ROOT% - echo "testing ..." - test-all.bat ci - time /t - shell: cmd + with: + tdinternal: false + source_branch: ${{ github.event.pull_request.head.ref }} + target_branch: ${{ github.event.pull_request.base.ref }} + pr_number: ${{ github.event.pull_request.number }} diff --git a/docs/en/06-advanced/05-data-in/07-mqtt.md b/docs/en/06-advanced/05-data-in/07-mqtt.md index 73ef3b534c..47ffa82bf2 100644 --- a/docs/en/06-advanced/05-data-in/07-mqtt.md +++ b/docs/en/06-advanced/05-data-in/07-mqtt.md @@ -86,9 +86,15 @@ The keep alive interval is the time interval negotiated between the client and t In **Clean Session**, choose whether to clear the session. The default value is true. -Fill in the Topic names to be consumed in **Subscription Topics and QoS Configuration**. Use the following format: `topic1::0,topic2::1`. +In the **Topics Qos Config**, fill in the topic name and QoS to subscribe. Use the following format: `{topic_name}::{qos}` (e.g., `my_topic::0`). MQTT protocol 5.0 supports shared subscriptions, allowing multiple clients to subscribe to the same topic for load balancing. Use the following format: `$share/{group_name}/{topic_name}::{qos}`, where `$share` is a fixed prefix indicating the enablement of shared subscription, and `group_name` is the client group name, similar to Kafka's consumer group. -Click the **Check Connectivity** button to check if the data source is available. +In the **Topic Analysis**, fill in the MQTT topic parsing rules. The format is the same as the MQTT Topic, parsing each level of the MQTT Topic into corresponding variable names, with `_` indicating that the current level is ignored during parsing. For example: if the MQTT Topic `a/+/c` corresponds to the parsing rule `v1/v2/_`, it means assigning the first level `a` to variable `v1`, the value of the second level (where the wildcard `+` represents any value) to variable `v2`, and ignoring the value of the third level `c`, which will not be assigned to any variable. In the `payload parsing` below, the variables obtained from Topic parsing can also participate in various transformations and calculations. + +In the **Compression**, configure the message body compression algorithm. After receiving the message, taosX uses the corresponding compression algorithm to decompress the message body and obtain the original data. Options include none (no compression), gzip, snappy, lz4, and zstd, with the default being none. + +In the **Char Encoding**, configure the message body encoding format. After receiving the message, taosX uses the corresponding encoding format to decode the message body and obtain the original data. Options include UTF_8, GBK, GB18030, and BIG5, with the default being UTF_8. + +Click the **Check Connection** button to check if the data source is available.
diff --git a/docs/en/08-operation/03-deployment/03-kubernetes.md b/docs/en/08-operation/03-deployment/03-kubernetes.md index dbb6022ce6..fdec50b9d6 100644 --- a/docs/en/08-operation/03-deployment/03-kubernetes.md +++ b/docs/en/08-operation/03-deployment/03-kubernetes.md @@ -339,7 +339,7 @@ Helm operates Kubernetes using kubectl and kubeconfig configurations, which can The TDengine Chart has not yet been released to the Helm repository, it can currently be downloaded directly from GitHub: ```shell -wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-enterpise-3.5.0.tgz +wget https://github.com/taosdata/TDengine-Operator/raw/refs/heads/3.0/helm/tdengine-enterprise-3.5.0.tgz ``` Note that it's for the enterprise edition, and the community edition is not yet available. diff --git a/docs/en/14-reference/01-components/01-taosd.md b/docs/en/14-reference/01-components/01-taosd.md index 4527a7fcac..5ce9a40ecb 100644 --- a/docs/en/14-reference/01-components/01-taosd.md +++ b/docs/en/14-reference/01-components/01-taosd.md @@ -43,7 +43,7 @@ After modifying configuration file parameters, you need to restart the *taosd* s |resolveFQDNRetryTime | Cancelled after 3.x |Not supported |Number of retries when FQDN resolution fails| |timeToGetAvailableConn | Cancelled after 3.3.4.x |Maximum waiting time to get an available connection, range 10-50000000, in milliseconds, default value 500000| |maxShellConns | Cancelled after 3.x |Supported, effective after restart|Maximum number of connections allowed| -|maxRetryWaitTime | |Supported, effective after restart|Maximum timeout for reconnection,calculated from the time of retry,range is 0-86400000,in milliseconds, default value 10000| +|maxRetryWaitTime | |Supported, effective after restart|Maximum timeout for reconnection,calculated from the time of retry,range is 3000-86400000,in milliseconds, default value 10000| |shareConnLimit |Added in 3.3.4.0 |Supported, effective after restart|Number of requests a connection can share, range 1-512, default value 10| |readTimeout |Added in 3.3.4.0 |Supported, effective after restart|Minimum timeout for a single request, range 64-604800, in seconds, default value 900| diff --git a/docs/en/26-tdinternal/09-stream.md b/docs/en/26-tdinternal/09-stream.md index 2c8d41cccb..69424652dc 100644 --- a/docs/en/26-tdinternal/09-stream.md +++ b/docs/en/26-tdinternal/09-stream.md @@ -94,7 +94,7 @@ The sink task is responsible for receiving the output results from the agg task The above three types of tasks each play their roles in the stream computing architecture, distributed at different levels. Clearly, the number of source tasks directly depends on the number of vnodes, with each source task independently handling the data in its vnode without interference from other source tasks, and there are no sequential constraints. However, it is worth noting that if the final stream computing results converge to one table, then only one sink task will be deployed on the vnode where that table is located. The collaborative relationship between these three types of tasks is shown in the following diagram, together forming the complete execution process of stream computing tasks.
-Relationships between tasks +Relationships between tasks
Figure 3. Relationships between tasks
diff --git a/docs/en/27-train-faq/02-dst.md b/docs/en/27-train-faq/02-dst.md new file mode 100644 index 0000000000..6fda8f8a16 --- /dev/null +++ b/docs/en/27-train-faq/02-dst.md @@ -0,0 +1,293 @@ +--- +title: DST(Daylight Saving Time) Usage +description: Explanation and suggestions for using DST(Daylight Saving Time) in TDengine +--- + +## Background + +In the use of time-series databases, there are times when Daylight Saving Time (DST) is encountered. We analyze and explain the use and issues of DST in TDengine to help you use TDengine more smoothly. + +## Definitions + +### Time Zone + +A time zone is a region on Earth that uses the same standard time. Due to the Earth's rotation, to ensure that the time in each place is coordinated with the local sunrise and sunset, the world is divided into multiple time zones. + +### IANA Time Zone + +The IANA (Internet Assigned Numbers Authority) time zone database, also known as the tz database, provides a standard reference for global time zone information. It is the basis for modern systems and software to handle time zone-related operations. + +IANA uses the "Region/City" format (e.g., Europe/Berlin) to clearly identify time zones. + +TDengine supports the use of IANA time zones in different components (except for the time zone settings in Windows taos.cfg). + +### Standard Time and Local Time + +Standard time is the time determined based on a fixed meridian on Earth. It provides a unified reference point for each time zone. + +- Greenwich Mean Time (GMT): Historically used reference time, located at the 0° meridian. +- Coordinated Universal Time (UTC): The modern time standard, similar to GMT but more precise. + +The relationship between standard time and time zones is as follows: + +- Reference: Standard time (e.g., UTC) is the reference point for setting time zones. +- Offset: Different time zones are defined by their offset from standard time. For example, UTC+1 means 1 hour ahead of UTC. +- Regional Division: The world is divided into multiple time zones, each using one or more standard times. + +Relative to standard time, each region sets its local time based on its time zone: + +- Time Zone Offset: Local time equals standard time plus the offset of the time zone. For example, UTC+2 means 2 hours ahead of UTC. +- Daylight Saving Time (DST): Some regions adjust their local time during specific periods, such as moving the clock forward by one hour. See the next section for details. + +### Daylight Saving Time + +Daylight Saving Time (DST) is a system that advances the time by one hour to make better use of daylight and save energy. It usually starts in spring and ends in autumn. The specific start and end times of DST vary by region. The following explanation uses Berlin time as an example to illustrate DST and its effects. + +![DST Berlin](./02-dst/dst-berlin.png) + +According to this rule, you can see: + +- The time between 02:00:00 and 03:00:00 (excluding 03:00:00) on March 31, 2024, in Berlin local time does not exist (jump). +- The time between 02:00:00 and 03:00:00 (excluding 03:00:00) on October 27, 2024, in Berlin local time appears twice. + +#### DST and the IANA Time Zone Database + +- Recording Rules: The IANA time zone database records detailed DST rules for each region, including the start and end dates and times. +- Automatic Adjustment: Many operating systems and software use the IANA database to automatically handle DST adjustments. +- Historical Changes: The IANA database also tracks historical DST changes to ensure accuracy. + +#### DST and Timestamp Conversion + +- Converting a timestamp to local time is deterministic. For example, 1729990654 is Berlin time DST 2024-10-27 02:57:34, and 1729994254 is Berlin time standard time 2024-10-27 02:57:34 (these two local times are the same except for the time offset). +- Without specifying the time offset, converting local time to a timestamp is indeterminate. The time skipped during DST does not exist and cannot be converted to a timestamp, such as Berlin time 2024-03-31 02:34:56 does not exist and cannot be converted to a timestamp. The repeated time during the end of DST cannot determine which timestamp it is, such as 2024-10-27 02:57:34 without specifying the time offset cannot determine whether it is 1729990654 or 1729994254. Specifying the time offset can determine the timestamp, such as 2024-10-27 02:57:34 CEST(+02:00), specifying DST 2024-10-27 02:57:34 timestamp 1729990654. + +### RFC3339 Time Format + +RFC 3339 is an internet time format standard used to represent dates and times. It is based on the ISO 8601 standard but specifies some format details more specifically. + +The format is as follows: + +- Basic Format: `YYYY-MM-DDTHH:MM:SSZ` +- Time Zone Representation: + - Z represents Coordinated Universal Time (UTC). + - Offset format, such as +02:00, represents the time difference from UTC. + +With explicit time zone offsets, the RFC 3339 format can accurately parse and compare times globally. + +The advantages of RFC 3339 include: + +- Standardization: Provides a unified format for easy cross-system data exchange. +- Clarity: Clearly indicates time zone information, avoiding time misunderstandings. + +TDengine uses the RFC3339 format for display in REST API and Explorer UI. In SQL statements, you can use the RFC3339 format to write timestamp data: + +```sql +insert into t1 values('2024-10-27T01:59:59.000Z', 0); +select * from t1 where ts >= '2024-10-27T01:59:59.000Z'; +``` + +### Undefined Behavior + +Undefined behavior refers to specific code or operations that do not have a clearly defined result and do not guarantee compatibility with that result. TDengine may modify the current behavior in a future version without notifying users. Therefore, users should not rely on the current undefined behavior for judgment or application in TDengine. + +## Writing and Querying DST in TDengine + +We use the following table to show the impact of DST on writing and querying. + +![DST Table](./02-dst/dst-table.png) + +### Table Explanation + +- **TIMESTAMP**: TDengine uses a 64-bit integer to store raw timestamps. +- **UTC**: The UTC time representation corresponding to the timestamp. +- **Europe/Berlin**: The RFC3339 format time corresponding to the Europe/Berlin time zone. +- **Local**: The local time corresponding to the Europe/Berlin time zone (without time zone). + +### Table Analysis + +- At the **start of DST** (Berlin time March 31, 02:00), the time jumps directly from 02:00 to 03:00 (one hour forward). + - Light green is the timestamp one hour before the start of DST; + - Dark green is the timestamp one hour after the start of DST; + - Red indicates that the nonexistent local time was inserted into the TDengine database: + - Using SQL `INSERT INTO t1 values('2024-03-31 02:59:59',..)` to insert data from `2024-03-31 02:00:00` to `2024-03-31 02:59:59` will be automatically adjusted to -1000 (in TDengine, this is undefined behavior, currently this value is related to the database precision, millisecond database is -1000, microsecond database is -1000000, nanosecond database is -1000000000), because that moment does not exist in local time; +- At the **end of DST** (Berlin time October 27, 03:00), the time jumps from 03:00 to 02:00 (one hour back). + - Light blue indicates the timestamp one hour before the clock jump; + - Dark blue indicates the timestamp within one hour after the clock jump, its local time without time zone is the same as the previous hour. + - Purple indicates the timestamp one hour after the clock jump; +- **Local Time Changes**: It can be seen that due to the adjustment of DST, local time changes, which may cause some time periods to appear repeated or missing. +- **UTC Time Unchanged**: UTC time remains unchanged, ensuring the consistency and order of time. +- **RFC3339**: The RFC3339 format time shows the change in time offset, changing to +02:00 after the start of DST and to +01:00 after the end of DST. +- **Conditional Query**: + - At the **start of DST**, the skipped time (`[03-31 02:00:00,03-31 03:00:00)`) does not exist, so using that time for queries results in undefined behavior: `SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'` (the nonexistent local timestamp is converted to `-1000`): + + ```sql + taos> SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'; + ts | + ================= + -1000 | + Query OK, 1 row(s) in set (0.003635s) + ``` + + When the nonexistent timestamp is used together with the existing timestamp, the result is also not as expected, as shown below where the start local time does not exist: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 03:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + -1000 | 1969-12-31T23:59:59.000Z | + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + 1711846800000 | 2024-03-31T01:00:00.000Z | + 1711846801000 | 2024-03-31T01:00:01.000Z | + Query OK, 5 row(s) in set (0.003339s) + ``` + + In the following statements, the first SQL query end time does not exist, and the second end time exists. The first SQL query result is not as expected: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 02:00:00'; + Query OK, 0 row(s) in set (0.000930s) + + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 01:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + Query OK, 2 row(s) in set (0.001227s) + ``` + + - At the end of DST, the repeated time (`[10-27 02:00:00,10-27 03:00:00)` excluding `10-27 03:00:00`) appears twice, and using that time range for queries in TDengine is also undefined behavior. + - Querying the data between `[2024-10-27 02:00:00, 2024-10-27 03:00:00]` includes the repeated timestamps and the data at `2024-10-27 03:00:00`: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts BETWEEN '2024-10-27 02:00:00' AND '2024-10-27 03:00:00'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + 1729994400000 | 2024-10-27T02:00:00.000Z | 2024-10-27 03:00:00 | + Query OK, 5 row(s) in set (0.001370s) + ``` + + - However, the following query for the range [2024-10-27 02:00:00.000,2024-10-27 02:57:34.999] can only find the data at the first 2024-10-27 02:00:00 time point: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:00.999'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 1 row(s) in set (0.004480s) + ``` + + - The following query for the range [2024-10-27 02:00:01,2024-10-27 02:57:35] can find 3 rows of data (including one row of local time data at 02:59:59): + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:35';; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + =============================================================================================== + 2024-10-27 02:00:00.000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 2024-10-27 02:59:59.000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 2024-10-27 02:00:00.000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 3 row(s) in set (0.004428s) + ``` + +## Summary and Suggestions + +### Summary + +This explanation only addresses the impact of using local time. Using UNIX timestamps or RFC3339 has no impact. + +- Writing: + - It is not possible to write data for nonexistent times during the DST transition. + - Writing data for repeated times during the DST transition is undefined behavior. +- Querying: + - Querying with conditions that specify the skipped time during the start of DST results in undefined behavior. + - Querying with conditions that specify the repeated time during the end of DST results in undefined behavior. +- Display: + - Displaying with time zones is not affected. + - Displaying local time is accurate, but repeated times during the end of DST cannot be distinguished. + - Users should be cautious when using time without time zones for display and application. + +### Suggestions + +To avoid unnecessary impacts of DST on querying and writing in TDengine, it is recommended to use explicit time offsets for writing and querying. + +- Use UNIX Timestamps: Using UNIX timestamps can avoid time zone issues. + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1711846799000 | 2024-03-31T00:59:59.000Z | 2024-03-31T01:59:59.000+01:00 | 2024-03-31 01:59:59 | + | 1711846800000 | 2024-03-31T01:00:00.000Z | 2024-03-31T03:00:00.000+02:00 | 2024-03-31 03:00:00 | + + ```sql + taos> insert into t1 values(1711846799000, 1)(1711846800000, 2); + Insert OK, 2 row(s) affected (0.001434s) + + taos> select * from t1 where ts between 1711846799000 and 1711846800000; + ts | v1 | + =============================== + 1711846799000 | 1 | + 1711846800000 | 2 | + Query OK, 2 row(s) in set (0.003503s) + ``` + +- Use RFC3339 Time Format: The RFC3339 time format with time zone offsets can effectively avoid the uncertainty of DST. + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27T02:00:00.000+02:00 | 2024-10-27 02:00:00 | + | 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27T02:59:59.000+02:00 | 2024-10-27 02:59:59 | + | 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27T02:00:00.000+01:00 | 2024-10-27 02:00:00 | + | 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27T02:59:59.000+01:00 | 2024-10-27 02:59:59 | + + ```sql + taos> insert into t1 values ('2024-10-27T02:00:00.000+02:00', 1) + ('2024-10-27T02:59:59.000+02:00', 2) + ('2024-10-27T02:00:00.000+01:00', 3) + ('2024-10-27T02:59:59.000+01:00', 4); + Insert OK, 4 row(s) affected (0.001514s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+01:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 3 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 4 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 4 row(s) in set (0.004275s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+02:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 2 row(s) in set (0.004275s) + ``` + +- Pay Attention to Time Zone Settings When Querying: When querying and displaying, if local time is needed, be sure to consider the impact of DST. + - taosAdapter: When using the REST API, it supports setting the IANA time zone, and the result is returned in RFC3339 format. + + ```shell + $ curl -uroot:taosdata 'localhost:6041/rest/sql?tz=Europe/Berlin'\ + -d "select ts from tz1.t1" + {"code":0,"column_meta":[["ts","TIMESTAMP",8]],"data":[["1970-01-01T00:59:59.000+01:00"],["2024-03-31T01:00:00.000+01:00"],["2024-03-31T01:59:59.000+01:00"],["2024-03-31T03:00:00.000+02:00"],["2024-03-31T03:00:01.000+02:00"],["2024-10-27T02:00:00.000+02:00"],["2024-10-27T02:59:59.000+02:00"],["2024-10-27T02:00:00.000+01:00"],["2024-10-27T02:59:59.000+01:00"],["2024-10-27T03:00:00.000+01:00"]],"rows":10} + ``` + + - Explorer: When using the Explorer page for SQL queries, users can configure the client time zone to display in RFC3339 format. + + ![Explorer DST](./02-dst/explorer-with-tz.png) + +## Reference Documents + +- IANA Time Zone Database: [https://www.iana.org/time-zones](https://www.iana.org/time-zones) +- RFC3339: [https://datatracker.ietf.org/doc/html/rfc3339](https://datatracker.ietf.org/doc/html/rfc3339) diff --git a/docs/en/27-train-faq/02-dst/dst-berlin.png b/docs/en/27-train-faq/02-dst/dst-berlin.png new file mode 100644 index 0000000000..fa1125de25 Binary files /dev/null and b/docs/en/27-train-faq/02-dst/dst-berlin.png differ diff --git a/docs/en/27-train-faq/02-dst/dst-table.png b/docs/en/27-train-faq/02-dst/dst-table.png new file mode 100644 index 0000000000..f5eddbe010 Binary files /dev/null and b/docs/en/27-train-faq/02-dst/dst-table.png differ diff --git a/docs/en/27-train-faq/02-dst/explorer-with-tz.png b/docs/en/27-train-faq/02-dst/explorer-with-tz.png new file mode 100644 index 0000000000..95758f117a Binary files /dev/null and b/docs/en/27-train-faq/02-dst/explorer-with-tz.png differ diff --git a/docs/en/assets/architecture-01.png b/docs/en/assets/architecture-01.png index a9e2bf9f70..71f6b06f5a 100644 Binary files a/docs/en/assets/architecture-01.png and b/docs/en/assets/architecture-01.png differ diff --git a/docs/en/assets/architecture-02.png b/docs/en/assets/architecture-02.png index e558a830ae..92580995c1 100644 Binary files a/docs/en/assets/architecture-02.png and b/docs/en/assets/architecture-02.png differ diff --git a/docs/en/assets/architecture-03.png b/docs/en/assets/architecture-03.png index 467580248c..60d786a782 100644 Binary files a/docs/en/assets/architecture-03.png and b/docs/en/assets/architecture-03.png differ diff --git a/docs/en/assets/architecture-04.png b/docs/en/assets/architecture-04.png index e034f97f89..06ecf72a26 100644 Binary files a/docs/en/assets/architecture-04.png and b/docs/en/assets/architecture-04.png differ diff --git a/docs/en/assets/data-compression-01.png b/docs/en/assets/data-compression-01.png index 64a36f4a42..31abb35926 100644 Binary files a/docs/en/assets/data-compression-01.png and b/docs/en/assets/data-compression-01.png differ diff --git a/docs/en/assets/data-subscription-engine-01.png b/docs/en/assets/data-subscription-engine-01.png index d9306cdb74..088d0070be 100644 Binary files a/docs/en/assets/data-subscription-engine-01.png and b/docs/en/assets/data-subscription-engine-01.png differ diff --git a/docs/en/assets/data-subscription-engine-02.png b/docs/en/assets/data-subscription-engine-02.png index 0c9a061107..0bbd9ff0af 100644 Binary files a/docs/en/assets/data-subscription-engine-02.png and b/docs/en/assets/data-subscription-engine-02.png differ diff --git a/docs/en/assets/data-subscription-engine-03.png b/docs/en/assets/data-subscription-engine-03.png index 0ddb005d66..df61f32208 100644 Binary files a/docs/en/assets/data-subscription-engine-03.png and b/docs/en/assets/data-subscription-engine-03.png differ diff --git a/docs/en/assets/data-subscription-engine-04.png b/docs/en/assets/data-subscription-engine-04.png index 1bb4bb3014..921d6d7de6 100644 Binary files a/docs/en/assets/data-subscription-engine-04.png and b/docs/en/assets/data-subscription-engine-04.png differ diff --git a/docs/en/assets/data-subscription-engine-05.png b/docs/en/assets/data-subscription-engine-05.png index fbc2147c91..5e7867170c 100644 Binary files a/docs/en/assets/data-subscription-engine-05.png and b/docs/en/assets/data-subscription-engine-05.png differ diff --git a/docs/en/assets/data-subscription-engine-06.png b/docs/en/assets/data-subscription-engine-06.png index dd2b472890..256c6abc9c 100644 Binary files a/docs/en/assets/data-subscription-engine-06.png and b/docs/en/assets/data-subscription-engine-06.png differ diff --git a/docs/en/assets/data-subscription-engine-07.png b/docs/en/assets/data-subscription-engine-07.png index 6cd1670b64..978fa022ee 100644 Binary files a/docs/en/assets/data-subscription-engine-07.png and b/docs/en/assets/data-subscription-engine-07.png differ diff --git a/docs/en/assets/mqtt-05.png b/docs/en/assets/mqtt-05.png index 5345b3923f..5644330e14 100644 Binary files a/docs/en/assets/mqtt-05.png and b/docs/en/assets/mqtt-05.png differ diff --git a/docs/en/assets/query-engine-01.png b/docs/en/assets/query-engine-01.png index 08d42dc038..da850c3116 100644 Binary files a/docs/en/assets/query-engine-01.png and b/docs/en/assets/query-engine-01.png differ diff --git a/docs/en/assets/query-engine-02.png b/docs/en/assets/query-engine-02.png index ae7ab92c0a..dcb5360f62 100644 Binary files a/docs/en/assets/query-engine-02.png and b/docs/en/assets/query-engine-02.png differ diff --git a/docs/en/assets/storage-engine-01.png b/docs/en/assets/storage-engine-01.png index 40cba22780..f5cc5867bc 100644 Binary files a/docs/en/assets/storage-engine-01.png and b/docs/en/assets/storage-engine-01.png differ diff --git a/docs/en/assets/storage-engine-02.png b/docs/en/assets/storage-engine-02.png index c6ff24bccb..f201b4dbc4 100644 Binary files a/docs/en/assets/storage-engine-02.png and b/docs/en/assets/storage-engine-02.png differ diff --git a/docs/en/assets/storage-engine-03.png b/docs/en/assets/storage-engine-03.png index fdc252a4ab..d083906367 100644 Binary files a/docs/en/assets/storage-engine-03.png and b/docs/en/assets/storage-engine-03.png differ diff --git a/docs/en/assets/storage-engine-04.png b/docs/en/assets/storage-engine-04.png index 606bd4eaeb..d265fcd6a7 100644 Binary files a/docs/en/assets/storage-engine-04.png and b/docs/en/assets/storage-engine-04.png differ diff --git a/docs/en/assets/storage-engine-05.png b/docs/en/assets/storage-engine-05.png index 4211ac2343..36febdfc85 100644 Binary files a/docs/en/assets/storage-engine-05.png and b/docs/en/assets/storage-engine-05.png differ diff --git a/docs/en/assets/storage-engine-06.png b/docs/en/assets/storage-engine-06.png index d310929b31..4f8eaa1547 100644 Binary files a/docs/en/assets/storage-engine-06.png and b/docs/en/assets/storage-engine-06.png differ diff --git a/docs/en/assets/storage-engine-07.png b/docs/en/assets/storage-engine-07.png index 7c82e017e9..bfbf0c03d3 100644 Binary files a/docs/en/assets/storage-engine-07.png and b/docs/en/assets/storage-engine-07.png differ diff --git a/docs/en/assets/storage-engine-08.png b/docs/en/assets/storage-engine-08.png index ecd1262e94..a2c4201fe7 100644 Binary files a/docs/en/assets/storage-engine-08.png and b/docs/en/assets/storage-engine-08.png differ diff --git a/docs/en/assets/storage-engine-09.png b/docs/en/assets/storage-engine-09.png index d07acee6ae..304926b04e 100644 Binary files a/docs/en/assets/storage-engine-09.png and b/docs/en/assets/storage-engine-09.png differ diff --git a/docs/en/assets/storage-engine-10.png b/docs/en/assets/storage-engine-10.png index 475dbae2fb..deba103a86 100644 Binary files a/docs/en/assets/storage-engine-10.png and b/docs/en/assets/storage-engine-10.png differ diff --git a/docs/en/assets/storage-engine-11.png b/docs/en/assets/storage-engine-11.png index 4d942ad728..43a3d625c8 100644 Binary files a/docs/en/assets/storage-engine-11.png and b/docs/en/assets/storage-engine-11.png differ diff --git a/docs/en/assets/stream-processing-engine-01.png b/docs/en/assets/stream-processing-engine-01.png index c8a8087273..407eb3c542 100644 Binary files a/docs/en/assets/stream-processing-engine-01.png and b/docs/en/assets/stream-processing-engine-01.png differ diff --git a/docs/en/assets/stream-processing-engine-02.png b/docs/en/assets/stream-processing-engine-02.png index 16c3be56b9..04ecf582de 100644 Binary files a/docs/en/assets/stream-processing-engine-02.png and b/docs/en/assets/stream-processing-engine-02.png differ diff --git a/docs/en/assets/stream-processing-engine-03.png b/docs/en/assets/stream-processing-engine-03.png index d9fae4908d..138f4eda39 100644 Binary files a/docs/en/assets/stream-processing-engine-03.png and b/docs/en/assets/stream-processing-engine-03.png differ diff --git a/docs/zh/05-basic/03-query.md b/docs/zh/05-basic/03-query.md index 0b2f290667..3f5f25af34 100644 --- a/docs/zh/05-basic/03-query.md +++ b/docs/zh/05-basic/03-query.md @@ -4,6 +4,10 @@ title: TDengine 数据查询 toc_max_heading_level: 4 --- +import win from './window.png'; +import swin from './session-window.png'; +import ewin from './event-window.png'; + 相较于其他众多时序数据库和实时数据库,TDengine 的一个独特优势在于,自其首个版本发布之初便支持标准的 SQL 查询功能。这一特性极大地降低了用户在使用过程中的学习难度。本章将以智能电表的数据模型为例介绍如何在 TDengine 中运用 SQL 查询来处理时序数据。如果需要进一步了解 SQL 语法的细节和功能,建议参阅 TDengine 的官方文档。通过本章的学习,你将能够熟练掌握 TDengine 的 SQL 查询技巧,进而高效地对时序数据进行操作和分析。 ## 基本查询 @@ -145,7 +149,7 @@ Query OK, 10 row(s) in set (2.415961s) 窗口划分逻辑如下图所示: -![常用窗口划分逻辑](./window.png) +常用窗口划分逻辑 窗口子句语法如下: @@ -408,7 +412,8 @@ Query OK, 22 row(s) in set (0.153403s) 会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示,如果设置时间戳的连续的间隔小于等于 12 秒,则以下 6 条记录构成 2 个会话窗口,分别是:[2019-04-28 14:22:10,2019-04-28 14:22:30] 和 [2019-04-28 14:23:10,2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒,超过了连续时间间隔(12 秒)。 -![会话窗口示意图](./session-window.png) +会话窗口示意图 + 在 tol_value 时间间隔范围内的结果都认为归属于同一个窗口,如果连续的两条记录的时间超过 tol_val,则自动开启下一个窗口。 @@ -461,7 +466,7 @@ Query OK, 10 row(s) in set (0.043489s) select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c2 < 10 ``` -![事件窗口示意图](./event-window.png) +事件窗口示意图 示例 SQL: diff --git a/docs/zh/05-basic/event-window.png b/docs/zh/05-basic/event-window.png index 3a2a628ffe..a249dac3c3 100644 Binary files a/docs/zh/05-basic/event-window.png and b/docs/zh/05-basic/event-window.png differ diff --git a/docs/zh/05-basic/session-window.png b/docs/zh/05-basic/session-window.png index 416a14a0a9..b1f4024322 100644 Binary files a/docs/zh/05-basic/session-window.png and b/docs/zh/05-basic/session-window.png differ diff --git a/docs/zh/05-basic/window.png b/docs/zh/05-basic/window.png index 5e3efb8a3b..a6f5883917 100644 Binary files a/docs/zh/05-basic/window.png and b/docs/zh/05-basic/window.png differ diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 2216cab915..1e6667b31f 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -147,7 +147,7 @@ taosd 命令行参数如下 - 类型:整数 - 单位:毫秒 - 默认值:10000 -- 最小值:0 +- 最小值:3000 - 最大值:86400000 - 动态修改:支持通过 SQL 修改,重启后生效 - 支持版本:v3.3.4.0 引入 diff --git a/docs/zh/27-train-faq/02-dst.md b/docs/zh/27-train-faq/02-dst.md new file mode 100644 index 0000000000..5c430fd42f --- /dev/null +++ b/docs/zh/27-train-faq/02-dst.md @@ -0,0 +1,291 @@ +--- +title: 夏令时使用指南 +description: TDengine 中关于夏令时使用问题的解释和建议 +--- + +## 背景 + +在时序数据库的使用中,有时会遇到使用夏令时的情况。我们将 TDengine 中使用夏令时的情况和问题进行分析说明,以便您在 TDengine 的使用中更加顺利。 + +## 定义 + +### 时区 + +时区是地球上使用相同标准时间的区域。由于地球的自转,为了保证各地的时间与当地的日出日落相协调,全球划分为多个时区。 + +### IANA 时区 + +IANA(Internet Assigned Numbers Authority)时区数据库,也称为 tz database,提供全球时区信息的标准参考。它是现代各类系统和软件处理时区相关操作的基础。 + +IANA 使用“区域/城市”格式(如 Europe/Berlin)来明确标识时区。 + +TDengine 在不同组件中均支持使用 IANA 时区(除 Windows taos.cfg 时区设置外)。 + +### 标准时间与当地时间 + +标准时间是根据地球上某个固定经线确定的时间。它为各个时区提供了一个统一的参考点。 + +- 格林尼治标准时间(GMT):历史上使用的参考时间,位于 0° 经线。 +- 协调世界时(UTC):现代的时间标准,类似于GMT,但更加精确。 + +标准时间与时区的关系如下: + +- 基准:标准时间(如 UTC)是时区设定的基准点。 +- 偏移量:不同时区通过相对于标准时间的偏移量来定义。例如,UTC+1 表示比 UTC 快 1 小时。 +- 区域划分:全球被划分为多个时区,每个时区使用一个或多个标准时间。 + +相对于标准时间,每个地区根据其所在时区设定其当地时间: + +- 时区偏移:当地时间等于标准时间加上该时区的偏移量。例如,UTC+2 表示比 UTC 时间快 2 小时。 +- 夏令时(DST):某些地区在特定时间段调整当地时间,例如将时钟拨快一小时。详见下节。 + +### 夏令时 + +夏令时(Daylight Saving Time,DST)是一种通过将时间提前一小时,以充分利用日光、节约能源的制度。通常在春季开始,秋季结束。夏令时的具体开始和结束时间因地区而异。以下均以柏林时间为例,对夏令时和夏令时的影响做说明。 + +按照这个规则,可以看到: + +- 柏林当地时间 2024 年 03 月 31 日 02:00:00 到 03:00:00 (不含 03:00:00)之间的时间不存在(跳变)。 +- 柏林当地时间 2024 年 10 月 27 日 02:00:00 到 03:00:00 (不含 03:00:00)之间的时间出现了两次。 + +#### 夏令时与 IANA 时区数据库 + +- 记录规则:IANA 时区数据库详细记录了各地的夏令时规则,包括开始和结束的日期与时间。 +- 自动调整:许多操作系统和软件利用 IANA 数据库来自动处理夏令时的调整。 +- 历史变更:IANA 数据库还追踪历史上的夏令时变化,以确保准确性。 + +#### 夏令时与时间戳转换 + +- 时间戳转为当地时间是确定的。例如,1729990654 为柏林时间**夏令时** `2024-10-27 02:57:34`,1729994254 为柏林时间**冬令时** `2024-10-27 02:57:34`(这两个本地时间除时间偏移量外是一样的)。 +- 不指定时间偏移量时,当地时间转为时间戳是不确定的。夏令时跳过的时间不存在会造成无法转换成时间戳,如 **柏林时间** `2024-03-31 02:34:56` 不存在,所以无法转换为时间戳。夏令时结束时重复导致无法确定是哪个时间戳,如 `2024-10-27 02:57:34` 不指定时间偏移量无法确定 是 1729990654 还是 1729994254。指定时间偏移量才能确定时间戳,如 `2024-10-27 02:57:34 CEST(+02:00) `,指定了夏令时 `2024-10-27 02:57:34` 时间戳 1729990654 。 + +### RFC3339 时间格式 + +RFC 3339 是一种互联网时间格式标准,用于表示日期和时间。它基于 ISO 8601 标准,但更具体地规定了一些格式细节。 + +其格式如下: + +- 基本格式:`YYYY-MM-DDTHH:MM:SSZ` +- 时区表示: + - Z 表示协调世界时(UTC)。 + - 偏移量格式,例如 +02:00,表示与 UTC 的时差。 + +通过明确的时区偏移,RFC 3339 格式可以在全球范围内准确地解析和比较时间。 + +RFC 3339 的优势包括: + +- 标准化:提供统一的格式,方便跨系统数据交换。 +- 清晰性:明确时区信息,避免时间误解。 + +TDengine 在 REST API 和 Explorer UI 中,均使用 RFC3339 格式进行展示。在 SQL 语句中,可使用 RFC3339 格式写入时间戳数据: + +```sql +insert into t1 values('2024-10-27T01:59:59.000Z', 0); +select * from t1 where ts >= '2024-10-27T01:59:59.000Z'; +``` + +### 未定义行为 + +未定义行为(Undefined Behavior)是指特定代码或操作没有明确规定的结果,也不会对该结果作出兼容性的保证,TDengine 可能在某个版本后对当前的行为作出修改而不会通知用户。所以,在 TDengine 中,用户不可依赖当前未定义的行为进行判断或应用。 + +## 夏令时在 TDengine 中的写入与查询 + +我们使用下表来展示夏令时在写入和查询中的影响。 + +![DST Berlin](./02-dst/dst-berlin.png) + +### 表格说明 + +- **TIMESTAMP**:TDengine 中使用 64位整数来存储原始时间戳。 +- **UTC**:时间戳对应的 UTC 时间表示。 +- **Europe/Berlin**:表示时区 Europe/Berlin 对应的 RFC3339 格式时间。 +- **Local**:表示时区 Europe/Berlin 对应的当地时间(不含时区)。 + +### 表格分析 + +- 在**夏令时开始**(柏林时间 3 月 31 日 02:00)时,时间直接从 02:00 跳到 03:00(往后跳一小时)。 + - 浅绿色是夏令时开始前一小时的时间戳; + - 深绿色是夏令时开始后一小时的时间戳; + - 红色为 TDengine 数据库中插入了不存在的当地时间: + - 使用 SQL `INSERT INTO t1 values('2024-03-31 02:59:59',..)` 插入 `2024-03-31 02:00:00` 到 `2024-03-31 02:59:59` 的数据会被自动调整为 -1000(在 TDengine 中属于未定义行为,当前该值与数据库精度 precision 有关,毫秒数据库为 -1000,微秒数据库为 -1000000,纳秒数据库为 -1000000000),因为那一时刻在本地时间中不存在; +- 在**夏令时结束**(柏林时间 10 月 27 日 03:00)时,时间从 03:00 跳到 02:00 (往前跳一小时)。 + - 浅蓝色表示时钟跳变前一小时的时间戳; + - 深蓝色表示时钟跳变后一小时内的时间戳,其无时区的当地时间与上一小时一致。 + - 紫色表示时钟跳变一小时后的时间戳; +- **当地时间变化**:可见,由于夏令时的调整而导致了当地时间的变化,可能导致某些时间段出现重复或缺失。 +- **UTC 时间不变**:UTC 时间保持不变,确保了时间的一致性和顺序性。 +- **RFC3339**:RFC3339 格式时间显示了时间偏移量的变化,在夏令时开始后变为 +02:00,结束后变为 +01:00 。 +- **条件查询**: + - **夏令时开始**时,跳过的时间(`[03-31 02:00:00,03-31 03:00:00)`)不存在,所以在使用该时间进行查询时,行为不确定:`SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'`(不存在的本地时间戳被转换为 `-1000`): + + ```sql + taos> SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'; + ts | + ================= + -1000 | + Query OK, 1 row(s) in set (0.003635s) + ``` + + 当不存在的时间戳与存在的时间戳共同使用时,其结果同样不符合预期,以下为起始本地时间不存在: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 03:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + -1000 | 1969-12-31T23:59:59.000Z | + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + 1711846800000 | 2024-03-31T01:00:00.000Z | + 1711846801000 | 2024-03-31T01:00:01.000Z | + Query OK, 5 row(s) in set (0.003339s) + ``` + + 以下语句中第一个 SQL 查询截止时间不存在,第二个截止时间存在,第一个 SQL 查询结果不符合预期: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 02:00:00'; + Query OK, 0 row(s) in set (0.000930s) + + taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 01:59:59'; + ts | to_iso8601(ts,'Z') | + ================================================== + 1711843200000 | 2024-03-31T00:00:00.000Z | + 1711846799000 | 2024-03-31T00:59:59.000Z | + Query OK, 2 row(s) in set (0.001227s) + ``` + + - 夏令时结束时,跳变的时间(`[10-27 02:00:00,10-27 03:00:00)` 不包含 `10-27 03:00:00`)重复了两次,TDengine 在使用该区间内的时间戳进行查询时,也属于未定义行为。 + - 查询 `[2024-10-27 02:00:00, 2024-10-27 03:00:00]` 之间的数据结果,包含了两次重复的时间戳和 `2024-10-27 03:00:00` 这个时间点的数据: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts BETWEEN '2024-10-27 02:00:00' AND '2024-10-27 03:00:00'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + 1729994400000 | 2024-10-27T02:00:00.000Z | 2024-10-27 03:00:00 | + Query OK, 5 row(s) in set (0.001370s) + ```` + + - 但以下查询 [2024-10-27 02:00:00.000,2024-10-27 02:57:34.999] 区间只能查询到第一个2024-10-27 02:00:00 时间点的数据: + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:00.999'; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ======================================================================================= + 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 1 row(s) in set (0.004480s) + ``` + + - 以下查询 `[2024-10-27 02:00:01,2024-10-27 02:57:35]` 却能查到 3 条数据(包含一条 02:59:59 的当地时间数据): + + ```sql + taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:35';; + ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ================================================================================================ + 2024-10-27 02:00:00.000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 2024-10-27 02:59:59.000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 2024-10-27 02:00:00.000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + Query OK, 3 row(s) in set (0.004428s) + ``` + +## 总结与建议 + +### 总结 + +仅针对使用当地时间带来的影响作说明,使用 UNIX 时间戳或 RFC3339 无影响。 + +- 写入: + - 无法写入夏令时跳变时不存在的时间数据。 + - 写入夏令时跳变时重复的时间是未定义行为。 +- 查询: + - 查询条件指定夏令时开始时跳变的时间,其查询结果为未定义行为。 + - 查询条件指定夏令时结束时重复的时间,其查询结果为未定义行为。 +- 显示: + - 带时区显示不受影响。 + - 显示当地时间是准确的,但夏令时结束时重复的时间会无法区分。 + - 用户应谨慎使用不带时区的时间进行展示和应用。 + +### 建议 + +为避免夏令时给查询和写入造成不必要的影响,在 TDengine 中,建议使用明确的时间偏移量进行写入和查询。 + +- 使用 UNIX 时间戳:使用 UNIX 时间戳可避免时区问题。 + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1711846799000 | 2024-03-31T00:59:59.000Z | 2024-03-31T01:59:59.000+01:00 | 2024-03-31 01:59:59 | + | 1711846800000 | 2024-03-31T01:00:00.000Z | 2024-03-31T03:00:00.000+02:00 | 2024-03-31 03:00:00 | + + ```sql + taos> insert into t1 values(1711846799000, 1)(1711846800000, 2); + Insert OK, 2 row(s) affected (0.001434s) + + taos> select * from t1 where ts between 1711846799000 and 1711846800000; + ts | v1 | + =============================== + 1711846799000 | 1 | + 1711846800000 | 2 | + Query OK, 2 row(s) in set (0.003503s) + ``` + +- 使用 RFC3339 时间格式:带时区偏移量的 RFC3339 时间格式可以有效避免夏令时的不确定性。 + + | TIMESTAMP | UTC | Europe/Berlin | Local | + | ------------: | :----------------------: | :---------------------------: | :-----------------: | + | 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27T02:00:00.000+02:00 | 2024-10-27 02:00:00 | + | 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27T02:59:59.000+02:00 | 2024-10-27 02:59:59 | + | 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27T02:00:00.000+01:00 | 2024-10-27 02:00:00 | + | 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27T02:59:59.000+01:00 | 2024-10-27 02:59:59 | + + ```sql + taos> insert into t1 values ('2024-10-27T02:00:00.000+02:00', 1) + ('2024-10-27T02:59:59.000+02:00', 2) + ('2024-10-27T02:00:00.000+01:00', 3) + ('2024-10-27T02:59:59.000+01:00', 4); + Insert OK, 4 row(s) affected (0.001514s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+01:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + 1729990800000 | 3 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 | + 1729994399000 | 4 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 4 row(s) in set (0.004275s) + + taos> SELECT *, + to_iso8601(ts,'Z'), + to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 + WHERE ts >= '2024-10-27T02:00:00.000+02:00' + AND ts <= '2024-10-27T02:59:59.000+02:00'; + ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') | + ===================================================================================================== + 1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 | + 1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 | + Query OK, 2 row(s) in set (0.004275s) + ``` + +- 查询时注意时区设定:在查询和显示时,如果需要本地时间,务必考虑夏令时的影响。 + - taosAdapter:使用 REST API 时,支持设置 IANA 时区,结果使用 RFC3339 格式返回。 + + ```shell + $ curl -uroot:taosdata 'localhost:6041/rest/sql?tz=Europe/Berlin'\ + -d "select ts from tz1.t1" + {"code":0,"column_meta":[["ts","TIMESTAMP",8]],"data":[["1970-01-01T00:59:59.000+01:00"],["2024-03-31T01:00:00.000+01:00"],["2024-03-31T01:59:59.000+01:00"],["2024-03-31T03:00:00.000+02:00"],["2024-03-31T03:00:01.000+02:00"],["2024-10-27T02:00:00.000+02:00"],["2024-10-27T02:59:59.000+02:00"],["2024-10-27T02:00:00.000+01:00"],["2024-10-27T02:59:59.000+01:00"],["2024-10-27T03:00:00.000+01:00"]],"rows":10} + ``` + + - Explorer:使用 Explorer 页面进行 SQL 查询时,用户可配置客户端时区,以 RFC3339 格式显示。 + + ![Explorer DST](./02-dst/explorer-with-tz.png) + +## 参考文档 + +- IANA Time Zone Database: [https://www.iana.org/time-zones](https://www.iana.org/time-zones) +- RFC3339: [https://datatracker.ietf.org/doc/html/rfc3339](https://datatracker.ietf.org/doc/html/rfc3339) diff --git a/docs/zh/27-train-faq/02-dst/dst-berlin.png b/docs/zh/27-train-faq/02-dst/dst-berlin.png new file mode 100644 index 0000000000..8c64626dd0 Binary files /dev/null and b/docs/zh/27-train-faq/02-dst/dst-berlin.png differ diff --git a/docs/zh/27-train-faq/02-dst/dst-table.png b/docs/zh/27-train-faq/02-dst/dst-table.png new file mode 100644 index 0000000000..f5eddbe010 Binary files /dev/null and b/docs/zh/27-train-faq/02-dst/dst-table.png differ diff --git a/docs/zh/27-train-faq/02-dst/explorer-with-tz.png b/docs/zh/27-train-faq/02-dst/explorer-with-tz.png new file mode 100644 index 0000000000..95758f117a Binary files /dev/null and b/docs/zh/27-train-faq/02-dst/explorer-with-tz.png differ diff --git a/include/common/tanalytics.h b/include/common/tanalytics.h index 42c3ce9391..0fb1d543f7 100644 --- a/include/common/tanalytics.h +++ b/include/common/tanalytics.h @@ -25,12 +25,13 @@ extern "C" { #endif -#define ANALY_FORECAST_DEFAULT_ROWS 10 -#define ANALY_FORECAST_DEFAULT_CONF 95 -#define ANALY_FORECAST_DEFAULT_WNCHECK 1 -#define ANALY_FORECAST_MAX_HISTORY_ROWS 40000 -#define ANALY_MAX_FC_ROWS 1024 -#define ANALY_ANOMALY_WINDOW_MAX_ROWS 40000 +#define ANALY_FORECAST_DEFAULT_ROWS 10 +#define ANALY_FORECAST_DEFAULT_CONF 95 +#define ANALY_FORECAST_DEFAULT_WNCHECK 1 +#define ANALY_FORECAST_MAX_ROWS 40000 +#define ANALY_ANOMALY_WINDOW_MAX_ROWS 40000 +#define ANALY_DEFAULT_TIMEOUT 60 +#define ANALY_MAX_TIMEOUT 600 typedef struct { EAnalAlgoType type; @@ -48,7 +49,7 @@ typedef enum { typedef enum { ANALYTICS_HTTP_TYPE_GET = 0, ANALYTICS_HTTP_TYPE_POST, -} EAnalHttpType; +} EAnalyHttpType; typedef struct { TdFilePtr filePtr; @@ -66,7 +67,7 @@ typedef struct { int32_t taosAnalyticsInit(); void taosAnalyticsCleanup(); -SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf *pBuf); +SJson *taosAnalySendReqRetJson(const char *url, EAnalyHttpType type, SAnalyticBuf *pBuf, int64_t timeout); int32_t taosAnalyGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen); bool taosAnalyGetOptStr(const char *option, const char *optName, char *optValue, int32_t optMaxLen); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 266bd4a618..986874c5bb 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -27,6 +27,7 @@ #include "scheduler.h" #include "tcache.h" #include "tcompare.h" +#include "tconv.h" #include "tglobal.h" #include "thttp.h" #include "tmsg.h" @@ -36,7 +37,6 @@ #include "tsched.h" #include "ttime.h" #include "tversion.h" -#include "tconv.h" #include "cus_name.h" @@ -63,13 +63,13 @@ } \ } while (0) -STscDbg tscDbg = {0}; -SAppInfo appInfo; -int64_t lastClusterId = 0; -int32_t clientReqRefPool = -1; -int32_t clientConnRefPool = -1; -int32_t clientStop = -1; -SHashObj* pTimezoneMap = NULL; +STscDbg tscDbg = {0}; +SAppInfo appInfo; +int64_t lastClusterId = 0; +int32_t clientReqRefPool = -1; +int32_t clientConnRefPool = -1; +int32_t clientStop = -1; +SHashObj *pTimezoneMap = NULL; int32_t timestampDeltaLimit = 900; // s @@ -964,7 +964,7 @@ void taos_init_imp(void) { ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg"); initQueryModuleMsgHandle(); - if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL){ + if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) { tscInitRes = terrno; tscError("failed to init conv"); return; diff --git a/source/common/src/tanalytics.c b/source/common/src/tanalytics.c index deb68af3ea..397accc0b1 100644 --- a/source/common/src/tanalytics.c +++ b/source/common/src/tanalytics.c @@ -276,7 +276,7 @@ _OVER: return code; } -static int32_t taosCurlPostRequest(const char *url, SCurlResp *pRsp, const char *buf, int32_t bufLen) { +static int32_t taosCurlPostRequest(const char *url, SCurlResp *pRsp, const char *buf, int32_t bufLen, int32_t timeout) { struct curl_slist *headers = NULL; CURL *curl = NULL; CURLcode code = 0; @@ -292,7 +292,7 @@ static int32_t taosCurlPostRequest(const char *url, SCurlResp *pRsp, const char if (curl_easy_setopt(curl, CURLOPT_URL, url) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, taosCurlWriteData) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, pRsp) != 0) goto _OVER; - if (curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 60000) != 0) goto _OVER; + if (curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_POST, 1) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, bufLen) != 0) goto _OVER; if (curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buf) != 0) goto _OVER; @@ -311,7 +311,7 @@ _OVER: return code; } -SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf *pBuf) { +SJson *taosAnalySendReqRetJson(const char *url, EAnalyHttpType type, SAnalyticBuf *pBuf, int64_t timeout) { int32_t code = -1; char *pCont = NULL; int64_t contentLen; @@ -329,7 +329,7 @@ SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf terrno = code; goto _OVER; } - if (taosCurlPostRequest(url, &curlRsp, pCont, contentLen) != 0) { + if (taosCurlPostRequest(url, &curlRsp, pCont, contentLen, timeout) != 0) { terrno = TSDB_CODE_ANA_URL_CANT_ACCESS; goto _OVER; } @@ -767,7 +767,7 @@ static int32_t taosAnalyBufGetCont(SAnalyticBuf *pBuf, char **ppCont, int64_t *p int32_t taosAnalyticsInit() { return 0; } void taosAnalyticsCleanup() {} -SJson *taosAnalySendReqRetJson(const char *url, EAnalHttpType type, SAnalyticBuf *pBuf) { return NULL; } +SJson *taosAnalySendReqRetJson(const char *url, EAnalyHttpType type, SAnalyticBuf *pBuf, int64_t timeout) { return NULL; } int32_t taosAnalyGetAlgoUrl(const char *algoName, EAnalAlgoType type, char *url, int32_t urlLen) { return 0; } bool taosAnalyGetOptStr(const char *option, const char *optName, char *optValue, int32_t optMaxLen) { return true; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 00f9504bc9..299e947d35 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "tglobal.h" #include "cJSON.h" #include "defines.h" #include "os.h" #include "osString.h" #include "tconfig.h" -#include "tglobal.h" #include "tgrant.h" #include "tjson.h" #include "tlog.h" @@ -28,7 +28,6 @@ #include "tutil.h" - #define CONFIG_PATH_LEN (TSDB_FILENAME_LEN + 12) #define CONFIG_FILE_LEN (CONFIG_PATH_LEN + 32) @@ -501,9 +500,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { TAOS_RETURN(TSDB_CODE_SUCCESS); } -struct SConfig *taosGetCfg() { - return tsCfg; -} +struct SConfig *taosGetCfg() { return tsCfg; } static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -692,7 +689,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL) != 0); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_SERVER, + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 3000, 86400000, CFG_SCOPE_SERVER, CFG_DYN_BOTH_LAZY, CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN( diff --git a/source/dnode/mnode/impl/src/mndAnode.c b/source/dnode/mnode/impl/src/mndAnode.c index c08d4aead4..bd0a4f3138 100644 --- a/source/dnode/mnode/impl/src/mndAnode.c +++ b/source/dnode/mnode/impl/src/mndAnode.c @@ -789,7 +789,7 @@ static int32_t mndGetAnodeAlgoList(const char *url, SAnodeObj *pObj) { char anodeUrl[TSDB_ANALYTIC_ANODE_URL_LEN + 1] = {0}; snprintf(anodeUrl, TSDB_ANALYTIC_ANODE_URL_LEN, "%s/%s", url, "list"); - SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL); + SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL, 0); if (pJson == NULL) return terrno; int32_t code = mndDecodeAlgoList(pJson, pObj); @@ -805,7 +805,7 @@ static int32_t mndGetAnodeStatus(SAnodeObj *pObj, char *status, int32_t statusLe char anodeUrl[TSDB_ANALYTIC_ANODE_URL_LEN + 1] = {0}; snprintf(anodeUrl, TSDB_ANALYTIC_ANODE_URL_LEN, "%s/%s", pObj->url, "status"); - SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL); + SJson *pJson = taosAnalySendReqRetJson(anodeUrl, ANALYTICS_HTTP_TYPE_GET, NULL, 0); if (pJson == NULL) return terrno; code = tjsonGetDoubleValue(pJson, "protocol", &tmp); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7d65673226..c80edd070d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -928,12 +928,6 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask // now the fill-history task starts to scan data from wal files. code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); -// if (code == TSDB_CODE_SUCCESS) { -// code = tqScanWalAsync(pTq, false); -// if (code) { -// tqError("vgId:%d failed to start scan wal file, code:%s", vgId, tstrerror(code)); -// } -// } } } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 08ca02e20e..b9cbb33746 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -148,6 +148,7 @@ static void doStartScanWal(void* param, void* tmrId) { return; } + // failed to lock, try 500ms later code = streamMetaTryRlock(pMeta); if (code == 0) { numOfTasks = taosArrayGetSize(pMeta->pTaskList); @@ -156,25 +157,23 @@ static void doStartScanWal(void* param, void* tmrId) { numOfTasks = 0; } - if (numOfTasks == 0) { - goto _end; - } + if (numOfTasks > 0) { + tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d", vgId, numOfTasks); - tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d", vgId, numOfTasks); - - #if 0 +#if 0 // wait for the vnode is freed, and invalid read may occur. taosMsleep(10000); - #endif +#endif - code = streamTaskSchedTask(&pParam->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); - if (code) { - tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code)); + code = streamTaskSchedTask(&pParam->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); + if (code) { + tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code)); + } } _end: streamTmrStart(doStartScanWal, SCAN_WAL_IDLE_DURATION, pParam, pTimer, &pMeta->scanInfo.scanTimer, vgId, "scan-wal"); - tqDebug("vgId:%d scan-wal will start in %dms", vgId, SCAN_WAL_IDLE_DURATION*SCAN_WAL_WAIT_COUNT); + tqDebug("vgId:%d try scan-wal will start in %dms", vgId, SCAN_WAL_IDLE_DURATION*SCAN_WAL_WAIT_COUNT); code = taosReleaseRef(streamMetaRefPool, pParam->metaId); if (code) { @@ -192,7 +191,7 @@ void tqScanWalAsync(STQ* pTq) { // 1. the vnode should be the leader. // 2. the stream isn't disabled - if ((pMeta->role == NODE_ROLE_FOLLOWER) || tsDisableStream) { + if ((pMeta->role != NODE_ROLE_LEADER) || tsDisableStream) { tqInfo("vgId:%d follower node or stream disabled, not scan wal", vgId); return; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 1afccd3d01..4295969c85 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -960,11 +960,6 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { streamMetaWUnLock(pMeta); -// if (scanWal && (vgId != SNODE_HANDLE)) { -// tqDebug("vgId:%d start scan wal for executing tasks", vgId); -// code = tqScanWalAsync(pMeta->ahandle, true); -// } - return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 8baf08ef94..ca3b82239b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -610,11 +610,12 @@ static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { code = taosThreadMutexTryLock(&pReader->readerMutex); if (code != TSDB_CODE_SUCCESS) { - tsdbError("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + // Failing to acquire the lock is reasonable, not an error + tsdbWarn("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); } else { tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex); } - TSDB_CHECK_CODE(code, lino, _end); + return code; _end: if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index 46aae38ad4..379177bb06 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -47,6 +47,7 @@ typedef struct { char algoName[TSDB_ANALYTIC_ALGO_NAME_LEN]; char algoUrl[TSDB_ANALYTIC_ALGO_URL_LEN]; char anomalyOpt[TSDB_ANALYTIC_ALGO_OPTION_LEN]; + int64_t timeout; SAnomalyWindowSupp anomalySup; SWindowRowsSup anomalyWinRowSup; SColumn anomalyCol; @@ -89,6 +90,20 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p goto _error; } + bool hasTimeout = taosAnalyGetOptInt(pAnomalyNode->anomalyOpt, "timeout", &pInfo->timeout); + if (!hasTimeout) { + qDebug("not set the timeout val, set default:%d", ANALY_DEFAULT_TIMEOUT); + pInfo->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + if (pInfo->timeout <= 0 || pInfo->timeout > ANALY_MAX_TIMEOUT) { + qDebug("timeout val:%" PRId64 "s is invalid (greater than 10min or less than 1s), use default:%dms", + pInfo->timeout, ANALY_DEFAULT_TIMEOUT); + pInfo->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + qDebug("timeout val is set to: %" PRId64 "s", pInfo->timeout); + } + } + pOperator->exprSupp.hasWindowOrGroup = true; pInfo->tsSlotId = ((SColumnNode*)pAnomalyNode->window.pTspk)->slotId; tstrncpy(pInfo->anomalyOpt, pAnomalyNode->anomalyOpt, sizeof(pInfo->anomalyOpt)); @@ -451,7 +466,7 @@ static int32_t anomalyAnalysisWindow(SOperatorInfo* pOperator) { code = taosAnalyBufClose(&analyBuf); QUERY_CHECK_CODE(code, lino, _OVER); - pJson = taosAnalySendReqRetJson(pInfo->algoUrl, ANALYTICS_HTTP_TYPE_POST, &analyBuf); + pJson = taosAnalySendReqRetJson(pInfo->algoUrl, ANALYTICS_HTTP_TYPE_POST, &analyBuf, pInfo->timeout * 1000); if (pJson == NULL) { code = terrno; goto _OVER; diff --git a/source/libs/executor/src/forecastoperator.c b/source/libs/executor/src/forecastoperator.c index e318530352..e9185824a3 100644 --- a/source/libs/executor/src/forecastoperator.c +++ b/source/libs/executor/src/forecastoperator.c @@ -38,6 +38,7 @@ typedef struct { int64_t optRows; int64_t cachedRows; int32_t numOfBlocks; + int64_t timeout; int16_t resTsSlot; int16_t resValSlot; int16_t resLowSlot; @@ -76,10 +77,10 @@ static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock, con int32_t lino = 0; SAnalyticBuf* pBuf = &pSupp->analyBuf; - if (pSupp->cachedRows > ANALY_FORECAST_MAX_HISTORY_ROWS) { + if (pSupp->cachedRows > ANALY_FORECAST_MAX_ROWS) { code = TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; qError("%s rows:%" PRId64 " for forecast cache, error happens, code:%s, upper limit:%d", id, pSupp->cachedRows, - tstrerror(code), ANALY_FORECAST_MAX_HISTORY_ROWS); + tstrerror(code), ANALY_FORECAST_MAX_ROWS); return code; } @@ -157,8 +158,8 @@ static int32_t forecastCloseBuf(SForecastSupp* pSupp, const char* id) { qDebug("%s forecast rows not found from %s, use default:%" PRId64, id, pSupp->algoOpt, pSupp->optRows); } - if (pSupp->optRows > ANALY_MAX_FC_ROWS) { - qError("%s required too many forecast rows, max allowed:%d, required:%" PRId64, id, ANALY_MAX_FC_ROWS, + if (pSupp->optRows > ANALY_FORECAST_MAX_ROWS) { + qError("%s required too many forecast rows, max allowed:%d, required:%" PRId64, id, ANALY_FORECAST_MAX_ROWS, pSupp->optRows); return TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; } @@ -198,12 +199,12 @@ static int32_t forecastCloseBuf(SForecastSupp* pSupp, const char* id) { static int32_t forecastAnalysis(SForecastSupp* pSupp, SSDataBlock* pBlock, const char* pId) { SAnalyticBuf* pBuf = &pSupp->analyBuf; int32_t resCurRow = pBlock->info.rows; - int8_t tmpI8; - int16_t tmpI16; - int32_t tmpI32; - int64_t tmpI64; - float tmpFloat; - double tmpDouble; + int8_t tmpI8 = 0; + int16_t tmpI16 = 0; + int32_t tmpI32 = 0; + int64_t tmpI64 = 0; + float tmpFloat = 0; + double tmpDouble = 0; int32_t code = 0; SColumnInfoData* pResValCol = taosArrayGet(pBlock->pDataBlock, pSupp->resValSlot); @@ -211,12 +212,13 @@ static int32_t forecastAnalysis(SForecastSupp* pSupp, SSDataBlock* pBlock, const return terrno; } - SColumnInfoData* pResTsCol = (pSupp->resTsSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resTsSlot) : NULL); - SColumnInfoData* pResLowCol = (pSupp->resLowSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resLowSlot) : NULL); + SColumnInfoData* pResTsCol = ((pSupp->resTsSlot != -1) ? taosArrayGet(pBlock->pDataBlock, pSupp->resTsSlot) : NULL); + SColumnInfoData* pResLowCol = + ((pSupp->resLowSlot != -1) ? taosArrayGet(pBlock->pDataBlock, pSupp->resLowSlot) : NULL); SColumnInfoData* pResHighCol = (pSupp->resHighSlot != -1 ? taosArrayGet(pBlock->pDataBlock, pSupp->resHighSlot) : NULL); - SJson* pJson = taosAnalySendReqRetJson(pSupp->algoUrl, ANALYTICS_HTTP_TYPE_POST, pBuf); + SJson* pJson = taosAnalySendReqRetJson(pSupp->algoUrl, ANALYTICS_HTTP_TYPE_POST, pBuf, pSupp->timeout * 1000); if (pJson == NULL) { return terrno; } @@ -527,18 +529,32 @@ static int32_t forecastParseInput(SForecastSupp* pSupp, SNodeList* pFuncs) { return 0; } -static int32_t forecastParseAlgo(SForecastSupp* pSupp) { +static int32_t forecastParseAlgo(SForecastSupp* pSupp, const char* id) { pSupp->maxTs = 0; pSupp->minTs = INT64_MAX; pSupp->numOfRows = 0; if (!taosAnalyGetOptStr(pSupp->algoOpt, "algo", pSupp->algoName, sizeof(pSupp->algoName))) { - qError("failed to get forecast algorithm name from %s", pSupp->algoOpt); + qError("%s failed to get forecast algorithm name from %s", id, pSupp->algoOpt); return TSDB_CODE_ANA_ALGO_NOT_FOUND; } + bool hasTimeout = taosAnalyGetOptInt(pSupp->algoOpt, "timeout", &pSupp->timeout); + if (!hasTimeout) { + qDebug("%s not set the timeout val, set default:%d", id, ANALY_DEFAULT_TIMEOUT); + pSupp->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + if (pSupp->timeout <= 0 || pSupp->timeout > ANALY_MAX_TIMEOUT) { + qDebug("%s timeout val:%" PRId64 "s is invalid (greater than 10min or less than 1s), use default:%dms", + id, pSupp->timeout, ANALY_DEFAULT_TIMEOUT); + pSupp->timeout = ANALY_DEFAULT_TIMEOUT; + } else { + qDebug("%s timeout val is set to: %" PRId64 "s", id, pSupp->timeout); + } + } + if (taosAnalyGetAlgoUrl(pSupp->algoName, ANALY_ALGO_TYPE_FORECAST, pSupp->algoUrl, sizeof(pSupp->algoUrl)) != 0) { - qError("failed to get forecast algorithm url from %s", pSupp->algoName); + qError("%s failed to get forecast algorithm url from %s", id, pSupp->algoName); return TSDB_CODE_ANA_ALGO_NOT_LOAD; } @@ -589,6 +605,7 @@ int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNo goto _error; } + const char* pId = pTaskInfo->id.str; SForecastSupp* pSupp = &pInfo->forecastSupp; SForecastFuncPhysiNode* pForecastPhyNode = (SForecastFuncPhysiNode*)pPhyNode; SExprSupp* pExprSup = &pOperator->exprSupp; @@ -620,7 +637,7 @@ int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNo code = forecastParseOutput(pSupp, pExprSup); QUERY_CHECK_CODE(code, lino, _error); - code = forecastParseAlgo(pSupp); + code = forecastParseAlgo(pSupp, pId); QUERY_CHECK_CODE(code, lino, _error); code = forecastCreateBuf(pSupp); @@ -644,7 +661,7 @@ int32_t createForecastOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNo *pOptrInfo = pOperator; - qDebug("forecast env is initialized, option:%s", pSupp->algoOpt); + qDebug("%s forecast env is initialized, option:%s", pId, pSupp->algoOpt); return TSDB_CODE_SUCCESS; _error: diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 91be791dff..876b05d55f 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -797,7 +797,7 @@ static bool funcNotSupportStringSma(SFunctionNode* pFunc) { } EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { - if(funcNotSupportStringSma(pFunc)) { + if (funcNotSupportStringSma(pFunc)) { return FUNC_DATA_REQUIRED_DATA_LOAD; } return FUNC_DATA_REQUIRED_SMA_LOAD; @@ -6611,7 +6611,7 @@ int32_t blockDBUsageFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { double compressRadio = 0; if (rawDataSize != 0) { compressRadio = totalDiskSize * 100 / (double)rawDataSize; - len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f]", compressRadio); + len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f%]", compressRadio); } else { len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[NULL]"); } diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 1d1bc66414..257ad3d8ea 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -532,9 +532,17 @@ int32_t sifStr2Num(char *buf, int32_t len, int8_t type, void *val) { static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { int32_t code = 0; int8_t ltype = left->colValType, rtype = right->colValType; - if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) { - return TSDB_CODE_INVALID_PARA; + // if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) { + // return TSDB_CODE_INVALID_PARA; + // } + if (IS_VAR_DATA_TYPE(ltype)) { + if (ltype == TSDB_DATA_TYPE_VARCHAR || ltype == TSDB_DATA_TYPE_BINARY || ltype == TSDB_DATA_TYPE_VARBINARY) { + return 0; + } else { + return TSDB_CODE_INVALID_PARA; + } } + if (ltype == TSDB_DATA_TYPE_FLOAT) { float f = 0; if (IS_NUMERIC_TYPE(rtype)) { diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index eb8f2c741a..f07d6f4cc1 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -814,15 +814,17 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l } int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t dbRefId, ECHECKPOINT_BACKUP_TYPE type) { - int32_t code = 0; - char* path = NULL; - + int32_t code = 0; + char* path = NULL; + int64_t chkptSize = 0; SStreamMeta* pMeta = pTask->pMeta; const char* idStr = pTask->id.idStr; int64_t now = taosGetTimestampMs(); SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES); if (toDelFiles == NULL) { + stError("s-task:%s failed to prepare array list during upload checkpoint, code:%s", pTask->id.idStr, + tstrerror(terrno)); return terrno; } @@ -848,11 +850,11 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d } } - if (code == TSDB_CODE_SUCCESS) { - int32_t size = taosArrayGetSize(toDelFiles); - stDebug("s-task:%s remove redundant %d files", idStr, size); + int32_t num = taosArrayGetSize(toDelFiles); + if (code == TSDB_CODE_SUCCESS && num > 0) { + stDebug("s-task:%s remove redundant %d files", idStr, num); - for (int i = 0; i < size; i++) { + for (int i = 0; i < num; i++) { char* pName = taosArrayGetP(toDelFiles, i); code = deleteCheckpointFile(idStr, pName); if (code != 0) { @@ -868,12 +870,13 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d double el = (taosGetTimestampMs() - now) / 1000.0; if (code == TSDB_CODE_SUCCESS) { - stDebug("s-task:%s complete update checkpointId:%" PRId64 ", elapsed time:%.2fs remove local checkpoint data %s", - idStr, checkpointId, el, path); - taosRemoveDir(path); + code = taosGetDirSize(path, &chkptSize); + stDebug("s-task:%s complete upload checkpointId:%" PRId64 + ", elapsed time:%.2fs, checkpointSize:%.2fKiB local dir:%s", + idStr, checkpointId, el, SIZE_IN_KiB(chkptSize), path); } else { - stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " keep local checkpoint data, elapsed time:%.2fs", idStr, - checkpointId, el); + stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " elapsed time:%.2fs, checkpointSize:%.2fKiB", idStr, + checkpointId, el, SIZE_IN_KiB(chkptSize)); } taosMemoryFree(path); @@ -883,7 +886,7 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t checkpointId) { ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType(); if (type == DATA_UPLOAD_DISABLE) { - stDebug("s-task:%s not allowed to upload checkpoint data", pTask->id.idStr); + stDebug("s-task:%s not config to backup checkpoint data at snode, checkpointId:%"PRId64, pTask->id.idStr, checkpointId); return 0; } @@ -925,6 +928,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s gen checkpoint:%" PRId64 " failed, code:%s", id, ckId, tstrerror(terrno)); } + + int64_t et = taosGetTimestampMs(); + stDebug("s-task:%s gen local checkpoint completed, elapsed time:%.2fs", id, (et - startTs) / 1000.0); } // TODO: monitoring the checkpoint-source msg diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index ca5b6630fd..a6d0142010 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -243,6 +243,8 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { continue; } + // todo: this lock may blocked by lock in streamMetaStartOneTask function, which may lock a very long time when + // trying to load remote checkpoint data streamMutexLock(&pTask->lock); STaskStatusEntry entry = streamTaskGetStatusEntry(pTask); streamMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 605cf3fe21..350c71204a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1364,6 +1364,10 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) } pMeta->role = (isLeader) ? NODE_ROLE_LEADER : NODE_ROLE_FOLLOWER; + if (!isLeader) { + streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId); + } + streamMetaWUnLock(pMeta); if (isLeader) { diff --git a/source/util/src/mpChunk.c b/source/util/src/mpChunk.c index 2c1c415c04..c7bdcd809f 100755 --- a/source/util/src/mpChunk.c +++ b/source/util/src/mpChunk.c @@ -190,6 +190,8 @@ int32_t mpChunkNSAllocMem(SMemPool* pPool, SMPSession* pSession, int64_t size, u void* pRes = NULL; int64_t totalSize = size + sizeof(SMPMemHeader) + sizeof(SMPMemTailer) + alignment; + + MP_ERR_JRET(mpChunkNewNS(pPool, &pChunk, totalSize)); SMPMemHeader* pHeader = (SMPMemHeader*)pChunk->pMemStart; MP_INIT_MEM_HEADER(pHeader, size, false); diff --git a/tests/script/tsim/analytics/basic0.sim b/tests/script/tsim/analytics/basic0.sim index 999b2fff37..0d9a29a19b 100644 --- a/tests/script/tsim/analytics/basic0.sim +++ b/tests/script/tsim/analytics/basic0.sim @@ -91,6 +91,14 @@ sql_error select count(*) from ct1 anomaly_window(c6, 'algo=ksigma,k=2'); sql_error select forecast(c6, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 +print ==================== invalid timeout parameter, will reset the parameters. +sql select forecast(c1, 'algo=holtwinters, timeout=6000') from ct1; +sql select forecast(c1, 'algo=holtwinters, timeout=0') from ct1; + +print =========================== valid timeout +sql select forecast(c1, 'algo=holtwinters, timeout=120') from ct1; + + sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1 sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1 diff --git a/tests/system-test/0-others/test_show_disk_usage.py b/tests/system-test/0-others/test_show_disk_usage.py index eb5bdf1aa7..1c863d7cdf 100644 --- a/tests/system-test/0-others/test_show_disk_usage.py +++ b/tests/system-test/0-others/test_show_disk_usage.py @@ -102,7 +102,8 @@ class TDTestCase: elif "Compress_radio=" in item[0]: value = item[0].split("=")[1].split(" ")[0].replace("[", "").replace("]", "") if value != 'NULL': - compress_radio = float(value) + tValue = value[0:len(value) - 1] + compress_radio = float(tValue) #tdLog.debug("compress_occupied: %s" % compress_radio) return disk_occupied, compress_radio diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 110a644e90..1ee2bc4ce6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -19,7 +19,8 @@ IF(TD_WEBSOCKET) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND RUSTFLAGS=-Ctarget-feature=-crt-static cargo build --release --locked -p taos-ws-sys --features rustls + COMMAND cargo update + COMMAND RUSTFLAGS=-Ctarget-feature=-crt-static cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include @@ -37,7 +38,8 @@ IF(TD_WEBSOCKET) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND cargo build --release --locked -p taos-ws-sys --features rustls + COMMAND cargo update + COMMAND cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/taosws.dll ${CMAKE_BINARY_DIR}/build/lib COMMAND cp target/release/taosws.dll.lib ${CMAKE_BINARY_DIR}/build/lib/taosws.lib @@ -56,7 +58,8 @@ IF(TD_WEBSOCKET) PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND cargo build --release --locked -p taos-ws-sys --features rustls + COMMAND cargo update + COMMAND cargo build --release -p taos-ws-sys --features rustls INSTALL_COMMAND COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include