Merge branch 'main' into fix/checkpoint
This commit is contained in:
Haojun Liao
GitHub
commit
2eeb13591c
|
|
@ -1,25 +1,20 @@
|
|||
name: TDengine CI Test
|
||||
|
||||
on:
|
||||
# pull_request:
|
||||
# branches:
|
||||
# - 'main'
|
||||
# - '3.0'
|
||||
# - '3.1'
|
||||
# paths-ignore:
|
||||
# - 'packaging/**'
|
||||
# - 'docs/**'
|
||||
repository_dispatch:
|
||||
types: [run-tests]
|
||||
pull_request:
|
||||
branches:
|
||||
- 'main'
|
||||
- '3.0'
|
||||
- '3.1'
|
||||
paths-ignore:
|
||||
- 'packaging/**'
|
||||
- 'docs/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.event.client_payload.ref}}-${{ github.event_name == 'repository_dispatch' && 'dispatch' || ''}}
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-TDengine
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CONTAINER_NAME: 'taosd-test'
|
||||
WKDIR: '/var/lib/jenkins/workspace'
|
||||
WK: '/var/lib/jenkins/workspace/TDinternal'
|
||||
WKC: '/var/lib/jenkins/workspace/TDinternal/community'
|
||||
|
||||
jobs:
|
||||
|
|
@ -28,430 +23,80 @@ jobs:
|
|||
group: CI
|
||||
labels: [self-hosted, Linux, X64, testing]
|
||||
outputs:
|
||||
tdinternal: ${{ steps.parameters.outputs.tdinternal }}
|
||||
run_function_test: ${{ steps.parameters.outputs.run_function_test }}
|
||||
run_tdgpt_test: ${{ steps.parameters.outputs.run_tdgpt_test }}
|
||||
source_branch: ${{ steps.parameters.outputs.source_branch }}
|
||||
target_branch: ${{ steps.parameters.outputs.target_branch }}
|
||||
pr_number: ${{ steps.parameters.outputs.pr_number }}
|
||||
steps:
|
||||
- name: Determine trigger source and fetch parameters
|
||||
id: parameters
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# check the trigger source and get branch information
|
||||
if [ "${{ github.event_name }}" == "repository_dispatch" ]; then
|
||||
tdinternal="true"
|
||||
source_branch=${{ github.event.client_payload.tdinternal_source_branch }}
|
||||
target_branch=${{ github.event.client_payload.tdinternal_target_branch }}
|
||||
pr_number=${{ github.event.client_payload.tdinternal_pr_number }}
|
||||
target_branch=${{ github.event.pull_request.base.ref }}
|
||||
|
||||
# Check whether to run tdgpt test cases
|
||||
cd ${{ env.WKC }}
|
||||
changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :)
|
||||
echo "changed files exclude doc: ${changed_files_non_doc}"
|
||||
|
||||
if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then
|
||||
run_tdgpt_test="true"
|
||||
else
|
||||
run_tdgpt_test="false"
|
||||
fi
|
||||
echo "run tdgpt test: ${run_tdgpt_test}"
|
||||
|
||||
# Check whether to run function test cases
|
||||
changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \
|
||||
grep -v "^docs/en/" | \
|
||||
grep -v "^docs/zh/" | \
|
||||
grep -v ".md$" | \
|
||||
grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \
|
||||
tr '\n' ' ' || :)
|
||||
echo "changed files exclude tdgpt: ${changed_files_non_tdgpt}"
|
||||
|
||||
if [ -n "$changed_files_non_tdgpt" ]; then
|
||||
run_function_test="true"
|
||||
else
|
||||
tdinternal="false"
|
||||
source_branch=${{ github.event.pull_request.head.ref }}
|
||||
target_branch=${{ github.event.pull_request.base.ref }}
|
||||
pr_number=${{ github.event.pull_request.number }}
|
||||
|
||||
# check whether to run tdgpt test cases
|
||||
cd ${{ env.WKC }}
|
||||
changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD $target_branch`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :)
|
||||
|
||||
if [[ "$changed_files_non_doc" != '' && "$changed_files_non_doc" =~ /forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics/ ]]; then
|
||||
run_tdgpt_test="true"
|
||||
else
|
||||
run_tdgpt_test="false"
|
||||
fi
|
||||
|
||||
# check whether to run function test cases
|
||||
changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD $target_branch`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -Ev "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics" | tr '\n' ' ' ||:)
|
||||
if [ "$changed_files_non_tdgpt" != '' ]; then
|
||||
run_function_test="true"
|
||||
else
|
||||
run_function_test="false"
|
||||
fi
|
||||
run_function_test="false"
|
||||
fi
|
||||
|
||||
echo "tdinternal=$tdinternal" >> $GITHUB_OUTPUT
|
||||
echo "run function test: ${run_function_test}"
|
||||
|
||||
# Output the results for GitHub Actions
|
||||
echo "run_function_test=$run_function_test" >> $GITHUB_OUTPUT
|
||||
echo "run_tdgpt_test=$run_tdgpt_test" >> $GITHUB_OUTPUT
|
||||
echo "source_branch=$source_branch" >> $GITHUB_OUTPUT
|
||||
echo "target_branch=$target_branch" >> $GITHUB_OUTPUT
|
||||
echo "pr_number=$pr_number" >> $GITHUB_OUTPUT
|
||||
|
||||
echo ${{ github.event.pull_request.head.ref }}
|
||||
echo ${{ github.event.pull_request.base.ref }}
|
||||
echo ${{ github.event.pull_request.number }}
|
||||
|
||||
run-tests-on-linux:
|
||||
uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml
|
||||
needs: fetch-parameters
|
||||
runs-on:
|
||||
group: CI
|
||||
labels: [self-hosted, Linux, X64, testing]
|
||||
timeout-minutes: 200
|
||||
env:
|
||||
IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }}
|
||||
RUN_RUNCTION_TEST: ${{ needs.fetch-parameters.outputs.run_function_test }}
|
||||
RUN_TDGPT_TEST: ${{ needs.fetch-parameters.outputs.run_tdgpt_test }}
|
||||
SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }}
|
||||
TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }}
|
||||
PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }}
|
||||
steps:
|
||||
- name: Output the environment information
|
||||
run: |
|
||||
echo "::group::Environment Info"
|
||||
date
|
||||
hostname
|
||||
env
|
||||
echo "Runner: ${{ runner.name }}"
|
||||
echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}"
|
||||
echo "Workspace: ${{ env.WKDIR }}"
|
||||
git --version
|
||||
echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}"
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Prepare repositories
|
||||
run: |
|
||||
set -euo pipefail
|
||||
prepare_environment() {
|
||||
cd "$1"
|
||||
git reset --hard
|
||||
git clean -f
|
||||
git remote prune origin
|
||||
git fetch
|
||||
git checkout "$2"
|
||||
}
|
||||
prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}"
|
||||
prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}"
|
||||
|
||||
- name: Get latest codes and logs for TDinternal PR
|
||||
if: ${{ env.IS_TDINTERNAL == 'true' }}
|
||||
run: |
|
||||
cd ${{ env.WK }}
|
||||
git pull >/dev/null
|
||||
git log -5
|
||||
echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
|
||||
echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
|
||||
echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
|
||||
git checkout -qf FETCH_HEAD
|
||||
git log -5
|
||||
echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
cd ${{ env.WKC }}
|
||||
git remote prune origin
|
||||
git pull >/dev/null
|
||||
git log -5
|
||||
echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
- name: Get latest codes and logs for TDengine PR
|
||||
if: ${{ env.IS_TDINTERNAL == 'false' }}
|
||||
run: |
|
||||
cd ${{ env.WKC }}
|
||||
git remote prune origin
|
||||
git pull >/dev/null
|
||||
git log -5
|
||||
echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
|
||||
echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
|
||||
echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
|
||||
git checkout -qf FETCH_HEAD
|
||||
git log -5
|
||||
echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
cd ${{ env.WK }}
|
||||
git pull >/dev/null
|
||||
git log -5
|
||||
echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
- name: Update submodule
|
||||
run: |
|
||||
cd ${{ env.WKC }}
|
||||
git submodule update --init --recursive
|
||||
- name: Output the 'file_no_doc_changed' information to the file
|
||||
if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }}
|
||||
run: |
|
||||
mkdir -p ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}
|
||||
changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ env.TARGET_BRANCH }}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :)
|
||||
echo $changed_files_non_doc > ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt
|
||||
- name: Check assert testing
|
||||
if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }}
|
||||
run: |
|
||||
cd ${{ env.WKC }}/tests/parallel_test
|
||||
./run_check_assert_container.sh -d ${{ env.WKDIR }}
|
||||
- name: Check void function testing
|
||||
if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }}
|
||||
run: |
|
||||
cd ${{ env.WKC }}/tests/parallel_test
|
||||
./run_check_void_container.sh -d ${{ env.WKDIR }}
|
||||
- name: Build docker container
|
||||
if: ${{ env.RUN_RUNCTION_TEST == 'true' }}
|
||||
run: |
|
||||
date
|
||||
rm -rf ${{ env.WKC }}/debug
|
||||
cd ${{ env.WKC }}/tests/parallel_test
|
||||
time ./container_build.sh -w ${{ env.WKDIR }} -e
|
||||
- name: Get parameters for testing
|
||||
id: get_param
|
||||
run: |
|
||||
log_server_file="/home/log_server.json"
|
||||
timeout_cmd=""
|
||||
extra_param=""
|
||||
|
||||
if [ -f "$log_server_file" ]; then
|
||||
log_server_enabled=$(jq '.enabled' "$log_server_file")
|
||||
timeout_param=$(jq '.timeout' "$log_server_file")
|
||||
if [ "$timeout_param" != "null" ] && [ "$timeout_param" != "0" ]; then
|
||||
timeout_cmd="timeout $timeout_param"
|
||||
fi
|
||||
|
||||
if [ "$log_server_enabled" == "1" ]; then
|
||||
log_server=$(jq '.server' "$log_server_file" | sed 's/\\\"//g')
|
||||
if [ "$log_server" != "null" ] && [ "$log_server" != "" ]; then
|
||||
extra_param="-w $log_server"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
echo "timeout_cmd=$timeout_cmd" >> $GITHUB_OUTPUT
|
||||
echo "extra_param=$extra_param" >> $GITHUB_OUTPUT
|
||||
- name: Run function returns with a null pointer scan testing
|
||||
if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }}
|
||||
run: |
|
||||
cd ${{ env.WKC }}/tests/parallel_test
|
||||
./run_scan_container.sh -d ${{ env.WKDIR }} -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -f ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt ${{ steps.get_param.outputs.extra_param }}
|
||||
- name: Run tdgpt test cases
|
||||
if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' && env.RUN_TDGPT_TEST == 'true' }}
|
||||
run: |
|
||||
cd ${{ env.WKC }}/tests/parallel_test
|
||||
export DEFAULT_RETRY_TIME=2
|
||||
date
|
||||
timeout 600 time ./run.sh -e -m /home/m.json -t tdgpt_cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 300 ${{ steps.get_param.outputs.extra_param }}
|
||||
- name: Run function test cases
|
||||
if: ${{ env.RUN_RUNCTION_TEST == 'true'}}
|
||||
run: |
|
||||
cd ${{ env.WKC }}/tests/parallel_test
|
||||
export DEFAULT_RETRY_TIME=2
|
||||
date
|
||||
${{ steps.get_param.outputs.timeout_cmd }} time ./run.sh -e -m /home/m.json -t cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 1200 ${{ steps.get_param.outputs.extra_param }}
|
||||
if: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || needs.fetch-parameters.outputs.run_function_test == 'true' }}
|
||||
with:
|
||||
tdinternal: false
|
||||
run_function_test: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' || false }}
|
||||
run_tdgpt_test: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || false }}
|
||||
source_branch: ${{ github.event.pull_request.head.ref }}
|
||||
target_branch: ${{ github.event.pull_request.base.ref }}
|
||||
pr_number: ${{ github.event.pull_request.number }}
|
||||
|
||||
run-tests-on-mac:
|
||||
uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml
|
||||
needs: fetch-parameters
|
||||
if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }}
|
||||
runs-on:
|
||||
group: CI
|
||||
labels: [self-hosted, macOS, ARM64, testing]
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }}
|
||||
SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }}
|
||||
TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }}
|
||||
PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }}
|
||||
steps:
|
||||
- name: Output the environment information
|
||||
run: |
|
||||
echo "::group::Environment Info"
|
||||
date
|
||||
hostname
|
||||
env
|
||||
echo "Runner: ${{ runner.name }}"
|
||||
echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}"
|
||||
echo "Workspace: ${{ env.WKDIR }}"
|
||||
git --version
|
||||
echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}"
|
||||
echo "::endgroup::"
|
||||
- name: Prepare repositories
|
||||
run: |
|
||||
set -euo pipefail
|
||||
prepare_environment() {
|
||||
cd "$1"
|
||||
git reset --hard
|
||||
git clean -f
|
||||
git remote prune origin
|
||||
git fetch
|
||||
git checkout "$2"
|
||||
}
|
||||
prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}"
|
||||
prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}"
|
||||
- name: Get latest codes and logs for TDinternal PR
|
||||
if: ${{ env.IS_TDINTERNAL == 'true' }}
|
||||
run: |
|
||||
cd ${{ env.WK }}
|
||||
git pull >/dev/null
|
||||
git log -5
|
||||
echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
|
||||
echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
|
||||
echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
|
||||
git checkout -qf FETCH_HEAD
|
||||
git log -5
|
||||
echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
cd ${{ env.WKC }}
|
||||
git remote prune origin
|
||||
git pull >/dev/null
|
||||
git log -5
|
||||
echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
- name: Get latest codes and logs for TDengine PR
|
||||
if: ${{ env.IS_TDINTERNAL == 'false' }}
|
||||
run: |
|
||||
cd ${{ env.WKC }}
|
||||
git remote prune origin
|
||||
git pull >/dev/null
|
||||
git log -5
|
||||
echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
|
||||
echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
|
||||
echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
|
||||
git checkout -qf FETCH_HEAD
|
||||
git log -5
|
||||
echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
cd ${{ env.WK }}
|
||||
git pull >/dev/null
|
||||
git log -5
|
||||
echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
|
||||
- name: Update submodule
|
||||
run: |
|
||||
cd ${{ env.WKC }}
|
||||
git submodule update --init --recursive
|
||||
- name: Run tests
|
||||
run: |
|
||||
date
|
||||
cd ${{ env.WK }}
|
||||
rm -rf debug
|
||||
mkdir debug
|
||||
cd ${{ env.WK }}/debug
|
||||
echo $PATH
|
||||
echo "PATH=/opt/homebrew/bin:$PATH" >> $GITHUB_ENV
|
||||
cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false -DCMAKE_BUILD_TYPE=Release
|
||||
make -j10
|
||||
ctest -j10 || exit 7
|
||||
date
|
||||
with:
|
||||
tdinternal: false
|
||||
source_branch: ${{ github.event.pull_request.head.ref }}
|
||||
target_branch: ${{ github.event.pull_request.base.ref }}
|
||||
pr_number: ${{ github.event.pull_request.number }}
|
||||
|
||||
run-tests-on-windows:
|
||||
uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml
|
||||
needs: fetch-parameters
|
||||
if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }}
|
||||
runs-on:
|
||||
group: CI
|
||||
labels: [self-hosted, Windows, X64, testing]
|
||||
timeout-minutes: 126
|
||||
env:
|
||||
IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }}
|
||||
SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }}
|
||||
TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }}
|
||||
PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }}
|
||||
WIN_INTERNAL_ROOT: "C:\\workspace\\0\\TDinternal"
|
||||
WIN_COMMUNITY_ROOT: "C:\\workspace\\0\\TDinternal\\community"
|
||||
WIN_SYSTEM_TEST_ROOT: "C:\\workspace\\0\\TDinternal\\community\\tests\\system-test"
|
||||
WIN_VS_PATH: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat"
|
||||
WIN_CPU_TYPE: "x64"
|
||||
steps:
|
||||
- name: Output the environment information
|
||||
run: |
|
||||
hostname
|
||||
taskkill /f /t /im python.exe
|
||||
taskkill /f /t /im bash.exe
|
||||
taskkill /f /t /im taosd.exe
|
||||
ipconfig
|
||||
set
|
||||
date /t
|
||||
time /t
|
||||
rd /s /Q "%WIN_INTERNAL_ROOT%\debug" || exit 0
|
||||
shell: cmd
|
||||
- name: Prepare repositories
|
||||
run: |
|
||||
:: Prepare internal repository
|
||||
if exist "%WIN_INTERNAL_ROOT%" (
|
||||
cd /d "%WIN_INTERNAL_ROOT%"
|
||||
git reset --hard
|
||||
git clean -f
|
||||
git remote prune origin
|
||||
git fetch
|
||||
git checkout "%TARGET_BRANCH%"
|
||||
) else (
|
||||
echo Directory does not exist: "%WIN_INTERNAL_ROOT%"
|
||||
exit 1
|
||||
)
|
||||
|
||||
:: Prepare community repository
|
||||
if exist "%WIN_COMMUNITY_ROOT%" (
|
||||
cd /d "%WIN_COMMUNITY_ROOT%"
|
||||
git reset --hard
|
||||
git clean -f
|
||||
git remote prune origin
|
||||
git fetch
|
||||
git checkout "%TARGET_BRANCH%"
|
||||
) else (
|
||||
echo Directory does not exist: "%WIN_COMMUNITY_ROOT%"
|
||||
exit 1
|
||||
)
|
||||
shell: cmd
|
||||
- name: Get latest codes and logs for TDinternal PR
|
||||
if: ${{ env.IS_TDINTERNAL == 'true' }}
|
||||
run: |
|
||||
cd %WIN_INTERNAL_ROOT%
|
||||
git pull origin %TARGET_BRANCH%
|
||||
git fetch origin +refs/pull/%PR_NUMBER%/merge
|
||||
git checkout -qf FETCH_HEAD
|
||||
cd %WIN_COMMUNITY_ROOT%
|
||||
git remote prune origin
|
||||
git pull
|
||||
shell: cmd
|
||||
- name: Get latest codes and logs for TDengine PR
|
||||
if: ${{ env.IS_TDINTERNAL == 'false' }}
|
||||
run: |
|
||||
cd %WIN_INTERNAL_ROOT%
|
||||
git pull origin %TARGET_BRANCH%
|
||||
cd %WIN_COMMUNITY_ROOT%
|
||||
git remote prune origin
|
||||
git pull origin %TARGET_BRANCH%
|
||||
git fetch origin +refs/pull/%PR_NUMBER%/merge
|
||||
git checkout -qf FETCH_HEAD
|
||||
shell: cmd
|
||||
- name: Output branch and log information
|
||||
run: |
|
||||
cd %WIN_INTERNAL_ROOT%
|
||||
git branch
|
||||
git log -5
|
||||
|
||||
cd %WIN_COMMUNITY_ROOT%
|
||||
git branch
|
||||
git log -5
|
||||
shell: cmd
|
||||
- name: Update submodule
|
||||
run: |
|
||||
cd %WIN_COMMUNITY_ROOT%
|
||||
git submodule update --init --recursive
|
||||
shell: cmd
|
||||
- name: Build on windows
|
||||
run: |
|
||||
echo "building ..."
|
||||
time /t
|
||||
cd %WIN_INTERNAL_ROOT%
|
||||
mkdir debug
|
||||
cd debug
|
||||
time /t
|
||||
call "%WIN_VS_PATH%" %WIN_CPU_TYPE%
|
||||
set CL=/MP8
|
||||
echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake"
|
||||
time /t
|
||||
cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true || exit 7
|
||||
echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6"
|
||||
time /t
|
||||
jom -j 6 || exit 8
|
||||
time /t
|
||||
|
||||
cd %WIN_COMMUNITY_ROOT%/tests/ci
|
||||
pip3 install taospy==2.7.21
|
||||
pip3 install taos-ws-py==0.3.8
|
||||
xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
|
||||
shell: cmd
|
||||
- name: Run ctest
|
||||
run: |
|
||||
echo "windows ctest ..."
|
||||
time /t
|
||||
cd %WIN_INTERNAL_ROOT%\\debug
|
||||
ctest -j 1 || exit 7
|
||||
time /t
|
||||
shell: cmd
|
||||
- name: Run function test
|
||||
run: |
|
||||
echo "windows test ..."
|
||||
xcopy /e/y/i/f "%WIN_INTERNAL_ROOT%\debug\build\lib\taos.dll" C:\Windows\System32
|
||||
ls -l "C:\Windows\System32\taos.dll"
|
||||
time /t
|
||||
cd %WIN_SYSTEM_TEST_ROOT%
|
||||
echo "testing ..."
|
||||
test-all.bat ci
|
||||
time /t
|
||||
shell: cmd
|
||||
with:
|
||||
tdinternal: false
|
||||
source_branch: ${{ github.event.pull_request.head.ref }}
|
||||
target_branch: ${{ github.event.pull_request.base.ref }}
|
||||
pr_number: ${{ github.event.pull_request.number }}
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ If you want to compile taosAdapter or taosKeeper, you need to install Go 1.18 or
|
|||
### For Ubuntu 18.04、20.04、22.04
|
||||
|
||||
```bash
|
||||
sudo apt-get udpate
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc cmake build-essential git libjansson-dev \
|
||||
libsnappy-dev liblzma-dev zlib1g-dev pkg-config
|
||||
```
|
||||
|
|
|
|||
|
|
@ -86,9 +86,15 @@ The keep alive interval is the time interval negotiated between the client and t
|
|||
|
||||
In **Clean Session**, choose whether to clear the session. The default value is true.
|
||||
|
||||
Fill in the Topic names to be consumed in **Subscription Topics and QoS Configuration**. Use the following format: `topic1::0,topic2::1`.
|
||||
In the **Topics Qos Config**, fill in the topic name and QoS to subscribe. Use the following format: `{topic_name}::{qos}` (e.g., `my_topic::0`). MQTT protocol 5.0 supports shared subscriptions, allowing multiple clients to subscribe to the same topic for load balancing. Use the following format: `$share/{group_name}/{topic_name}::{qos}`, where `$share` is a fixed prefix indicating the enablement of shared subscription, and `group_name` is the client group name, similar to Kafka's consumer group.
|
||||
|
||||
Click the **Check Connectivity** button to check if the data source is available.
|
||||
In the **Topic Analysis**, fill in the MQTT topic parsing rules. The format is the same as the MQTT Topic, parsing each level of the MQTT Topic into corresponding variable names, with `_` indicating that the current level is ignored during parsing. For example: if the MQTT Topic `a/+/c` corresponds to the parsing rule `v1/v2/_`, it means assigning the first level `a` to variable `v1`, the value of the second level (where the wildcard `+` represents any value) to variable `v2`, and ignoring the value of the third level `c`, which will not be assigned to any variable. In the `payload parsing` below, the variables obtained from Topic parsing can also participate in various transformations and calculations.
|
||||
|
||||
In the **Compression**, configure the message body compression algorithm. After receiving the message, taosX uses the corresponding compression algorithm to decompress the message body and obtain the original data. Options include none (no compression), gzip, snappy, lz4, and zstd, with the default being none.
|
||||
|
||||
In the **Char Encoding**, configure the message body encoding format. After receiving the message, taosX uses the corresponding encoding format to decode the message body and obtain the original data. Options include UTF_8, GBK, GB18030, and BIG5, with the default being UTF_8.
|
||||
|
||||
Click the **Check Connection** button to check if the data source is available.
|
||||
|
||||
<figure>
|
||||
<Image img={imgStep05} alt=""/>
|
||||
|
|
|
|||
|
|
@ -339,7 +339,7 @@ Helm operates Kubernetes using kubectl and kubeconfig configurations, which can
|
|||
The TDengine Chart has not yet been released to the Helm repository, it can currently be downloaded directly from GitHub:
|
||||
|
||||
```shell
|
||||
wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-enterpise-3.5.0.tgz
|
||||
wget https://github.com/taosdata/TDengine-Operator/raw/refs/heads/3.0/helm/tdengine-enterprise-3.5.0.tgz
|
||||
```
|
||||
|
||||
Note that it's for the enterprise edition, and the community edition is not yet available.
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ The sink task is responsible for receiving the output results from the agg task
|
|||
The above three types of tasks each play their roles in the stream computing architecture, distributed at different levels. Clearly, the number of source tasks directly depends on the number of vnodes, with each source task independently handling the data in its vnode without interference from other source tasks, and there are no sequential constraints. However, it is worth noting that if the final stream computing results converge to one table, then only one sink task will be deployed on the vnode where that table is located. The collaborative relationship between these three types of tasks is shown in the following diagram, together forming the complete execution process of stream computing tasks.
|
||||
|
||||
<figure>
|
||||
<Image img={imgStep02} alt="Relationships between tasks"/>
|
||||
<Image img={imgStep03} alt="Relationships between tasks"/>
|
||||
<figcaption>Figure 3. Relationships between tasks</figcaption>
|
||||
</figure>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,293 @@
|
|||
---
|
||||
title: DST(Daylight Saving Time) Usage
|
||||
description: Explanation and suggestions for using DST(Daylight Saving Time) in TDengine
|
||||
---
|
||||
|
||||
## Background
|
||||
|
||||
In the use of time-series databases, there are times when Daylight Saving Time (DST) is encountered. We analyze and explain the use and issues of DST in TDengine to help you use TDengine more smoothly.
|
||||
|
||||
## Definitions
|
||||
|
||||
### Time Zone
|
||||
|
||||
A time zone is a region on Earth that uses the same standard time. Due to the Earth's rotation, to ensure that the time in each place is coordinated with the local sunrise and sunset, the world is divided into multiple time zones.
|
||||
|
||||
### IANA Time Zone
|
||||
|
||||
The IANA (Internet Assigned Numbers Authority) time zone database, also known as the tz database, provides a standard reference for global time zone information. It is the basis for modern systems and software to handle time zone-related operations.
|
||||
|
||||
IANA uses the "Region/City" format (e.g., Europe/Berlin) to clearly identify time zones.
|
||||
|
||||
TDengine supports the use of IANA time zones in different components (except for the time zone settings in Windows taos.cfg).
|
||||
|
||||
### Standard Time and Local Time
|
||||
|
||||
Standard time is the time determined based on a fixed meridian on Earth. It provides a unified reference point for each time zone.
|
||||
|
||||
- Greenwich Mean Time (GMT): Historically used reference time, located at the 0° meridian.
|
||||
- Coordinated Universal Time (UTC): The modern time standard, similar to GMT but more precise.
|
||||
|
||||
The relationship between standard time and time zones is as follows:
|
||||
|
||||
- Reference: Standard time (e.g., UTC) is the reference point for setting time zones.
|
||||
- Offset: Different time zones are defined by their offset from standard time. For example, UTC+1 means 1 hour ahead of UTC.
|
||||
- Regional Division: The world is divided into multiple time zones, each using one or more standard times.
|
||||
|
||||
Relative to standard time, each region sets its local time based on its time zone:
|
||||
|
||||
- Time Zone Offset: Local time equals standard time plus the offset of the time zone. For example, UTC+2 means 2 hours ahead of UTC.
|
||||
- Daylight Saving Time (DST): Some regions adjust their local time during specific periods, such as moving the clock forward by one hour. See the next section for details.
|
||||
|
||||
### Daylight Saving Time
|
||||
|
||||
Daylight Saving Time (DST) is a system that advances the time by one hour to make better use of daylight and save energy. It usually starts in spring and ends in autumn. The specific start and end times of DST vary by region. The following explanation uses Berlin time as an example to illustrate DST and its effects.
|
||||
|
||||
|
||||
|
||||
According to this rule, you can see:
|
||||
|
||||
- The time between 02:00:00 and 03:00:00 (excluding 03:00:00) on March 31, 2024, in Berlin local time does not exist (jump).
|
||||
- The time between 02:00:00 and 03:00:00 (excluding 03:00:00) on October 27, 2024, in Berlin local time appears twice.
|
||||
|
||||
#### DST and the IANA Time Zone Database
|
||||
|
||||
- Recording Rules: The IANA time zone database records detailed DST rules for each region, including the start and end dates and times.
|
||||
- Automatic Adjustment: Many operating systems and software use the IANA database to automatically handle DST adjustments.
|
||||
- Historical Changes: The IANA database also tracks historical DST changes to ensure accuracy.
|
||||
|
||||
#### DST and Timestamp Conversion
|
||||
|
||||
- Converting a timestamp to local time is deterministic. For example, 1729990654 is Berlin time DST 2024-10-27 02:57:34, and 1729994254 is Berlin time standard time 2024-10-27 02:57:34 (these two local times are the same except for the time offset).
|
||||
- Without specifying the time offset, converting local time to a timestamp is indeterminate. The time skipped during DST does not exist and cannot be converted to a timestamp, such as Berlin time 2024-03-31 02:34:56 does not exist and cannot be converted to a timestamp. The repeated time during the end of DST cannot determine which timestamp it is, such as 2024-10-27 02:57:34 without specifying the time offset cannot determine whether it is 1729990654 or 1729994254. Specifying the time offset can determine the timestamp, such as 2024-10-27 02:57:34 CEST(+02:00), specifying DST 2024-10-27 02:57:34 timestamp 1729990654.
|
||||
|
||||
### RFC3339 Time Format
|
||||
|
||||
RFC 3339 is an internet time format standard used to represent dates and times. It is based on the ISO 8601 standard but specifies some format details more specifically.
|
||||
|
||||
The format is as follows:
|
||||
|
||||
- Basic Format: `YYYY-MM-DDTHH:MM:SSZ`
|
||||
- Time Zone Representation:
|
||||
- Z represents Coordinated Universal Time (UTC).
|
||||
- Offset format, such as +02:00, represents the time difference from UTC.
|
||||
|
||||
With explicit time zone offsets, the RFC 3339 format can accurately parse and compare times globally.
|
||||
|
||||
The advantages of RFC 3339 include:
|
||||
|
||||
- Standardization: Provides a unified format for easy cross-system data exchange.
|
||||
- Clarity: Clearly indicates time zone information, avoiding time misunderstandings.
|
||||
|
||||
TDengine uses the RFC3339 format for display in REST API and Explorer UI. In SQL statements, you can use the RFC3339 format to write timestamp data:
|
||||
|
||||
```sql
|
||||
insert into t1 values('2024-10-27T01:59:59.000Z', 0);
|
||||
select * from t1 where ts >= '2024-10-27T01:59:59.000Z';
|
||||
```
|
||||
|
||||
### Undefined Behavior
|
||||
|
||||
Undefined behavior refers to specific code or operations that do not have a clearly defined result and do not guarantee compatibility with that result. TDengine may modify the current behavior in a future version without notifying users. Therefore, users should not rely on the current undefined behavior for judgment or application in TDengine.
|
||||
|
||||
## Writing and Querying DST in TDengine
|
||||
|
||||
We use the following table to show the impact of DST on writing and querying.
|
||||
|
||||
|
||||
|
||||
### Table Explanation
|
||||
|
||||
- **TIMESTAMP**: TDengine uses a 64-bit integer to store raw timestamps.
|
||||
- **UTC**: The UTC time representation corresponding to the timestamp.
|
||||
- **Europe/Berlin**: The RFC3339 format time corresponding to the Europe/Berlin time zone.
|
||||
- **Local**: The local time corresponding to the Europe/Berlin time zone (without time zone).
|
||||
|
||||
### Table Analysis
|
||||
|
||||
- At the **start of DST** (Berlin time March 31, 02:00), the time jumps directly from 02:00 to 03:00 (one hour forward).
|
||||
- Light green is the timestamp one hour before the start of DST;
|
||||
- Dark green is the timestamp one hour after the start of DST;
|
||||
- Red indicates that the nonexistent local time was inserted into the TDengine database:
|
||||
- Using SQL `INSERT INTO t1 values('2024-03-31 02:59:59',..)` to insert data from `2024-03-31 02:00:00` to `2024-03-31 02:59:59` will be automatically adjusted to -1000 (in TDengine, this is undefined behavior, currently this value is related to the database precision, millisecond database is -1000, microsecond database is -1000000, nanosecond database is -1000000000), because that moment does not exist in local time;
|
||||
- At the **end of DST** (Berlin time October 27, 03:00), the time jumps from 03:00 to 02:00 (one hour back).
|
||||
- Light blue indicates the timestamp one hour before the clock jump;
|
||||
- Dark blue indicates the timestamp within one hour after the clock jump, its local time without time zone is the same as the previous hour.
|
||||
- Purple indicates the timestamp one hour after the clock jump;
|
||||
- **Local Time Changes**: It can be seen that due to the adjustment of DST, local time changes, which may cause some time periods to appear repeated or missing.
|
||||
- **UTC Time Unchanged**: UTC time remains unchanged, ensuring the consistency and order of time.
|
||||
- **RFC3339**: The RFC3339 format time shows the change in time offset, changing to +02:00 after the start of DST and to +01:00 after the end of DST.
|
||||
- **Conditional Query**:
|
||||
- At the **start of DST**, the skipped time (`[03-31 02:00:00,03-31 03:00:00)`) does not exist, so using that time for queries results in undefined behavior: `SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'` (the nonexistent local timestamp is converted to `-1000`):
|
||||
|
||||
```sql
|
||||
taos> SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59';
|
||||
ts |
|
||||
=================
|
||||
-1000 |
|
||||
Query OK, 1 row(s) in set (0.003635s)
|
||||
```
|
||||
|
||||
When the nonexistent timestamp is used together with the existing timestamp, the result is also not as expected, as shown below where the start local time does not exist:
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 03:59:59';
|
||||
ts | to_iso8601(ts,'Z') |
|
||||
==================================================
|
||||
-1000 | 1969-12-31T23:59:59.000Z |
|
||||
1711843200000 | 2024-03-31T00:00:00.000Z |
|
||||
1711846799000 | 2024-03-31T00:59:59.000Z |
|
||||
1711846800000 | 2024-03-31T01:00:00.000Z |
|
||||
1711846801000 | 2024-03-31T01:00:01.000Z |
|
||||
Query OK, 5 row(s) in set (0.003339s)
|
||||
```
|
||||
|
||||
In the following statements, the first SQL query end time does not exist, and the second end time exists. The first SQL query result is not as expected:
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 02:00:00';
|
||||
Query OK, 0 row(s) in set (0.000930s)
|
||||
|
||||
taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 01:59:59';
|
||||
ts | to_iso8601(ts,'Z') |
|
||||
==================================================
|
||||
1711843200000 | 2024-03-31T00:00:00.000Z |
|
||||
1711846799000 | 2024-03-31T00:59:59.000Z |
|
||||
Query OK, 2 row(s) in set (0.001227s)
|
||||
```
|
||||
|
||||
- At the end of DST, the repeated time (`[10-27 02:00:00,10-27 03:00:00)` excluding `10-27 03:00:00`) appears twice, and using that time range for queries in TDengine is also undefined behavior.
|
||||
- Querying the data between `[2024-10-27 02:00:00, 2024-10-27 03:00:00]` includes the repeated timestamps and the data at `2024-10-27 03:00:00`:
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts BETWEEN '2024-10-27 02:00:00' AND '2024-10-27 03:00:00';
|
||||
ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
=======================================================================================
|
||||
1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
1729994400000 | 2024-10-27T02:00:00.000Z | 2024-10-27 03:00:00 |
|
||||
Query OK, 5 row(s) in set (0.001370s)
|
||||
```
|
||||
|
||||
- However, the following query for the range [2024-10-27 02:00:00.000,2024-10-27 02:57:34.999] can only find the data at the first 2024-10-27 02:00:00 time point:
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:00.999';
|
||||
ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
=======================================================================================
|
||||
1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
Query OK, 1 row(s) in set (0.004480s)
|
||||
```
|
||||
|
||||
- The following query for the range [2024-10-27 02:00:01,2024-10-27 02:57:35] can find 3 rows of data (including one row of local time data at 02:59:59):
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:35';;
|
||||
ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
===============================================================================================
|
||||
2024-10-27 02:00:00.000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
2024-10-27 02:59:59.000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
2024-10-27 02:00:00.000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
Query OK, 3 row(s) in set (0.004428s)
|
||||
```
|
||||
|
||||
## Summary and Suggestions
|
||||
|
||||
### Summary
|
||||
|
||||
This explanation only addresses the impact of using local time. Using UNIX timestamps or RFC3339 has no impact.
|
||||
|
||||
- Writing:
|
||||
- It is not possible to write data for nonexistent times during the DST transition.
|
||||
- Writing data for repeated times during the DST transition is undefined behavior.
|
||||
- Querying:
|
||||
- Querying with conditions that specify the skipped time during the start of DST results in undefined behavior.
|
||||
- Querying with conditions that specify the repeated time during the end of DST results in undefined behavior.
|
||||
- Display:
|
||||
- Displaying with time zones is not affected.
|
||||
- Displaying local time is accurate, but repeated times during the end of DST cannot be distinguished.
|
||||
- Users should be cautious when using time without time zones for display and application.
|
||||
|
||||
### Suggestions
|
||||
|
||||
To avoid unnecessary impacts of DST on querying and writing in TDengine, it is recommended to use explicit time offsets for writing and querying.
|
||||
|
||||
- Use UNIX Timestamps: Using UNIX timestamps can avoid time zone issues.
|
||||
|
||||
| TIMESTAMP | UTC | Europe/Berlin | Local |
|
||||
| ------------: | :----------------------: | :---------------------------: | :-----------------: |
|
||||
| 1711846799000 | 2024-03-31T00:59:59.000Z | 2024-03-31T01:59:59.000+01:00 | 2024-03-31 01:59:59 |
|
||||
| 1711846800000 | 2024-03-31T01:00:00.000Z | 2024-03-31T03:00:00.000+02:00 | 2024-03-31 03:00:00 |
|
||||
|
||||
```sql
|
||||
taos> insert into t1 values(1711846799000, 1)(1711846800000, 2);
|
||||
Insert OK, 2 row(s) affected (0.001434s)
|
||||
|
||||
taos> select * from t1 where ts between 1711846799000 and 1711846800000;
|
||||
ts | v1 |
|
||||
===============================
|
||||
1711846799000 | 1 |
|
||||
1711846800000 | 2 |
|
||||
Query OK, 2 row(s) in set (0.003503s)
|
||||
```
|
||||
|
||||
- Use RFC3339 Time Format: The RFC3339 time format with time zone offsets can effectively avoid the uncertainty of DST.
|
||||
|
||||
| TIMESTAMP | UTC | Europe/Berlin | Local |
|
||||
| ------------: | :----------------------: | :---------------------------: | :-----------------: |
|
||||
| 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27T02:00:00.000+02:00 | 2024-10-27 02:00:00 |
|
||||
| 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27T02:59:59.000+02:00 | 2024-10-27 02:59:59 |
|
||||
| 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27T02:00:00.000+01:00 | 2024-10-27 02:00:00 |
|
||||
| 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27T02:59:59.000+01:00 | 2024-10-27 02:59:59 |
|
||||
|
||||
```sql
|
||||
taos> insert into t1 values ('2024-10-27T02:00:00.000+02:00', 1)
|
||||
('2024-10-27T02:59:59.000+02:00', 2)
|
||||
('2024-10-27T02:00:00.000+01:00', 3)
|
||||
('2024-10-27T02:59:59.000+01:00', 4);
|
||||
Insert OK, 4 row(s) affected (0.001514s)
|
||||
|
||||
taos> SELECT *,
|
||||
to_iso8601(ts,'Z'),
|
||||
to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1
|
||||
WHERE ts >= '2024-10-27T02:00:00.000+02:00'
|
||||
AND ts <= '2024-10-27T02:59:59.000+01:00';
|
||||
ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
=====================================================================================================
|
||||
1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
1729990800000 | 3 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729994399000 | 4 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
Query OK, 4 row(s) in set (0.004275s)
|
||||
|
||||
taos> SELECT *,
|
||||
to_iso8601(ts,'Z'),
|
||||
to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1
|
||||
WHERE ts >= '2024-10-27T02:00:00.000+02:00'
|
||||
AND ts <= '2024-10-27T02:59:59.000+02:00';
|
||||
ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
=====================================================================================================
|
||||
1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
Query OK, 2 row(s) in set (0.004275s)
|
||||
```
|
||||
|
||||
- Pay Attention to Time Zone Settings When Querying: When querying and displaying, if local time is needed, be sure to consider the impact of DST.
|
||||
- taosAdapter: When using the REST API, it supports setting the IANA time zone, and the result is returned in RFC3339 format.
|
||||
|
||||
```shell
|
||||
$ curl -uroot:taosdata 'localhost:6041/rest/sql?tz=Europe/Berlin'\
|
||||
-d "select ts from tz1.t1"
|
||||
{"code":0,"column_meta":[["ts","TIMESTAMP",8]],"data":[["1970-01-01T00:59:59.000+01:00"],["2024-03-31T01:00:00.000+01:00"],["2024-03-31T01:59:59.000+01:00"],["2024-03-31T03:00:00.000+02:00"],["2024-03-31T03:00:01.000+02:00"],["2024-10-27T02:00:00.000+02:00"],["2024-10-27T02:59:59.000+02:00"],["2024-10-27T02:00:00.000+01:00"],["2024-10-27T02:59:59.000+01:00"],["2024-10-27T03:00:00.000+01:00"]],"rows":10}
|
||||
```
|
||||
|
||||
- Explorer: When using the Explorer page for SQL queries, users can configure the client time zone to display in RFC3339 format.
|
||||
|
||||
|
||||
|
||||
## Reference Documents
|
||||
|
||||
- IANA Time Zone Database: [https://www.iana.org/time-zones](https://www.iana.org/time-zones)
|
||||
- RFC3339: [https://datatracker.ietf.org/doc/html/rfc3339](https://datatracker.ietf.org/doc/html/rfc3339)
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 234 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 80 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 72 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 52 KiB After Width: | Height: | Size: 268 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 44 KiB After Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB After Width: | Height: | Size: 216 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 51 KiB After Width: | Height: | Size: 184 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 243 KiB After Width: | Height: | Size: 334 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 162 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 48 KiB After Width: | Height: | Size: 293 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 90 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 103 KiB After Width: | Height: | Size: 263 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 282 KiB After Width: | Height: | Size: 162 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 40 KiB After Width: | Height: | Size: 51 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 52 KiB After Width: | Height: | Size: 90 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB After Width: | Height: | Size: 57 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 338 KiB After Width: | Height: | Size: 264 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 501 KiB After Width: | Height: | Size: 342 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 93 KiB After Width: | Height: | Size: 63 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 140 KiB After Width: | Height: | Size: 91 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 96 KiB After Width: | Height: | Size: 68 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 149 KiB After Width: | Height: | Size: 140 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 131 KiB After Width: | Height: | Size: 109 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 117 KiB After Width: | Height: | Size: 137 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 133 KiB After Width: | Height: | Size: 82 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 141 KiB After Width: | Height: | Size: 200 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 70 KiB After Width: | Height: | Size: 107 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 65 KiB After Width: | Height: | Size: 57 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 178 KiB After Width: | Height: | Size: 135 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 190 KiB After Width: | Height: | Size: 283 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 26 KiB After Width: | Height: | Size: 68 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 61 KiB After Width: | Height: | Size: 80 KiB |
|
|
@ -4,6 +4,10 @@ title: TDengine 数据查询
|
|||
toc_max_heading_level: 4
|
||||
---
|
||||
|
||||
import win from './window.png';
|
||||
import swin from './session-window.png';
|
||||
import ewin from './event-window.png';
|
||||
|
||||
相较于其他众多时序数据库和实时数据库，TDengine 的一个独特优势在于，自其首个版本发布之初便支持标准的 SQL 查询功能。这一特性极大地降低了用户在使用过程中的学习难度。本章将以智能电表的数据模型为例介绍如何在 TDengine 中运用 SQL 查询来处理时序数据。如果需要进一步了解 SQL 语法的细节和功能，建议参阅 TDengine 的官方文档。通过本章的学习，你将能够熟练掌握 TDengine 的 SQL 查询技巧，进而高效地对时序数据进行操作和分析。
|
||||
|
||||
## 基本查询
|
||||
|
|
@ -145,7 +149,7 @@ Query OK, 10 row(s) in set (2.415961s)
|
|||
|
||||
窗口划分逻辑如下图所示：
|
||||
|
||||
|
||||
<img src={win} width="500" alt="常用窗口划分逻辑" />
|
||||
|
||||
窗口子句语法如下：
|
||||
|
||||
|
|
@ -408,7 +412,8 @@ Query OK, 22 row(s) in set (0.153403s)
|
|||
|
||||
会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示，如果设置时间戳的连续的间隔小于等于 12 秒，则以下 6 条记录构成 2 个会话窗口，分别是：[2019-04-28 14:22:10，2019-04-28 14:22:30] 和 [2019-04-28 14:23:10，2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒，超过了连续时间间隔（12 秒）。
|
||||
|
||||
|
||||
<img src={swin} width="320" alt="会话窗口示意图" />
|
||||
|
||||
|
||||
在 tol_value 时间间隔范围内的结果都认为归属于同一个窗口，如果连续的两条记录的时间超过 tol_val，则自动开启下一个窗口。
|
||||
|
||||
|
|
@ -461,7 +466,7 @@ Query OK, 10 row(s) in set (0.043489s)
|
|||
select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c2 < 10
|
||||
```
|
||||
|
||||
|
||||
<img src={ewin} width="350" alt="事件窗口示意图" />
|
||||
|
||||
示例 SQL:
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 188 KiB After Width: | Height: | Size: 95 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 22 KiB After Width: | Height: | Size: 47 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 188 KiB After Width: | Height: | Size: 97 KiB |
|
|
@ -0,0 +1,291 @@
|
|||
---
|
||||
title: 夏令时使用指南
|
||||
description: TDengine 中关于夏令时使用问题的解释和建议
|
||||
---
|
||||
|
||||
## 背景
|
||||
|
||||
在时序数据库的使用中，有时会遇到使用夏令时的情况。我们将 TDengine 中使用夏令时的情况和问题进行分析说明，以便您在 TDengine 的使用中更加顺利。
|
||||
|
||||
## 定义
|
||||
|
||||
### 时区
|
||||
|
||||
时区是地球上使用相同标准时间的区域。由于地球的自转，为了保证各地的时间与当地的日出日落相协调，全球划分为多个时区。
|
||||
|
||||
### IANA 时区
|
||||
|
||||
IANA（Internet Assigned Numbers Authority）时区数据库，也称为 tz database，提供全球时区信息的标准参考。它是现代各类系统和软件处理时区相关操作的基础。
|
||||
|
||||
IANA 使用“区域/城市”格式（如 Europe/Berlin）来明确标识时区。
|
||||
|
||||
TDengine 在不同组件中均支持使用 IANA 时区（除 Windows taos.cfg 时区设置外）。
|
||||
|
||||
### 标准时间与当地时间
|
||||
|
||||
标准时间是根据地球上某个固定经线确定的时间。它为各个时区提供了一个统一的参考点。
|
||||
|
||||
- 格林尼治标准时间（GMT）：历史上使用的参考时间，位于 0° 经线。
|
||||
- 协调世界时（UTC）：现代的时间标准，类似于GMT，但更加精确。
|
||||
|
||||
标准时间与时区的关系如下：
|
||||
|
||||
- 基准：标准时间（如 UTC）是时区设定的基准点。
|
||||
- 偏移量：不同时区通过相对于标准时间的偏移量来定义。例如，UTC+1 表示比 UTC 快 1 小时。
|
||||
- 区域划分：全球被划分为多个时区，每个时区使用一个或多个标准时间。
|
||||
|
||||
相对于标准时间，每个地区根据其所在时区设定其当地时间：
|
||||
|
||||
- 时区偏移：当地时间等于标准时间加上该时区的偏移量。例如，UTC+2 表示比 UTC 时间快 2 小时。
|
||||
- 夏令时（DST）：某些地区在特定时间段调整当地时间，例如将时钟拨快一小时。详见下节。
|
||||
|
||||
### 夏令时
|
||||
|
||||
夏令时（Daylight Saving Time，DST）是一种通过将时间提前一小时，以充分利用日光、节约能源的制度。通常在春季开始，秋季结束。夏令时的具体开始和结束时间因地区而异。以下均以柏林时间为例，对夏令时和夏令时的影响做说明。
|
||||
|
||||
按照这个规则，可以看到：
|
||||
|
||||
- 柏林当地时间 2024 年 03 月 31 日 02:00:00 到 03:00:00 （不含 03:00:00）之间的时间不存在（跳变）。
|
||||
- 柏林当地时间 2024 年 10 月 27 日 02:00:00 到 03:00:00 （不含 03:00:00）之间的时间出现了两次。
|
||||
|
||||
#### 夏令时与 IANA 时区数据库
|
||||
|
||||
- 记录规则：IANA 时区数据库详细记录了各地的夏令时规则，包括开始和结束的日期与时间。
|
||||
- 自动调整：许多操作系统和软件利用 IANA 数据库来自动处理夏令时的调整。
|
||||
- 历史变更：IANA 数据库还追踪历史上的夏令时变化，以确保准确性。
|
||||
|
||||
#### 夏令时与时间戳转换
|
||||
|
||||
- 时间戳转为当地时间是确定的。例如，1729990654 为柏林时间**夏令时** `2024-10-27 02:57:34`，1729994254 为柏林时间**冬令时** `2024-10-27 02:57:34`（这两个本地时间除时间偏移量外是一样的）。
|
||||
- 不指定时间偏移量时，当地时间转为时间戳是不确定的。夏令时跳过的时间不存在会造成无法转换成时间戳，如 **柏林时间** `2024-03-31 02:34:56` 不存在，所以无法转换为时间戳。夏令时结束时重复导致无法确定是哪个时间戳，如 `2024-10-27 02:57:34` 不指定时间偏移量无法确定 是 1729990654 还是 1729994254。指定时间偏移量才能确定时间戳，如 `2024-10-27 02:57:34 CEST(+02:00) `，指定了夏令时 `2024-10-27 02:57:34` 时间戳 1729990654 。
|
||||
|
||||
### RFC3339 时间格式
|
||||
|
||||
RFC 3339 是一种互联网时间格式标准，用于表示日期和时间。它基于 ISO 8601 标准，但更具体地规定了一些格式细节。
|
||||
|
||||
其格式如下：
|
||||
|
||||
- 基本格式：`YYYY-MM-DDTHH:MM:SSZ`
|
||||
- 时区表示：
|
||||
- Z 表示协调世界时（UTC）。
|
||||
- 偏移量格式，例如 +02:00，表示与 UTC 的时差。
|
||||
|
||||
通过明确的时区偏移，RFC 3339 格式可以在全球范围内准确地解析和比较时间。
|
||||
|
||||
RFC 3339 的优势包括：
|
||||
|
||||
- 标准化：提供统一的格式，方便跨系统数据交换。
|
||||
- 清晰性：明确时区信息，避免时间误解。
|
||||
|
||||
TDengine 在 REST API 和 Explorer UI 中，均使用 RFC3339 格式进行展示。在 SQL 语句中，可使用 RFC3339 格式写入时间戳数据：
|
||||
|
||||
```sql
|
||||
insert into t1 values('2024-10-27T01:59:59.000Z', 0);
|
||||
select * from t1 where ts >= '2024-10-27T01:59:59.000Z';
|
||||
```
|
||||
|
||||
### 未定义行为
|
||||
|
||||
未定义行为（Undefined Behavior）是指特定代码或操作没有明确规定的结果，也不会对该结果作出兼容性的保证，TDengine 可能在某个版本后对当前的行为作出修改而不会通知用户。所以，在 TDengine 中，用户不可依赖当前未定义的行为进行判断或应用。
|
||||
|
||||
## 夏令时在 TDengine 中的写入与查询
|
||||
|
||||
我们使用下表来展示夏令时在写入和查询中的影响。
|
||||
|
||||
|
||||
|
||||
### 表格说明
|
||||
|
||||
- **TIMESTAMP**：TDengine 中使用 64位整数来存储原始时间戳。
|
||||
- **UTC**：时间戳对应的 UTC 时间表示。
|
||||
- **Europe/Berlin**：表示时区 Europe/Berlin 对应的 RFC3339 格式时间。
|
||||
- **Local**：表示时区 Europe/Berlin 对应的当地时间（不含时区）。
|
||||
|
||||
### 表格分析
|
||||
|
||||
- 在**夏令时开始**（柏林时间 3 月 31 日 02:00）时，时间直接从 02:00 跳到 03:00（往后跳一小时）。
|
||||
- 浅绿色是夏令时开始前一小时的时间戳；
|
||||
- 深绿色是夏令时开始后一小时的时间戳；
|
||||
- 红色为 TDengine 数据库中插入了不存在的当地时间：
|
||||
- 使用 SQL `INSERT INTO t1 values('2024-03-31 02:59:59',..)` 插入 `2024-03-31 02:00:00` 到 `2024-03-31 02:59:59` 的数据会被自动调整为 -1000（在 TDengine 中属于未定义行为，当前该值与数据库精度 precision 有关，毫秒数据库为 -1000，微秒数据库为 -1000000，纳秒数据库为 -1000000000），因为那一时刻在本地时间中不存在；
|
||||
- 在**夏令时结束**（柏林时间 10 月 27 日 03:00）时，时间从 03:00 跳到 02:00 （往前跳一小时）。
|
||||
- 浅蓝色表示时钟跳变前一小时的时间戳；
|
||||
- 深蓝色表示时钟跳变后一小时内的时间戳，其无时区的当地时间与上一小时一致。
|
||||
- 紫色表示时钟跳变一小时后的时间戳；
|
||||
- **当地时间变化**：可见，由于夏令时的调整而导致了当地时间的变化，可能导致某些时间段出现重复或缺失。
|
||||
- **UTC 时间不变**：UTC 时间保持不变，确保了时间的一致性和顺序性。
|
||||
- **RFC3339**：RFC3339 格式时间显示了时间偏移量的变化，在夏令时开始后变为 +02:00，结束后变为 +01:00 。
|
||||
- **条件查询**：
|
||||
- **夏令时开始**时，跳过的时间（`[03-31 02:00:00,03-31 03:00:00)`）不存在，所以在使用该时间进行查询时，行为不确定：`SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59'`（不存在的本地时间戳被转换为 `-1000`）:
|
||||
|
||||
```sql
|
||||
taos> SELECT ts FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 02:59:59';
|
||||
ts |
|
||||
=================
|
||||
-1000 |
|
||||
Query OK, 1 row(s) in set (0.003635s)
|
||||
```
|
||||
|
||||
当不存在的时间戳与存在的时间戳共同使用时，其结果同样不符合预期，以下为起始本地时间不存在：
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 02:00:00' AND '2024-03-31 03:59:59';
|
||||
ts | to_iso8601(ts,'Z') |
|
||||
==================================================
|
||||
-1000 | 1969-12-31T23:59:59.000Z |
|
||||
1711843200000 | 2024-03-31T00:00:00.000Z |
|
||||
1711846799000 | 2024-03-31T00:59:59.000Z |
|
||||
1711846800000 | 2024-03-31T01:00:00.000Z |
|
||||
1711846801000 | 2024-03-31T01:00:01.000Z |
|
||||
Query OK, 5 row(s) in set (0.003339s)
|
||||
```
|
||||
|
||||
以下语句中第一个 SQL 查询截止时间不存在，第二个截止时间存在，第一个 SQL 查询结果不符合预期：
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 02:00:00';
|
||||
Query OK, 0 row(s) in set (0.000930s)
|
||||
|
||||
taos> SELECT ts, to_iso8601(ts,'Z') FROM t1 WHERE ts BETWEEN '2024-03-31 01:00:00' AND '2024-03-31 01:59:59';
|
||||
ts | to_iso8601(ts,'Z') |
|
||||
==================================================
|
||||
1711843200000 | 2024-03-31T00:00:00.000Z |
|
||||
1711846799000 | 2024-03-31T00:59:59.000Z |
|
||||
Query OK, 2 row(s) in set (0.001227s)
|
||||
```
|
||||
|
||||
- 夏令时结束时，跳变的时间（`[10-27 02:00:00,10-27 03:00:00)` 不包含 `10-27 03:00:00`）重复了两次，TDengine 在使用该区间内的时间戳进行查询时，也属于未定义行为。
|
||||
- 查询 `[2024-10-27 02:00:00, 2024-10-27 03:00:00]` 之间的数据结果，包含了两次重复的时间戳和 `2024-10-27 03:00:00` 这个时间点的数据：
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts BETWEEN '2024-10-27 02:00:00' AND '2024-10-27 03:00:00';
|
||||
ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
=======================================================================================
|
||||
1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
1729994400000 | 2024-10-27T02:00:00.000Z | 2024-10-27 03:00:00 |
|
||||
Query OK, 5 row(s) in set (0.001370s)
|
||||
````
|
||||
|
||||
- 但以下查询 [2024-10-27 02:00:00.000,2024-10-27 02:57:34.999] 区间只能查询到第一个2024-10-27 02:00:00 时间点的数据：
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:00.999';
|
||||
ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
=======================================================================================
|
||||
1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
Query OK, 1 row(s) in set (0.004480s)
|
||||
```
|
||||
|
||||
- 以下查询 `[2024-10-27 02:00:01,2024-10-27 02:57:35]` 却能查到 3 条数据（包含一条 02:59:59 的当地时间数据）：
|
||||
|
||||
```sql
|
||||
taos> SELECT ts, to_iso8601(ts,'Z'), TO_CHAR(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1 WHERE ts >= '2024-10-27 02:00:00' AND ts <= '2024-10-27 02:57:35';;
|
||||
ts | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
================================================================================================
|
||||
2024-10-27 02:00:00.000 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
2024-10-27 02:59:59.000 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
2024-10-27 02:00:00.000 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
Query OK, 3 row(s) in set (0.004428s)
|
||||
```
|
||||
|
||||
## 总结与建议
|
||||
|
||||
### 总结
|
||||
|
||||
仅针对使用当地时间带来的影响作说明，使用 UNIX 时间戳或 RFC3339 无影响。
|
||||
|
||||
- 写入：
|
||||
- 无法写入夏令时跳变时不存在的时间数据。
|
||||
- 写入夏令时跳变时重复的时间是未定义行为。
|
||||
- 查询：
|
||||
- 查询条件指定夏令时开始时跳变的时间，其查询结果为未定义行为。
|
||||
- 查询条件指定夏令时结束时重复的时间，其查询结果为未定义行为。
|
||||
- 显示：
|
||||
- 带时区显示不受影响。
|
||||
- 显示当地时间是准确的，但夏令时结束时重复的时间会无法区分。
|
||||
- 用户应谨慎使用不带时区的时间进行展示和应用。
|
||||
|
||||
### 建议
|
||||
|
||||
为避免夏令时给查询和写入造成不必要的影响，在 TDengine 中，建议使用明确的时间偏移量进行写入和查询。
|
||||
|
||||
- 使用 UNIX 时间戳：使用 UNIX 时间戳可避免时区问题。
|
||||
|
||||
| TIMESTAMP | UTC | Europe/Berlin | Local |
|
||||
| ------------: | :----------------------: | :---------------------------: | :-----------------: |
|
||||
| 1711846799000 | 2024-03-31T00:59:59.000Z | 2024-03-31T01:59:59.000+01:00 | 2024-03-31 01:59:59 |
|
||||
| 1711846800000 | 2024-03-31T01:00:00.000Z | 2024-03-31T03:00:00.000+02:00 | 2024-03-31 03:00:00 |
|
||||
|
||||
```sql
|
||||
taos> insert into t1 values(1711846799000, 1)(1711846800000, 2);
|
||||
Insert OK, 2 row(s) affected (0.001434s)
|
||||
|
||||
taos> select * from t1 where ts between 1711846799000 and 1711846800000;
|
||||
ts | v1 |
|
||||
===============================
|
||||
1711846799000 | 1 |
|
||||
1711846800000 | 2 |
|
||||
Query OK, 2 row(s) in set (0.003503s)
|
||||
```
|
||||
|
||||
- 使用 RFC3339 时间格式：带时区偏移量的 RFC3339 时间格式可以有效避免夏令时的不确定性。
|
||||
|
||||
| TIMESTAMP | UTC | Europe/Berlin | Local |
|
||||
| ------------: | :----------------------: | :---------------------------: | :-----------------: |
|
||||
| 1729987200000 | 2024-10-27T00:00:00.000Z | 2024-10-27T02:00:00.000+02:00 | 2024-10-27 02:00:00 |
|
||||
| 1729990799000 | 2024-10-27T00:59:59.000Z | 2024-10-27T02:59:59.000+02:00 | 2024-10-27 02:59:59 |
|
||||
| 1729990800000 | 2024-10-27T01:00:00.000Z | 2024-10-27T02:00:00.000+01:00 | 2024-10-27 02:00:00 |
|
||||
| 1729994399000 | 2024-10-27T01:59:59.000Z | 2024-10-27T02:59:59.000+01:00 | 2024-10-27 02:59:59 |
|
||||
|
||||
```sql
|
||||
taos> insert into t1 values ('2024-10-27T02:00:00.000+02:00', 1)
|
||||
('2024-10-27T02:59:59.000+02:00', 2)
|
||||
('2024-10-27T02:00:00.000+01:00', 3)
|
||||
('2024-10-27T02:59:59.000+01:00', 4);
|
||||
Insert OK, 4 row(s) affected (0.001514s)
|
||||
|
||||
taos> SELECT *,
|
||||
to_iso8601(ts,'Z'),
|
||||
to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1
|
||||
WHERE ts >= '2024-10-27T02:00:00.000+02:00'
|
||||
AND ts <= '2024-10-27T02:59:59.000+01:00';
|
||||
ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
=====================================================================================================
|
||||
1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
1729990800000 | 3 | 2024-10-27T01:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729994399000 | 4 | 2024-10-27T01:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
Query OK, 4 row(s) in set (0.004275s)
|
||||
|
||||
taos> SELECT *,
|
||||
to_iso8601(ts,'Z'),
|
||||
to_char(ts, 'YYYY-MM-DD HH:mi:ss') FROM t1
|
||||
WHERE ts >= '2024-10-27T02:00:00.000+02:00'
|
||||
AND ts <= '2024-10-27T02:59:59.000+02:00';
|
||||
ts | v1 | to_iso8601(ts,'Z') | to_char(ts, 'YYYY-MM-DD HH:mi:ss') |
|
||||
=====================================================================================================
|
||||
1729987200000 | 1 | 2024-10-27T00:00:00.000Z | 2024-10-27 02:00:00 |
|
||||
1729990799000 | 2 | 2024-10-27T00:59:59.000Z | 2024-10-27 02:59:59 |
|
||||
Query OK, 2 row(s) in set (0.004275s)
|
||||
```
|
||||
|
||||
- 查询时注意时区设定：在查询和显示时，如果需要本地时间，务必考虑夏令时的影响。
|
||||
- taosAdapter：使用 REST API 时，支持设置 IANA 时区，结果使用 RFC3339 格式返回。
|
||||
|
||||
```shell
|
||||
$ curl -uroot:taosdata 'localhost:6041/rest/sql?tz=Europe/Berlin'\
|
||||
-d "select ts from tz1.t1"
|
||||
{"code":0,"column_meta":[["ts","TIMESTAMP",8]],"data":[["1970-01-01T00:59:59.000+01:00"],["2024-03-31T01:00:00.000+01:00"],["2024-03-31T01:59:59.000+01:00"],["2024-03-31T03:00:00.000+02:00"],["2024-03-31T03:00:01.000+02:00"],["2024-10-27T02:00:00.000+02:00"],["2024-10-27T02:59:59.000+02:00"],["2024-10-27T02:00:00.000+01:00"],["2024-10-27T02:59:59.000+01:00"],["2024-10-27T03:00:00.000+01:00"]],"rows":10}
|
||||
```
|
||||
|
||||
- Explorer：使用 Explorer 页面进行 SQL 查询时，用户可配置客户端时区，以 RFC3339 格式显示。
|
||||
|
||||
|
||||
|
||||
## 参考文档
|
||||
|
||||
- IANA Time Zone Database: [https://www.iana.org/time-zones](https://www.iana.org/time-zones)
|
||||
- RFC3339: [https://datatracker.ietf.org/doc/html/rfc3339](https://datatracker.ietf.org/doc/html/rfc3339)
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 80 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 72 KiB |
|
|
@ -920,12 +920,6 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask
|
|||
|
||||
// now the fill-history task starts to scan data from wal files.
|
||||
code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
|
||||
// if (code == TSDB_CODE_SUCCESS) {
|
||||
// code = tqScanWalAsync(pTq, false);
|
||||
// if (code) {
|
||||
// tqError("vgId:%d failed to start scan wal file, code:%s", vgId, tstrerror(code));
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -148,6 +148,7 @@ static void doStartScanWal(void* param, void* tmrId) {
|
|||
return;
|
||||
}
|
||||
|
||||
// failed to lock, try 500ms later
|
||||
code = streamMetaTryRlock(pMeta);
|
||||
if (code == 0) {
|
||||
numOfTasks = taosArrayGetSize(pMeta->pTaskList);
|
||||
|
|
@ -156,16 +157,13 @@ static void doStartScanWal(void* param, void* tmrId) {
|
|||
numOfTasks = 0;
|
||||
}
|
||||
|
||||
if (numOfTasks == 0) {
|
||||
goto _end;
|
||||
}
|
||||
if (numOfTasks > 0) {
|
||||
tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d", vgId, numOfTasks);
|
||||
|
||||
tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d", vgId, numOfTasks);
|
||||
|
||||
#if 0
|
||||
#if 0
|
||||
// wait for the vnode is freed, and invalid read may occur.
|
||||
taosMsleep(10000);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
code = streamTaskSchedTask(&pParam->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA, false);
|
||||
if (code) {
|
||||
|
|
@ -174,7 +172,7 @@ static void doStartScanWal(void* param, void* tmrId) {
|
|||
|
||||
_end:
|
||||
streamTmrStart(doStartScanWal, SCAN_WAL_IDLE_DURATION, pParam, pTimer, &pMeta->scanInfo.scanTimer, vgId, "scan-wal");
|
||||
tqDebug("vgId:%d scan-wal will start in %dms", vgId, SCAN_WAL_IDLE_DURATION*SCAN_WAL_WAIT_COUNT);
|
||||
tqDebug("vgId:%d try scan-wal will start in %dms", vgId, SCAN_WAL_IDLE_DURATION*SCAN_WAL_WAIT_COUNT);
|
||||
|
||||
code = taosReleaseRef(streamMetaRefPool, pParam->metaId);
|
||||
if (code) {
|
||||
|
|
@ -192,7 +190,7 @@ void tqScanWalAsync(STQ* pTq) {
|
|||
|
||||
// 1. the vnode should be the leader.
|
||||
// 2. the stream isn't disabled
|
||||
if ((pMeta->role == NODE_ROLE_FOLLOWER) || tsDisableStream) {
|
||||
if ((pMeta->role != NODE_ROLE_LEADER) || tsDisableStream) {
|
||||
tqInfo("vgId:%d follower node or stream disabled, not scan wal", vgId);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -952,11 +952,6 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
|
|||
|
||||
streamMetaWUnLock(pMeta);
|
||||
|
||||
// if (scanWal && (vgId != SNODE_HANDLE)) {
|
||||
// tqDebug("vgId:%d start scan wal for executing tasks", vgId);
|
||||
// code = tqScanWalAsync(pMeta->ahandle, true);
|
||||
// }
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -610,11 +610,12 @@ static int32_t tsdbTryAcquireReader(STsdbReader* pReader) {
|
|||
|
||||
code = taosThreadMutexTryLock(&pReader->readerMutex);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tsdbError("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
|
||||
// Failing to acquire the lock is reasonable, not an error
|
||||
tsdbWarn("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
|
||||
} else {
|
||||
tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex);
|
||||
}
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
return code;
|
||||
|
||||
_end:
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
|
|
|
|||
|
|
@ -797,7 +797,7 @@ static bool funcNotSupportStringSma(SFunctionNode* pFunc) {
|
|||
}
|
||||
|
||||
EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
|
||||
if(funcNotSupportStringSma(pFunc)) {
|
||||
if (funcNotSupportStringSma(pFunc)) {
|
||||
return FUNC_DATA_REQUIRED_DATA_LOAD;
|
||||
}
|
||||
return FUNC_DATA_REQUIRED_SMA_LOAD;
|
||||
|
|
@ -6618,7 +6618,7 @@ int32_t blockDBUsageFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
double compressRadio = 0;
|
||||
if (rawDataSize != 0) {
|
||||
compressRadio = totalDiskSize * 100 / (double)rawDataSize;
|
||||
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f]", compressRadio);
|
||||
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f%]", compressRadio);
|
||||
} else {
|
||||
len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[NULL]");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -532,9 +532,17 @@ int32_t sifStr2Num(char *buf, int32_t len, int8_t type, void *val) {
|
|||
static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) {
|
||||
int32_t code = 0;
|
||||
int8_t ltype = left->colValType, rtype = right->colValType;
|
||||
if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
// if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) {
|
||||
// return TSDB_CODE_INVALID_PARA;
|
||||
// }
|
||||
if (IS_VAR_DATA_TYPE(ltype)) {
|
||||
if (ltype == TSDB_DATA_TYPE_VARCHAR || ltype == TSDB_DATA_TYPE_BINARY || ltype == TSDB_DATA_TYPE_VARBINARY) {
|
||||
return 0;
|
||||
} else {
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
}
|
||||
|
||||
if (ltype == TSDB_DATA_TYPE_FLOAT) {
|
||||
float f = 0;
|
||||
if (IS_NUMERIC_TYPE(rtype)) {
|
||||
|
|
|
|||
|
|
@ -814,15 +814,17 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l
|
|||
}
|
||||
|
||||
int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t dbRefId, ECHECKPOINT_BACKUP_TYPE type) {
|
||||
int32_t code = 0;
|
||||
char* path = NULL;
|
||||
|
||||
int32_t code = 0;
|
||||
char* path = NULL;
|
||||
int64_t chkptSize = 0;
|
||||
SStreamMeta* pMeta = pTask->pMeta;
|
||||
const char* idStr = pTask->id.idStr;
|
||||
int64_t now = taosGetTimestampMs();
|
||||
|
||||
SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES);
|
||||
if (toDelFiles == NULL) {
|
||||
stError("s-task:%s failed to prepare array list during upload checkpoint, code:%s", pTask->id.idStr,
|
||||
tstrerror(terrno));
|
||||
return terrno;
|
||||
}
|
||||
|
||||
|
|
@ -848,11 +850,11 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d
|
|||
}
|
||||
}
|
||||
|
||||
if (code == TSDB_CODE_SUCCESS) {
|
||||
int32_t size = taosArrayGetSize(toDelFiles);
|
||||
stDebug("s-task:%s remove redundant %d files", idStr, size);
|
||||
int32_t num = taosArrayGetSize(toDelFiles);
|
||||
if (code == TSDB_CODE_SUCCESS && num > 0) {
|
||||
stDebug("s-task:%s remove redundant %d files", idStr, num);
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
for (int i = 0; i < num; i++) {
|
||||
char* pName = taosArrayGetP(toDelFiles, i);
|
||||
code = deleteCheckpointFile(idStr, pName);
|
||||
if (code != 0) {
|
||||
|
|
@ -868,12 +870,13 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d
|
|||
double el = (taosGetTimestampMs() - now) / 1000.0;
|
||||
|
||||
if (code == TSDB_CODE_SUCCESS) {
|
||||
stDebug("s-task:%s complete update checkpointId:%" PRId64 ", elapsed time:%.2fs remove local checkpoint data %s",
|
||||
idStr, checkpointId, el, path);
|
||||
taosRemoveDir(path);
|
||||
code = taosGetDirSize(path, &chkptSize);
|
||||
stDebug("s-task:%s complete upload checkpointId:%" PRId64
|
||||
", elapsed time:%.2fs, checkpointSize:%.2fKiB local dir:%s",
|
||||
idStr, checkpointId, el, SIZE_IN_KiB(chkptSize), path);
|
||||
} else {
|
||||
stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " keep local checkpoint data, elapsed time:%.2fs", idStr,
|
||||
checkpointId, el);
|
||||
stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " elapsed time:%.2fs, checkpointSize:%.2fKiB", idStr,
|
||||
checkpointId, el, SIZE_IN_KiB(chkptSize));
|
||||
}
|
||||
|
||||
taosMemoryFree(path);
|
||||
|
|
@ -883,7 +886,7 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d
|
|||
int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t checkpointId) {
|
||||
ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType();
|
||||
if (type == DATA_UPLOAD_DISABLE) {
|
||||
stDebug("s-task:%s not allowed to upload checkpoint data", pTask->id.idStr);
|
||||
stDebug("s-task:%s not config to backup checkpoint data at snode, checkpointId:%"PRId64, pTask->id.idStr, checkpointId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -925,6 +928,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
|
|||
if (code != TSDB_CODE_SUCCESS) {
|
||||
stError("s-task:%s gen checkpoint:%" PRId64 " failed, code:%s", id, ckId, tstrerror(terrno));
|
||||
}
|
||||
|
||||
int64_t et = taosGetTimestampMs();
|
||||
stDebug("s-task:%s gen local checkpoint completed, elapsed time:%.2fs", id, (et - startTs) / 1000.0);
|
||||
}
|
||||
|
||||
// TODO: monitoring the checkpoint-source msg
|
||||
|
|
|
|||
|
|
@ -243,6 +243,8 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
|
|||
continue;
|
||||
}
|
||||
|
||||
// todo: this lock may blocked by lock in streamMetaStartOneTask function, which may lock a very long time when
|
||||
// trying to load remote checkpoint data
|
||||
streamMutexLock(&pTask->lock);
|
||||
STaskStatusEntry entry = streamTaskGetStatusEntry(pTask);
|
||||
streamMutexUnlock(&pTask->lock);
|
||||
|
|
|
|||
|
|
@ -1362,6 +1362,10 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader)
|
|||
}
|
||||
|
||||
pMeta->role = (isLeader) ? NODE_ROLE_LEADER : NODE_ROLE_FOLLOWER;
|
||||
if (!isLeader) {
|
||||
streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId);
|
||||
}
|
||||
|
||||
streamMetaWUnLock(pMeta);
|
||||
|
||||
if (isLeader) {
|
||||
|
|
|
|||
|
|
@ -190,6 +190,8 @@ int32_t mpChunkNSAllocMem(SMemPool* pPool, SMPSession* pSession, int64_t size, u
|
|||
void* pRes = NULL;
|
||||
int64_t totalSize = size + sizeof(SMPMemHeader) + sizeof(SMPMemTailer) + alignment;
|
||||
|
||||
|
||||
|
||||
MP_ERR_JRET(mpChunkNewNS(pPool, &pChunk, totalSize));
|
||||
SMPMemHeader* pHeader = (SMPMemHeader*)pChunk->pMemStart;
|
||||
MP_INIT_MEM_HEADER(pHeader, size, false);
|
||||
|
|
|
|||
|
|
@ -102,7 +102,8 @@ class TDTestCase:
|
|||
elif "Compress_radio=" in item[0]:
|
||||
value = item[0].split("=")[1].split(" ")[0].replace("[", "").replace("]", "")
|
||||
if value != 'NULL':
|
||||
compress_radio = float(value)
|
||||
tValue = value[0:len(value) - 1]
|
||||
compress_radio = float(tValue)
|
||||
#tdLog.debug("compress_occupied: %s" % compress_radio)
|
||||
return disk_occupied, compress_radio
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue