From 7d218e3891ca02bda9a2f118c2b0136a010f585b Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 12 Mar 2025 08:15:14 +0800 Subject: [PATCH 01/44] ci: Update workflow to fix param issue of run_tdgpt_test --- .github/workflows/taosd-ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 31eda597ad..209a62c1a5 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -31,9 +31,18 @@ jobs: run: | set -euo pipefail target_branch=${{ github.event.pull_request.base.ref }} + + # Fetch the latest code from the target branch + cd ${{ env.WKC }} + git reset --hard + git clean -f + git remote prune origin + git fetch + git checkout "$target_branch" + git remote prune origin + git pull >/dev/null # Check whether to run tdgpt test cases - cd ${{ env.WKC }} changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) echo "changed files exclude doc: ${changed_files_non_doc}" From 8fbd28eb6e69685b359a8212aa4a1142d850ca70 Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Wed, 12 Mar 2025 14:33:18 +0800 Subject: [PATCH 02/44] fix: state col precal --- source/libs/executor/src/timewindowoperator.c | 7 +++++++ source/libs/planner/src/planPhysiCreater.c | 12 +++++++++-- tests/system-test/2-query/state_window.py | 20 +++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 71c71a547e..28cc911869 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -17,7 +17,9 @@ #include "function.h" #include "functionMgt.h" #include "operator.h" +#include "query.h" #include "querytask.h" +#include "taoserror.h" #include "tchecksum.h" #include "tcommon.h" #include "tcompare.h" @@ -1016,6 +1018,11 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) { continue; } + if (pStateColInfoData->pData == NULL) { + qError("%s:%d state column data is null", __FILE__, __LINE__); + pTaskInfo->code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR); + } char* val = colDataGetData(pStateColInfoData, j); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 9513e90c50..504f5d5bae 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2196,9 +2196,17 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); // push down expression to pOutputDataBlockDesc of child node if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { - code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pWindow->pExprs); + SNodeList* pOutput; + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pOutput); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlots(pCxt, pWindow->pExprs, pChildTupe); + code = addDataBlockSlots(pCxt, pOutput, pChildTupe); + } + if (TSDB_CODE_SUCCESS == code) { + if (pWindow->pExprs == NULL) { + pWindow->pExprs = pOutput; + } else { + code = nodesListAppendList(pWindow->pExprs, pOutput); + } } } diff --git a/tests/system-test/2-query/state_window.py b/tests/system-test/2-query/state_window.py index 7dedeb88f1..0b5427c234 100644 --- a/tests/system-test/2-query/state_window.py +++ b/tests/system-test/2-query/state_window.py @@ -203,7 +203,27 @@ class TDTestCase: tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) tdSql.query("select first(c2) from t0 session(ts, 1s) order by ts", queryTimes=1) + def ts6079(self): + ts = 1741757485230 + tdSql.execute("drop database if exists ts6079") + tdSql.execute("create database ts6079 vgroups 2 replica 1") + tdSql.execute("CREATE STABLE ts6079.`meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` VARCHAR(24))") + for tableIndex in range(10): + tdSql.execute(f"CREATE TABLE ts6079.t{tableIndex} USING ts6079.meters TAGS ({tableIndex}, 'tb{tableIndex}')") + for num in range(10): + tdSql.execute(f"INSERT INTO ts6079.t{tableIndex} VALUES({ts + num}, {num * 1.0}, {215 + num}, 0.0)") + + tdSql.query("select _wstart ,first(ts),last(ts),count(*),to_char(ts, 'yyyymmdd') as ts from ts6079.meters partition by to_char(ts, 'yyyymmdd') as ts state_window(cast(current as varchar(2)));") + tdSql.checkRows(10) + tdSql.checkData(0, 3, 10) + + tdSql.query("select _wstart ,first(ts),last(ts),count(*),to_char(ts,'yyyymmdd') as ts from ts6079.meters partition by to_char(ts,'yyyymmdd') as ts interval(5);") + tdSql.checkRows(2) + tdSql.checkData(0, 3, 50) + + def run(self): + self.ts6079() self.test_crash_for_session_window() self.test_crash_for_state_window1() self.test_crash_for_state_window2() From ff85dc876c3f53f13f36cef72bb80d565061436b Mon Sep 17 00:00:00 2001 From: facetosea <285808407@qq.com> Date: Wed, 12 Mar 2025 15:10:19 +0800 Subject: [PATCH 03/44] fix: test case --- tests/system-test/2-query/state_window.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/system-test/2-query/state_window.py b/tests/system-test/2-query/state_window.py index 0b5427c234..ab854f6deb 100644 --- a/tests/system-test/2-query/state_window.py +++ b/tests/system-test/2-query/state_window.py @@ -216,10 +216,6 @@ class TDTestCase: tdSql.query("select _wstart ,first(ts),last(ts),count(*),to_char(ts, 'yyyymmdd') as ts from ts6079.meters partition by to_char(ts, 'yyyymmdd') as ts state_window(cast(current as varchar(2)));") tdSql.checkRows(10) tdSql.checkData(0, 3, 10) - - tdSql.query("select _wstart ,first(ts),last(ts),count(*),to_char(ts,'yyyymmdd') as ts from ts6079.meters partition by to_char(ts,'yyyymmdd') as ts interval(5);") - tdSql.checkRows(2) - tdSql.checkData(0, 3, 50) def run(self): From b20b200157344acb8b5fca88eb6afeb4bb11efc7 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Wed, 12 Mar 2025 15:53:59 +0800 Subject: [PATCH 04/44] ci: modify desc in taosd-ci-build.yml --- .github/workflows/taosd-ci-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 2179200bb3..a9d429a95e 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -21,7 +21,7 @@ concurrency: jobs: build: - name: Build and test on ${{ matrix.os }} + name: Run on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: From e6a509a30979816d6bd38696c10383f8c1473516 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Wed, 12 Mar 2025 16:04:25 +0800 Subject: [PATCH 05/44] ci: modify des --- .github/workflows/taosd-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 31eda597ad..a14a32fedb 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -1,4 +1,4 @@ -name: TDengine CI Test +name: TDengine Test on: pull_request: From 467bb6d46da615600cda2cb24b83b3249063921d Mon Sep 17 00:00:00 2001 From: haoranchen Date: Wed, 12 Mar 2025 16:17:42 +0800 Subject: [PATCH 06/44] ci: add name for fetch parameters --- .github/workflows/taosd-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index a14a32fedb..d11b363679 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -19,6 +19,7 @@ env: jobs: fetch-parameters: + name: Prepare Test runs-on: group: CI labels: [self-hosted, Linux, X64, testing] From 794587cbccbcf58dcf4888812e13de2a2c7bba39 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 12 Mar 2025 17:32:29 +0800 Subject: [PATCH 07/44] ci: renamed test workflows yaml to workflows --- .github/workflows/{taosd-ci-build.yml => tdengine-build.yml} | 0 .github/workflows/{taosd-ci.yml => tdengine-ci-test.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{taosd-ci-build.yml => tdengine-build.yml} (100%) rename .github/workflows/{taosd-ci.yml => tdengine-ci-test.yml} (100%) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/tdengine-build.yml similarity index 100% rename from .github/workflows/taosd-ci-build.yml rename to .github/workflows/tdengine-build.yml diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/tdengine-ci-test.yml similarity index 100% rename from .github/workflows/taosd-ci.yml rename to .github/workflows/tdengine-ci-test.yml From 51b259b87fec37e6f3857c84437396fe616c821c Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 12 Mar 2025 16:48:07 +0800 Subject: [PATCH 08/44] ci: update ci workflow to fix param issue --- .github/workflows/taosd-ci.yml | 110 +++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 40 deletions(-) diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 209a62c1a5..8145bbce84 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -9,9 +9,26 @@ on: paths-ignore: - 'packaging/**' - 'docs/**' + workflow_dispatch: + inputs: + specified_source_branch: + description: 'Enter the source branch name of TDengine' + required: false + default: 'unavailable' + type: string + specified_target_branch: + description: 'Enter the target branch name of TDengine' + required: false + default: 'unavailable' + type: string + specified_pr_number: + description: 'Enter the PR number of TDengine' + required: false + default: 'unavailable' + type: string concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-TDengine + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.specified_target_branch }}-${{ github.event_name == 'workflow_dispatch' }}-TDengine cancel-in-progress: true env: @@ -30,45 +47,49 @@ jobs: id: parameters run: | set -euo pipefail - target_branch=${{ github.event.pull_request.base.ref }} - - # Fetch the latest code from the target branch - cd ${{ env.WKC }} - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "$target_branch" - git remote prune origin - git pull >/dev/null - - # Check whether to run tdgpt test cases - changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) - echo "changed files exclude doc: ${changed_files_non_doc}" - - if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then - run_tdgpt_test="true" - else - run_tdgpt_test="false" - fi - echo "run tdgpt test: ${run_tdgpt_test}" - - # Check whether to run function test cases - changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ - grep -v "^docs/en/" | \ - grep -v "^docs/zh/" | \ - grep -v ".md$" | \ - grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ - tr '\n' ' ' || :) - echo "changed files exclude tdgpt: ${changed_files_non_tdgpt}" - - if [ -n "$changed_files_non_tdgpt" ]; then + if ${{ github.event_name == 'wrokflow_dispatch' }}; then run_function_test="true" + run_tdgpt_test="false" else - run_function_test="false" - fi + target_branch=${{ github.event.pull_request.base.ref }} - echo "run function test: ${run_function_test}" + # Fetch the latest code from the target branch + cd ${{ env.WKC }} + git reset --hard + git clean -f + git remote prune origin + git fetch + git checkout "$target_branch" + git remote prune origin + git pull >/dev/null + + # Check whether to run tdgpt test cases + changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) + echo "changed files exclude doc: ${changed_files_non_doc}" + + if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then + run_tdgpt_test="true" + else + run_tdgpt_test="false" + fi + echo "run tdgpt test: ${run_tdgpt_test}" + + # Check whether to run function test cases + changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ + grep -v "^docs/en/" | \ + grep -v "^docs/zh/" | \ + grep -v ".md$" | \ + grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ + tr '\n' ' ' || :) + echo "changed files exclude tdgpt: ${changed_files_non_tdgpt}" + + if [ -n "$changed_files_non_tdgpt" ]; then + run_function_test="true" + else + run_function_test="false" + fi + + echo "run function test: ${run_function_test}" # Output the results for GitHub Actions echo "run_function_test=$run_function_test" >> $GITHUB_OUTPUT @@ -79,24 +100,33 @@ jobs: echo ${{ github.event.pull_request.number }} run-tests-on-linux: - uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@main + uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@ci/test/workflow needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false run_function_test: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} run_tdgpt_test: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' }} + specified_source_branch: ${{ github.event.inputs.specified_source_branch }} + specified_target_branch: ${{ github.event.inputs.specified_target_branch }} + specified_pr_number: ${{ github.event.inputs.specified_pr_number }} run-tests-on-mac: - uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml@main + uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml@ci/test/workflow needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false + specified_source_branch: ${{ github.event.inputs.specified_source_branch }} + specified_target_branch: ${{ github.event.inputs.specified_target_branch }} + specified_pr_number: ${{ github.event.inputs.specified_pr_number }} run-tests-on-windows: - uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml@main + uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml@ci/test/workflow needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false + specified_source_branch: ${{ github.event.inputs.specified_source_branch }} + specified_target_branch: ${{ github.event.inputs.specified_target_branch }} + specified_pr_number: ${{ github.event.inputs.specified_pr_number }} From 773371facfaee116374d0c21ff9d3a9767d75444 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 12 Mar 2025 17:42:52 +0800 Subject: [PATCH 09/44] ci: fix workflow syntxt issue --- .github/workflows/taosd-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 8145bbce84..0bab6662d0 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -90,6 +90,7 @@ jobs: fi echo "run function test: ${run_function_test}" + fi # Output the results for GitHub Actions echo "run_function_test=$run_function_test" >> $GITHUB_OUTPUT From 04a073db628d4db2e4c7da5742ed341c7dac2252 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 12 Mar 2025 18:02:18 +0800 Subject: [PATCH 10/44] ci: fix spelling issue of workflow_dispatch --- .github/workflows/taosd-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 0bab6662d0..71b00dc2d1 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -47,7 +47,7 @@ jobs: id: parameters run: | set -euo pipefail - if ${{ github.event_name == 'wrokflow_dispatch' }}; then + if ${{ github.event_name == 'workflow_dispatch' }}; then run_function_test="true" run_tdgpt_test="false" else From 3f846984c233fb40a7e848209e9b4b41a0de8cab Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 12 Mar 2025 18:03:05 +0800 Subject: [PATCH 11/44] feat: add batch query new feature --- .../14-reference/02-tools/10-taosbenchmark.md | 27 +- .../14-reference/02-tools/10-taosbenchmark.md | 33 ++- .../basic/json/queryErrorBatchNoMix.json | 36 +++ .../basic/json/queryErrorBatchRest.json | 36 +++ .../basic/json/queryModeSpecMixBatch.json | 36 +++ .../basic/json/queryModeSpecMixBatchRest.json | 36 +++ tests/army/tools/benchmark/basic/queryMain.py | 43 ++- tools/taos-tools/inc/bench.h | 1 + tools/taos-tools/src/benchJsonOpt.c | 17 ++ tools/taos-tools/src/benchMain.c | 29 ++- tools/taos-tools/src/benchQuery.c | 245 ++++++++++++++++-- 11 files changed, 495 insertions(+), 44 deletions(-) create mode 100644 tests/army/tools/benchmark/basic/json/queryErrorBatchNoMix.json create mode 100644 tests/army/tools/benchmark/basic/json/queryErrorBatchRest.json create mode 100644 tests/army/tools/benchmark/basic/json/queryModeSpecMixBatch.json create mode 100644 tests/army/tools/benchmark/basic/json/queryModeSpecMixBatchRest.json diff --git a/docs/en/14-reference/02-tools/10-taosbenchmark.md b/docs/en/14-reference/02-tools/10-taosbenchmark.md index cfc92b4e0b..c85893b51b 100644 --- a/docs/en/14-reference/02-tools/10-taosbenchmark.md +++ b/docs/en/14-reference/02-tools/10-taosbenchmark.md @@ -347,10 +347,14 @@ Specify the configuration parameters for tag and data columns in `super_tables` ### Query Parameters -In query scenarios, `filetype` must be set to `query`. +`filetype` must be set to `query`. + +`query_mode` connect method: +- "taosc": Native. +- "rest" : RESTful. + `query_times` specifies the number of times to run the query, numeric type. -Query scenarios can control the execution of slow query statements by setting `kill_slow_query_threshold` and `kill_slow_query_interval` parameters, where threshold controls that queries exceeding the specified exec_usec time will be killed by taosBenchmark, in seconds; interval controls the sleep time to avoid continuous slow query CPU consumption, in seconds. For other common parameters, see [General Configuration Parameters](#general-configuration-parameters) @@ -358,13 +362,26 @@ For other common parameters, see [General Configuration Parameters](#general-con Configuration parameters for querying specified tables (can specify supertables, subtables, or regular tables) are set in `specified_table_query`. - **mixed_query** : Query Mode . "yes" is `Mixed Query`, "no" is `General Query`, default is "no". - `General Query`: + `General Query`: Each SQL in `sqls` starts `threads` threads to query this SQL, Each thread exits after executing the `query_times` queries, and only after all threads executing this SQL have completed can the next SQL be executed. The total number of queries(`General Query`) = the number of `sqls` * `query_times` * `threads` - `Mixed Query`: + `Mixed Query`: All SQL statements in `sqls` are divided into `threads` groups, with each thread executing one group. Each SQL statement needs to execute `query_times` queries. - The total number of queries(`Mixed Query`) = the number of `sqls` * `query_times` + The total number of queries(`Mixed Query`) = the number of `sqls` * `query_times`. + +- **batch_query** : Batch query power switch. +"yes": indicates that it is enabled. +"no": indicates that it is not enabled, and other values report errors. +Batch query refers to dividing all SQL statements in SQL into `threads` groups, with each thread executing one group. +Each SQL statement is queried only once before exiting, and the main thread waits for all threads to complete before determining if the `query_interval` parameter is set. If sleep is required for a specified time, each thread group is restarted and the previous process is repeated until the number of queries is exhausted. +Functional limitations: +- Only supports scenarios where `mixed_query` is set to 'yes'. +- Restful queries are not supported, meaning `query_made` cannot be 'rest'. + - **query_interval** : Query interval, in millisecond, default is 0. +When the 'batch_query' switch is turned on, it indicates the interval time after each batch query is completed, When closed, it indicates the interval time between each SQL query completion. +If the execution time of the query exceeds the interval time, it will no longer wait. If the execution time of the query is less than the interval time, it is necessary to wait to make up for the interval time. + - **threads** : Number of threads executing the SQL query, default is 1. - **sqls**: - **sql**: The SQL command to execute, required. diff --git a/docs/zh/14-reference/02-tools/10-taosbenchmark.md b/docs/zh/14-reference/02-tools/10-taosbenchmark.md index 89e65d5358..a596d8e151 100644 --- a/docs/zh/14-reference/02-tools/10-taosbenchmark.md +++ b/docs/zh/14-reference/02-tools/10-taosbenchmark.md @@ -253,27 +253,44 @@ taosBenchmark -f ### 查询配置参数 查询场景下 `filetype` 必须设置为 `query`。 + +`query_mode` 查询连接方式，取值为： + - “taosc”: 通过 Native 连接方式查询。 + - “rest” : 通过 restful 连接方式查询。 + `query_times` 指定运行查询的次数，数值类型。 -查询场景可以通过设置 `kill_slow_query_threshold` 和 `kill_slow_query_interval` 参数来控制杀掉慢查询语句的执行，threshold 控制如果 exec_usec 超过指定时间的查询将被 taosBenchmark 杀掉，单位为秒。 -interval 控制休眠时间，避免持续查询慢查询消耗 CPU，单位为秒。 -其它通用参数详见 [通用配置参数](#通用配置参数) +其它通用参数详见 [通用配置参数](#通用配置参数)。 #### 执行指定查询语句 查询指定表（可以指定超级表、子表或普通表）的配置参数在 `specified_table_query` 中设置。 -- **mixed_query**：查询模式 - “yes”：`混合查询` - "no"(默认值)：`普通查询` - `普通查询`：`sqls` 中每个 sql 启动 `threads` 个线程查询此 sql, 执行完 `query_times` 次查询后退出，执行此 sql 的所有线程都完成后进入下一个 sql +- **mixed_query**：混合查询开关。 + “yes”: 开启 “混合查询”。 + “no” : 关闭 “混合查询” ，即 “普通查询”。 + + - 普通查询： + + `sqls` 中每个 sql 启动 `threads` 个线程查询此 sql, 执行完 `query_times` 次查询后退出，执行此 sql 的所有线程都完成后进入下一个 sql `查询总次数` = `sqls` 个数 * `query_times` * `threads` - `混合查询`：`sqls` 中所有 sql 分成 `threads` 个组，每个线程执行一组， 每个 sql 都需执行 `query_times` 次查询 + - 混合查询： + + `sqls` 中所有 sql 分成 `threads` 个组，每个线程执行一组， 每个 sql 都需执行 `query_times` 次查询 `查询总次数` = `sqls` 个数 * `query_times` +- **batch_query**：批查询功开关。 + 取值范围 “yes” 表示开启，"no" 不开启，其它值报错。 + 批查询是指 `sqls` 中所有 sql 分成 `threads` 个组，每个线程执行一组，每个 sql 只执行一次查询后退出，主线程等待所有线程都执行完，再判断是否设置有 `query_interval` 参数，如果有需要 sleep 指定时间，再启动各线程组重复前面的过程，直到查询次数耗尽为止。 + 功能限制条件： + - 只支持 `mixed_query` 为 "yes" 的场景。 + - 不支持 restful 查询，即 `query_mode` 不能为 "rest"。 + - **query_interval**：查询时间间隔，单位：millisecond，默认值为 0。 + "batch_query" 开关打开时，表示是每批查询完间隔时间；关闭时，表示每个 sql 查询完间隔时间 + 如果执行查询的时间超过间隔时间，那么将不再等待，如果执行查询的时间不足间隔时间，需等待补足间隔时间 - **threads**：执行查询 SQL 的线程数，默认值为 1。 diff --git a/tests/army/tools/benchmark/basic/json/queryErrorBatchNoMix.json b/tests/army/tools/benchmark/basic/json/queryErrorBatchNoMix.json new file mode 100644 index 0000000000..97b542b789 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/queryErrorBatchNoMix.json @@ -0,0 +1,36 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 5, + "query_mode": "taosc", + "specified_table_query": { + "concurrent": 5, + "query_interval": 0, + "mixed_query": "no", + "batch_query": "yes", + "sqls": [ + { + "sql": "select last_row(*) from test.meters" + }, + { + "sql": "select count(*) from test.meters" + }, + { + "sql": "select * from test.d0", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from test.d1" + }, + { + "sql": "select * from test.d2" + } + ] + } +} diff --git a/tests/army/tools/benchmark/basic/json/queryErrorBatchRest.json b/tests/army/tools/benchmark/basic/json/queryErrorBatchRest.json new file mode 100644 index 0000000000..d47ce55ae4 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/queryErrorBatchRest.json @@ -0,0 +1,36 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 5, + "query_mode": "rest", + "specified_table_query": { + "concurrent": 5, + "query_interval": 1000, + "mixed_query": "yes", + "batch_query": "yes", + "sqls": [ + { + "sql": "select last_row(*) from test.meters" + }, + { + "sql": "select count(*) from test.meters" + }, + { + "sql": "select * from test.d0", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from test.d1" + }, + { + "sql": "select * from test.d2" + } + ] + } +} diff --git a/tests/army/tools/benchmark/basic/json/queryModeSpecMixBatch.json b/tests/army/tools/benchmark/basic/json/queryModeSpecMixBatch.json new file mode 100644 index 0000000000..6740c585c3 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/queryModeSpecMixBatch.json @@ -0,0 +1,36 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 5, + "query_mode": "taosc", + "specified_table_query": { + "concurrent": 5, + "query_interval": 1000, + "mixed_query": "yes", + "batch_query": "yes", + "sqls": [ + { + "sql": "select last_row(*) from test.meters" + }, + { + "sql": "select count(*) from test.meters" + }, + { + "sql": "select * from test.d0", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from test.d1" + }, + { + "sql": "select * from test.d2" + } + ] + } +} diff --git a/tests/army/tools/benchmark/basic/json/queryModeSpecMixBatchRest.json b/tests/army/tools/benchmark/basic/json/queryModeSpecMixBatchRest.json new file mode 100644 index 0000000000..b2ffbe1434 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/queryModeSpecMixBatchRest.json @@ -0,0 +1,36 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 5, + "query_mode": "taosc", + "specified_table_query": { + "concurrent": 5, + "query_interval": 100, + "mixed_query": "yes", + "batch_query": "no", + "sqls": [ + { + "sql": "select last_row(*) from test.meters" + }, + { + "sql": "select count(*) from test.meters" + }, + { + "sql": "select * from test.d0", + "result": "./query_res1.txt" + }, + { + "sql": "select count(*) from test.d1" + }, + { + "sql": "select * from test.d2" + } + ] + } +} diff --git a/tests/army/tools/benchmark/basic/queryMain.py b/tests/army/tools/benchmark/basic/queryMain.py index 1d52163e31..f79ab0ef11 100644 --- a/tests/army/tools/benchmark/basic/queryMain.py +++ b/tests/army/tools/benchmark/basic/queryMain.py @@ -40,7 +40,7 @@ def removeQuotation(origin): class TDTestCase(TBase): def caseDescription(self): """ - [TD-11510] taosBenchmark test cases + taosBenchmark query->Basic test cases """ def runSeconds(self, command, timeout = 180): @@ -99,15 +99,15 @@ class TDTestCase(TBase): fval = float(value) # compare if equal and fval != expect: - tdLog.exit(f"check not expect. expect:{expect} real:{fval}, key:{key} end:{end} output:\n{output}") + tdLog.exit(f"check not expect. expect:{expect} real:{fval}, key:'{key}' end:'{end}' output:\n{output}") elif equal == False and fval <= expect: - tdLog.exit(f"failed because {fval} <= {expect}, key:{key} end:{end} output:\n{output}") + tdLog.exit(f"failed because {fval} <= {expect}, key:'{key}' end:'{end}' output:\n{output}") else: # succ if equal: - tdLog.info(f"check successfully. key:{key} expect:{expect} real:{fval}") + tdLog.info(f"check successfully. key:'{key}' expect:{expect} real:{fval}") else: - tdLog.info(f"check successfully. key:{key} {fval} > {expect}") + tdLog.info(f"check successfully. key:'{key}' {fval} > {expect}") def checkAfterRun(self, benchmark, jsonFile, specMode, tbCnt): @@ -137,16 +137,24 @@ class TDTestCase(TBase): sqls = data[label]["sqls"] - # mix + # batch_query + try: + batchQuery = data[label]["batch_query"] + except: + batchQuery = "no" + + # mixed_query try: mixedQuery = data[label]["mixed_query"] except: mixedQuery = "no" - tdLog.info(f"queryTimes={queryTimes} concurrent={concurrent} mixedQuery={mixedQuery} len(sqls)={len(sqls)} label={label}\n") + tdLog.info(f"queryTimes={queryTimes} concurrent={concurrent} mixedQuery={mixedQuery} " + f"batchQuery={batchQuery} len(sqls)={len(sqls)} label={label}\n") - totalQueries = 0 + totalQueries = 0 threadQueries = 0 + QPS = 10 if continueIfFail.lower() == "yes": allEnd = " " @@ -163,13 +171,20 @@ class TDTestCase(TBase): minKey = "min:" else: # spec mixed or super + if specMode: - # spec totalQueries = queryTimes * len(sqls) + # spec mixed + if batchQuery.lower() == "yes": + # batch + threadQueries = len(sqls) + QPS = 2 + else: + threadQueries = totalQueries else: # super totalQueries = queryTimes * len(sqls) * tbCnt - threadQueries = totalQueries + threadQueries = totalQueries nSql = len(sqls) if specMode and nSql < concurrent : @@ -191,7 +206,7 @@ class TDTestCase(TBase): ["p99: ", "s", 0, False], ["INFO: Spend ", " ", 0, False], ["completed total queries: ", ",", totalQueries, True], - ["the QPS of all threads:", allEnd, 10 , False] # all qps need > 5 + ["the QPS of all threads:", allEnd, QPS , False] # all qps need > 5 ] # check @@ -205,6 +220,7 @@ class TDTestCase(TBase): args = [ ["./tools/benchmark/basic/json/queryModeSpec", True], ["./tools/benchmark/basic/json/queryModeSpecMix", True], + ["./tools/benchmark/basic/json/queryModeSpecMixBatch", True], ["./tools/benchmark/basic/json/queryModeSuper", False] ] @@ -231,8 +247,9 @@ class TDTestCase(TBase): self.expectFailed(f"{benchmark} -f ./tools/benchmark/basic/json/queryErrorBothSpecSuper.json") # json format error self.expectFailed(f"{benchmark} -f ./tools/benchmark/basic/json/queryErrorFormat.json") - - + # batch query + self.expectFailed(f"{benchmark} -f ./tools/benchmark/basic/json/queryErrorBatchNoMix.json") + self.expectFailed(f"{benchmark} -f ./tools/benchmark/basic/json/queryErrorBatchRest.json") def run(self): tbCnt = 10 diff --git a/tools/taos-tools/inc/bench.h b/tools/taos-tools/inc/bench.h index d47bafbaf0..2931790f25 100644 --- a/tools/taos-tools/inc/bench.h +++ b/tools/taos-tools/inc/bench.h @@ -648,6 +648,7 @@ typedef struct SpecifiedQueryInfo_S { TAOS_RES *res[MAX_QUERY_SQL_COUNT]; uint64_t totalQueried; bool mixed_query; + bool batchQuery; // mixed query have batch and no batch query // error rate uint64_t totalFail; } SpecifiedQueryInfo; diff --git a/tools/taos-tools/src/benchJsonOpt.c b/tools/taos-tools/src/benchJsonOpt.c index 3e41908668..9792a8f910 100644 --- a/tools/taos-tools/src/benchJsonOpt.c +++ b/tools/taos-tools/src/benchJsonOpt.c @@ -1813,6 +1813,23 @@ int32_t readSpecQueryJson(tools_cJSON * specifiedQuery) { } } + // batchQuery + tools_cJSON *batchQueryObj = + tools_cJSON_GetObjectItem(specifiedQuery, "batch_query"); + if (tools_cJSON_IsString(batchQueryObj)) { + if (0 == strcasecmp(batchQueryObj->valuestring, "yes")) { + g_queryInfo.specifiedQueryInfo.batchQuery = true; + infoPrint("%s\n","batch_query is True"); + } else if (0 == strcasecmp(batchQueryObj->valuestring, "no")) { + g_queryInfo.specifiedQueryInfo.batchQuery = false; + infoPrint("%s\n","batch_query is False"); + } else { + errorPrint("Invalid batch_query value: %s\n", + batchQueryObj->valuestring); + return -1; + } + } + tools_cJSON *concurrent = tools_cJSON_GetObjectItem(specifiedQuery, "concurrent"); if (tools_cJSON_IsNumber(concurrent)) { diff --git a/tools/taos-tools/src/benchMain.c b/tools/taos-tools/src/benchMain.c index 86ad795d05..02999392a5 100644 --- a/tools/taos-tools/src/benchMain.c +++ b/tools/taos-tools/src/benchMain.c @@ -47,7 +47,7 @@ void* benchCancelHandler(void* arg) { } #endif -void checkArgumentValid() { +int checkArgumentValid() { // check prepared_rand valid if(g_arguments->prepared_rand < g_arguments->reqPerReq) { infoPrint("prepared_rand(%"PRIu64") < num_of_records_per_req(%d), so set num_of_records_per_req = prepared_rand\n", @@ -64,13 +64,32 @@ void checkArgumentValid() { false, 1)) { errorPrint("%s", "Failed to convert server address\n"); - return; + return -1; } encodeAuthBase64(); g_arguments->rest_server_ver_major = getServerVersionRest(g_arguments->port); } + // check batch query + if (g_arguments->test_mode == QUERY_TEST) { + if (g_queryInfo.specifiedQueryInfo.batchQuery) { + // batch_query = yes + if (!g_queryInfo.specifiedQueryInfo.mixed_query) { + // mixed_query = no + errorPrint("%s\n", "batch_query = yes require mixed_query is yes"); + return -1; + } + + // rest not support + if (g_queryInfo.iface == REST_IFACE) { + errorPrint("%s\n", "batch_query = yes not support restful."); + return -1; + } + } + } + + return 0; } int main(int argc, char* argv[]) { @@ -144,7 +163,11 @@ int main(int argc, char* argv[]) { } infoPrint("client version: %s\n", taos_get_client_info()); - checkArgumentValid(); + if (checkArgumentValid()) { + errorPrint("failed to readJsonConfig %s\n", g_arguments->metaFile); + exitLog(); + return -1; + } if (g_arguments->test_mode == INSERT_TEST) { if (insertTestProcess()) { diff --git a/tools/taos-tools/src/benchQuery.c b/tools/taos-tools/src/benchQuery.c index 0de71d8b18..8be8bf9f6c 100644 --- a/tools/taos-tools/src/benchQuery.c +++ b/tools/taos-tools/src/benchQuery.c @@ -80,10 +80,15 @@ int selectAndGetResult(qThreadInfo *pThreadInfo, char *command, bool record) { } // interlligent sleep -void autoSleep(uint64_t interval, uint64_t delay ) { +int32_t autoSleep(uint64_t interval, uint64_t delay ) { + int32_t msleep = 0; if (delay < interval * 1000) { - toolsMsleep((int32_t)(interval * 1000 - delay)); // ms + msleep = (int32_t)((interval - delay/1000)); + infoPrint("do sleep %dms ...\n", msleep); + toolsMsleep(msleep); // ms + debugPrint("%s\n","do sleep end"); } + return msleep; } // reset @@ -140,10 +145,13 @@ static void *specQueryMixThread(void *sarg) { int64_t st = 0; int64_t et = 0; - int64_t startTs = toolsGetTimestampMs(); - int64_t lastPrintTime = startTs; - uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; - uint64_t interval = g_queryInfo.specifiedQueryInfo.queryInterval; + int64_t startTs = toolsGetTimestampMs(); + int64_t lastPrintTime = startTs; + // batchQuery + bool batchQuery = g_queryInfo.specifiedQueryInfo.batchQuery; + uint64_t queryTimes = batchQuery ? 1 : g_queryInfo.specifiedQueryInfo.queryTimes; + uint64_t interval = batchQuery ? 0 : g_queryInfo.specifiedQueryInfo.queryInterval; + pThreadInfo->query_delay_list = benchArrayInit(queryTimes, sizeof(int64_t)); for (int i = pThreadInfo->start_sql; i <= pThreadInfo->end_sql; ++i) { SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); @@ -382,7 +390,7 @@ static void *stbQueryThread(void *sarg) { // --------------------------------- firse level function ------------------------------ // -void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { +void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend, BArray *pDelays) { // valid check if (infos == NULL || threadCnt == 0) { return ; @@ -444,9 +452,11 @@ void totalChildQuery(qThreadInfo* infos, int threadCnt, int64_t spend) { (int32_t)(delay_list->size * 0.99)))/1E6, *(int64_t *)(benchArrayGet(delay_list, (int32_t)(delay_list->size - 1)))/1E6); - } else { - errorPrint("%s() LN%d, delay_list size: %"PRId64"\n", - __func__, __LINE__, (int64_t)delay_list->size); + } + + // copy to another + if (pDelays) { + benchArrayAddBatch(pDelays, delay_list->pData, delay_list->size, false); } benchArrayDestroy(delay_list); } @@ -547,7 +557,7 @@ static int stbQuery(uint16_t iface, char* dbName) { } // total show - totalChildQuery(threadInfos, threadCnt, end - start); + totalChildQuery(threadInfos, threadCnt, end - start, NULL); ret = 0; @@ -825,7 +835,7 @@ static int specQueryMix(uint16_t iface, char* dbName) { } // statistic - totalChildQuery(infos, threadCnt, end - start); + totalChildQuery(infos, threadCnt, end - start, NULL); ret = 0; OVER: @@ -838,6 +848,206 @@ OVER: return ret; } +void totalBatchQuery(int32_t allSleep, BArray *pDelays) { + // sort + qsort(pDelays->pData, pDelays->size, pDelays->elemSize, compare); + + // total delays + double totalDelays = 0; + for (size_t i = 0; i < pDelays->size; i++) { + int64_t *delay = benchArrayGet(pDelays, i); + totalDelays += *delay; + } + + printf("\n"); + // show sleep times + if (allSleep > 0) { + infoPrint("All sleep spend: %.3fs\n", (float)allSleep/1000); + } + + // show P90 ... + if (pDelays->size) { + infoPrint( + "Total delay: " + "min delay: %.6fs, " + "avg delay: %.6fs, " + "p90: %.6fs, " + "p95: %.6fs, " + "p99: %.6fs, " + "max: %.6fs\n", + *(int64_t *)(benchArrayGet(pDelays, 0))/1E6, + (double)totalDelays/pDelays->size/1E6, + *(int64_t *)(benchArrayGet(pDelays, + (int32_t)(pDelays->size * 0.9)))/1E6, + *(int64_t *)(benchArrayGet(pDelays, + (int32_t)(pDelays->size * 0.95)))/1E6, + *(int64_t *)(benchArrayGet(pDelays, + (int32_t)(pDelays->size * 0.99)))/1E6, + *(int64_t *)(benchArrayGet(pDelays, + (int32_t)(pDelays->size - 1)))/1E6); + } +} + +// +// specQuery Mix Batch +// +static int specQueryBatch(uint16_t iface, char* dbName) { + // init + BArray *pDelays = NULL; + int ret = -1; + int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; + uint64_t interval = g_queryInfo.specifiedQueryInfo.queryInterval; + pthread_t * pids = benchCalloc(nConcurrent, sizeof(pthread_t), true); + qThreadInfo *infos = benchCalloc(nConcurrent, sizeof(qThreadInfo), true); + infoPrint("start batch query, sleep interval:%" PRIu64 "ms query times:%" PRIu64 " thread:%d \n", + interval, g_queryInfo.query_times, nConcurrent); + + // concurent calc + int total_sql_num = g_queryInfo.specifiedQueryInfo.sqls->size; + int start_sql = 0; + int a = total_sql_num / nConcurrent; + if (a < 1) { + warnPrint("sqls num:%d < concurent:%d, set concurrent %d\n", total_sql_num, nConcurrent, nConcurrent); + nConcurrent = total_sql_num; + a = 1; + } + int b = 0; + if (nConcurrent != 0) { + b = total_sql_num % nConcurrent; + } + + // + // connect + // + int connCnt = 0; + for (int i = 0; i < nConcurrent; ++i) { + qThreadInfo *pThreadInfo = infos + i; + // create conn + if (initQueryConn(pThreadInfo, iface)){ + ret = -1; + goto OVER; + } + + connCnt ++; + } + + + // reset total + g_queryInfo.specifiedQueryInfo.totalQueried = 0; + g_queryInfo.specifiedQueryInfo.totalFail = 0; + + // + // running + // + int threadCnt = 0; + int allSleep = 0; + pDelays = benchArrayInit(10, sizeof(int64_t)); + for (int m = 0; m < g_queryInfo.query_times; ++m) { + // reset + threadCnt = 0; + start_sql = 0; + + // create thread + for (int i = 0; i < nConcurrent; ++i) { + qThreadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + pThreadInfo->start_sql = start_sql; + pThreadInfo->end_sql = i < b ? start_sql + a : start_sql + a - 1; + start_sql = pThreadInfo->end_sql + 1; + pThreadInfo->total_delay = 0; + // total zero + pThreadInfo->nSucc = 0; + pThreadInfo->nFail = 0; + + // main run + int code = pthread_create(pids + i, NULL, specQueryMixThread, pThreadInfo); + if (code != 0) { + errorPrint("failed specQueryBatchThread create. error code =%d \n", code); + break; + } + + threadCnt ++; + } + + bool needExit = false; + if (threadCnt != nConcurrent) { + // if failed, set termainte flag true like ctrl+c exit + needExit = true; + g_arguments->terminate = true; + } + + // wait thread finished + int64_t start = toolsGetTimestampUs(); + for (int i = 0; i < threadCnt; ++i) { + pthread_join(pids[i], NULL); + qThreadInfo *pThreadInfo = infos + i; + // total queries + g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nSucc; + if (g_arguments->continueIfFail == YES_IF_FAILED) { + // yes need add failed count + g_queryInfo.specifiedQueryInfo.totalQueried += pThreadInfo->nFail; + g_queryInfo.specifiedQueryInfo.totalFail += pThreadInfo->nFail; + } + + // destory + if (needExit) { + benchArrayDestroy(pThreadInfo->query_delay_list); + pThreadInfo->query_delay_list = NULL; + } + } + int64_t end = toolsGetTimestampUs(); + + // create + if (needExit) { + errorPrint("failed to create thread. expect nConcurrent=%d real threadCnt=%d, exit testing.\n", nConcurrent, threadCnt); + goto OVER; + } + + // batch total + printf("\n"); + totalChildQuery(infos, threadCnt, end - start, pDelays); + + // show batch total + int64_t delay = end - start; + infoPrint("count:%d execute batch spend: %" PRId64 "ms\n", m + 1, delay/1000); + + // sleep + if ( g_queryInfo.specifiedQueryInfo.batchQuery && interval > 0) { + allSleep += autoSleep(interval, delay); + } + + // check cancel + if(g_arguments->terminate) { + break; + } + } + ret = 0; + + // all total + totalBatchQuery(allSleep, pDelays); + +OVER: + // close conn + for (int i = 0; i < connCnt; ++i) { + qThreadInfo *pThreadInfo = infos + i; + closeQueryConn(pThreadInfo, iface); + } + + // free threads + tmfree(pids); + tmfree(infos); + + // free sqls + freeSpecialQueryInfo(); + + // free delays + if (pDelays) { + benchArrayDestroy(pDelays); + } + + return ret; +} + // total query for end void totalQuery(int64_t spends) { // total QPS @@ -903,14 +1113,19 @@ int queryTestProcess() { // start running // - uint64_t startTs = toolsGetTimestampMs(); if(g_queryInfo.specifiedQueryInfo.sqls && g_queryInfo.specifiedQueryInfo.sqls->size > 0) { // specified table if (g_queryInfo.specifiedQueryInfo.mixed_query) { // mixed - if (specQueryMix(g_queryInfo.iface, g_queryInfo.dbName)) { - return -1; + if(g_queryInfo.specifiedQueryInfo.batchQuery) { + if (specQueryBatch(g_queryInfo.iface, g_queryInfo.dbName)) { + return -1; + } + } else { + if (specQueryMix(g_queryInfo.iface, g_queryInfo.dbName)) { + return -1; + } } } else { // no mixied From 47344da61982480d3e645af302fbd5b3b98ad847 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 12 Mar 2025 18:39:25 +0800 Subject: [PATCH 12/44] ci: disable fail-fast in tdengine-build workflow --- .github/workflows/tdengine-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tdengine-build.yml b/.github/workflows/tdengine-build.yml index a9d429a95e..1e89a51c04 100644 --- a/.github/workflows/tdengine-build.yml +++ b/.github/workflows/tdengine-build.yml @@ -24,6 +24,7 @@ jobs: name: Run on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: - ubuntu-20.04 From 055154173b15e9d70f95f0262df17232ff27f274 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 12 Mar 2025 18:47:58 +0800 Subject: [PATCH 13/44] ci: add workflow to cancel ongoing runs on pull request merge --- .github/workflows/cancel-workflow-on-merge.yml | 15 +++++++++++++++ .github/workflows/tdengine-ci-test.yml | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/cancel-workflow-on-merge.yml diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml new file mode 100644 index 0000000000..4cd915b526 --- /dev/null +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -0,0 +1,15 @@ +name: Cancel Workflow on Merge + +on: + pull_request: + types: [closed] + +jobs: + cancel-workflow: + runs-on: ubuntu-latest + steps: + - name: Cancel Workflow if Merged + if: ${{ github.event.pull_request.merged }} + run: | + echo "PR has been merged, cancelling workflow..." + gh run cancel --repo ${{ github.repository }} --workflow "TDengine Test" --branch ${{ github.event.pull_request.base.ref }} \ No newline at end of file diff --git a/.github/workflows/tdengine-ci-test.yml b/.github/workflows/tdengine-ci-test.yml index d11b363679..6ce0c4cef4 100644 --- a/.github/workflows/tdengine-ci-test.yml +++ b/.github/workflows/tdengine-ci-test.yml @@ -11,7 +11,7 @@ on: - 'docs/**' concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-TDengine + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true env: @@ -91,4 +91,4 @@ jobs: needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} with: - tdinternal: false + tdinternal: false \ No newline at end of file From 9f69dce0cb3c59c190796397fc1cfd078cc36252 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 12 Mar 2025 18:54:00 +0800 Subject: [PATCH 14/44] ci: add input parameters for specified source and target branches in tdengine-ci-test workflow --- .github/workflows/tdengine-ci-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tdengine-ci-test.yml b/.github/workflows/tdengine-ci-test.yml index 212ba691ff..e43955ab72 100644 --- a/.github/workflows/tdengine-ci-test.yml +++ b/.github/workflows/tdengine-ci-test.yml @@ -129,3 +129,6 @@ jobs: if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false + specified_source_branch: ${{ github.event.inputs.specified_source_branch }} + specified_target_branch: ${{ github.event.inputs.specified_target_branch }} + specified_pr_number: ${{ github.event.inputs.specified_pr_number }} \ No newline at end of file From 6fb49512320cdc3fd45619332ead2c417283369f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 12 Mar 2025 19:08:20 +0800 Subject: [PATCH 15/44] fix: temp test --- utils/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 9872a9dc55..56c8248aff 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -5,4 +5,4 @@ ADD_SUBDIRECTORY(test/c) # ADD_SUBDIRECTORY(comparisonTest/tdengine) IF(NOT "${TSZ_ENABLED}" MATCHES "false") ADD_SUBDIRECTORY(TSZ) -ENDIF() \ No newline at end of file +ENDIF() From 3383f7d87c796aea6d9af281182112e7cef6f6f3 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Wed, 12 Mar 2025 19:59:45 +0800 Subject: [PATCH 16/44] docs(kubernetes): support community image with new helm chart Close [TS-6081](https://jira.taosdata.com:18080/browse/TS-6081) --- .../03-deployment/03-kubernetes.md | 276 +++++++++++++++++- 1 file changed, 265 insertions(+), 11 deletions(-) diff --git a/docs/en/08-operation/03-deployment/03-kubernetes.md b/docs/en/08-operation/03-deployment/03-kubernetes.md index fdec50b9d6..386e5f1dc9 100644 --- a/docs/en/08-operation/03-deployment/03-kubernetes.md +++ b/docs/en/08-operation/03-deployment/03-kubernetes.md @@ -339,20 +339,272 @@ Helm operates Kubernetes using kubectl and kubeconfig configurations, which can The TDengine Chart has not yet been released to the Helm repository, it can currently be downloaded directly from GitHub: ```shell -wget https://github.com/taosdata/TDengine-Operator/raw/refs/heads/3.0/helm/tdengine-enterprise-3.5.0.tgz +wget https://github.com/taosdata/TDengine-Operator/raw/refs/heads/3.0/helm/tdengine-3.5.0.tgz ``` -Note that it's for the enterprise edition, and the community edition is not yet available. - Follow the steps below to install the TDengine Chart: ```shell # Edit the values.yaml file to set the topology of the cluster vim values.yaml -helm install tdengine tdengine-enterprise-3.5.0.tgz -f values.yaml +helm install tdengine tdengine-3.5.0.tgz -f values.yaml ``` -#### Case 1: Simple 1-node Deployment +If you are using community images, you can use the following command to install TDengine with Helm Chart: + +
+Helm Chart Use Cases for Community + +#### Community Case 1: Simple 1-node Deployment + +The following is a simple example of deploying a single-node TDengine cluster using Helm. + +```yaml +# This example is a simple deployment with one server replica. +name: "tdengine" + +image: + repository: # Leave a trailing slash for the repository, or "" for no repository + server: tdengine/tdengine:latest + +# Set timezone here, not in taoscfg +timezone: "Asia/Shanghai" + +labels: + app: "tdengine" + # Add more labels as needed. + +services: + server: + type: ClusterIP + replica: 1 + ports: + # TCP range required + tcp: [6041, 6030, 6060] + # UDP range, optional + udp: + volumes: + - name: data + mountPath: /var/lib/taos + spec: + storageClassName: "local-path" + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: "10Gi" + - name: log + mountPath: /var/log/taos/ + spec: + storageClassName: "local-path" + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: "10Gi" + files: + - name: cfg # must be lower case. + mountPath: /etc/taos/taos.cfg + content: | + dataDir /var/lib/taos/ + logDir /var/log/taos/ +``` + +Let's explain the above configuration: + +- name: The name of the deployment, here it is "tdengine". +- image: + - repository: The image repository address, remember to leave a trailing slash for the repository, or set it to an empty string to use docker.io. + - server: The specific name and tag of the server image. You need to ask your business partner for the TDengine Enterprise image. +- timezone: Set the timezone, here it is "Asia/Shanghai". +- labels: Add labels to the deployment, here is an app label with the value "tdengine", more labels can be added as needed. +- services: + - server: Configure the server service. + - type: The service type, here it is **ClusterIP**. + - replica: The number of replicas, here it is 1. + - ports: Configure the ports of the service. + - tcp: The required TCP port range, here it is [6041, 6030, 6060]. + - udp: The optional UDP port range, which is not configured here. + - volumes: Configure the volumes. + - name: The name of the volume, here there are two volumes, data and log. + - mountPath: The mount path of the volume. + - spec: The specification of the volume. + - storageClassName: The storage class name, here it is **local-path**. + - accessModes: The access mode, here it is **ReadWriteOnce**. + - resources.requests.storage: The requested storage size, here it is **10Gi**. + - files: Configure the files to mount in TDengine server. + - name: The name of the file, here it is **cfg**. + - mountPath: The mount path of the file, which is **taos.cfg**. + - content: The content of the file, here the **dataDir** and **logDir** are configured. + +After configuring the values.yaml file, use the following command to install the TDengine Chart: + +```shell +helm install simple tdengine-3.5.0.tgz -f values.yaml +``` + +After installation, you can see the instructions to see the status of the TDengine cluster: + +```shell +NAME: simple +LAST DEPLOYED: Sun Feb 9 13:40:00 2025 default +STATUS: deployed +REVISION: 1 +TEST SUITE: None +NOTES: +1. Get first POD name: + +export POD_NAME=$(kubectl get pods --namespace default \ + -l "app.kubernetes.io/name=tdengine,app.kubernetes.io/instance=simple" -o jsonpath="{.items[0].metadata.name}") + +2. Show dnodes/mnodes: + +kubectl --namespace default exec $POD_NAME -- taos -s "show dnodes; show mnodes" + +3. Run into TDengine CLI: + +kubectl --namespace default exec -it $POD_NAME -- taos +``` + +Follow the instructions to check the status of the TDengine cluster: + +```shell +root@u1-58:/data1/projects/helm# kubectl --namespace default exec $POD_NAME -- taos -s "show dnodes; show mnodes" +Welcome to the TDengine Command Line Interface, Client Version:3.3.5.8 +Copyright (c) 2023 by TDengine, all rights reserved. + +taos> show dnodes; show mnodes + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | +============================================================================================================================================================================= + 1 | oss-tdengine-0.oss-tdengine... | 0 | 21 | ready | 2025-03-12 19:05:42.224 | 2025-03-12 19:05:42.044 | | +Query OK, 1 row(s) in set (0.002545s) + + id | endpoint | role | status | create_time | role_time | +================================================================================================================================== + 1 | oss-tdengine-0.oss-tdengine... | leader | ready | 2025-03-12 19:05:42.239 | 2025-03-12 19:05:42.137 | +Query OK, 1 row(s) in set (0.001343s) +``` + +To clean up the TDengine cluster, use the following command: + +```shell +helm uninstall simple +kubectl delete pvc -l app.kubernetes.io/instance=simple +``` + +#### Community Case 2: 3-replica Deployment with Single taosX + +```yaml +# This example shows how to deploy a 3-replica TDengine cluster with separate taosx/explorer service. +# Users should know that the explorer/taosx service is not cluster-ready, so it is recommended to deploy it separately. +name: "tdengine" + +image: + repository: # Leave a trailing slash for the repository, or "" for no repository + server: tdengine/tdengine:latest + +# Set timezone here, not in taoscfg +timezone: "Asia/Shanghai" + +labels: + # Add more labels as needed. + +services: + server: + type: ClusterIP + replica: 3 + ports: + # TCP range required + tcp: [6041, 6030] + # UDP range, optional + udp: + volumes: + - name: data + mountPath: /var/lib/taos + spec: + storageClassName: "local-path" + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: "10Gi" + - name: log + mountPath: /var/log/taos/ + spec: + storageClassName: "local-path" + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: "10Gi" +``` + +You can see that the configuration is similar to the first one, with the addition of the taosx configuration. The taosx service is configured with similar storage configuration as the server service, and the server service is configured with 3 replicas. Since the taosx service is not cluster-ready, it is recommended to deploy it separately. + +After configuring the values.yaml file, use the following command to install the TDengine Chart: + +```shell +helm install replica3 tdengine-3.5.0.tgz -f values.yaml +``` + +To clean up the TDengine cluster, use the following command: + +```shell +helm uninstall replica3 +kubectl delete pvc -l app.kubernetes.io/instance=replica3 +``` + +You can use the following command to expose the explorer service to the outside world with ingress: + +```shell +tee replica3-ingress.yaml < + +With TDengine Enterprise images, you can use the following command to install TDengine with Helm Chart: + +
+Helm Chart Use Cases for Enterprise + +#### Enterprise Case 1: Simple 1-node Deployment The following is a simple example of deploying a single-node TDengine cluster using Helm. @@ -435,7 +687,7 @@ Let's explain the above configuration: After configuring the values.yaml file, use the following command to install the TDengine Chart: ```shell -helm install simple tdengine-enterprise-3.5.0.tgz -f values.yaml +helm install simple tdengine-3.5.0.tgz -f values.yaml ``` After installation, you can see the instructions to see the status of the TDengine cluster: @@ -487,7 +739,7 @@ helm uninstall simple kubectl delete pvc -l app.kubernetes.io/instance=simple ``` -#### Case 2: Tiered-Storage Deployment +#### Enterprise Case 2: Tiered-Storage Deployment The following is an example of deploying a TDengine cluster with tiered storage using Helm. @@ -563,10 +815,10 @@ You can see that the configuration is similar to the previous one, with the addi After configuring the values.yaml file, use the following command to install the TDengine Chart: ```shell -helm install tiered tdengine-enterprise-3.5.0.tgz -f values.yaml +helm install tiered tdengine-3.5.0.tgz -f values.yaml ``` -#### Case 3: 2-replica Deployment +#### Enterprise Case 3: 2-replica Deployment TDengine support 2-replica deployment with an arbitrator, which can be configured as follows: @@ -634,7 +886,7 @@ services: You can see that the configuration is similar to the first one, with the addition of the arbitrator configuration. The arbitrator service is configured with the same storage as the server service, and the server service is configured with 2 replicas (the arbitrator should be 1 replica and not able to be changed). -#### Case 4: 3-replica Deployment with Single taosX +#### Enterprise Case 4: 3-replica Deployment with Single taosX ```yaml # This example shows how to deploy a 3-replica TDengine cluster with separate taosx/explorer service. @@ -761,7 +1013,7 @@ You can see that the configuration is similar to the first one, with the additio After configuring the values.yaml file, use the following command to install the TDengine Chart: ```shell -helm install replica3 tdengine-enterprise-3.5.0.tgz -f values.yaml +helm install replica3 tdengine-3.5.0.tgz -f values.yaml ``` You can use the following command to expose the explorer service to the outside world with ingress: @@ -810,3 +1062,5 @@ Now you can access the explorer service through the domain name `replica3.local. ```shell curl http://replica3.local.tdengine.com ``` + +
From f28de98d74f7ac2e15fb2d42f675b9b0dcc5437a Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 12 Mar 2025 20:24:11 +0800 Subject: [PATCH 17/44] ci: remove the fetch-parameters step logic --- .github/workflows/taosd-ci.yml | 113 ++++++++++++--------------------- 1 file changed, 42 insertions(+), 71 deletions(-) diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 71b00dc2d1..23c0e47ebb 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -9,26 +9,9 @@ on: paths-ignore: - 'packaging/**' - 'docs/**' - workflow_dispatch: - inputs: - specified_source_branch: - description: 'Enter the source branch name of TDengine' - required: false - default: 'unavailable' - type: string - specified_target_branch: - description: 'Enter the target branch name of TDengine' - required: false - default: 'unavailable' - type: string - specified_pr_number: - description: 'Enter the PR number of TDengine' - required: false - default: 'unavailable' - type: string concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.specified_target_branch }}-${{ github.event_name == 'workflow_dispatch' }}-TDengine + group: ${{ github.workflow }}-${{ github.ref }}-TDengine cancel-in-progress: true env: @@ -47,51 +30,48 @@ jobs: id: parameters run: | set -euo pipefail - if ${{ github.event_name == 'workflow_dispatch' }}; then - run_function_test="true" - run_tdgpt_test="false" - else - target_branch=${{ github.event.pull_request.base.ref }} + # target_branch=${{ github.event.pull_request.base.ref }} - # Fetch the latest code from the target branch - cd ${{ env.WKC }} - git reset --hard - git clean -f - git remote prune origin - git fetch - git checkout "$target_branch" - git remote prune origin - git pull >/dev/null - - # Check whether to run tdgpt test cases - changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) - echo "changed files exclude doc: ${changed_files_non_doc}" - - if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then - run_tdgpt_test="true" - else - run_tdgpt_test="false" - fi - echo "run tdgpt test: ${run_tdgpt_test}" + # # Fetch the latest code from the target branch + # cd ${{ env.WKC }} + # git reset --hard + # git clean -f + # git remote prune origin + # git fetch + # git checkout "$target_branch" + # git remote prune origin + # git pull >/dev/null - # Check whether to run function test cases - changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ - grep -v "^docs/en/" | \ - grep -v "^docs/zh/" | \ - grep -v ".md$" | \ - grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ - tr '\n' ' ' || :) - echo "changed files exclude tdgpt: ${changed_files_non_tdgpt}" - - if [ -n "$changed_files_non_tdgpt" ]; then - run_function_test="true" - else - run_function_test="false" - fi + # # Check whether to run tdgpt test cases + # changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) + # echo "changed files exclude doc: ${changed_files_non_doc}" + + # if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then + # run_tdgpt_test="true" + # else + # run_tdgpt_test="false" + # fi + # echo "run tdgpt test: ${run_tdgpt_test}" - echo "run function test: ${run_function_test}" - fi + # # Check whether to run function test cases + # changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ + # grep -v "^docs/en/" | \ + # grep -v "^docs/zh/" | \ + # grep -v ".md$" | \ + # grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ + # tr '\n' ' ' || :) + # echo "changed files exclude tdgpt: ${changed_files_non_tdgpt}" + + # if [ -n "$changed_files_non_tdgpt" ]; then + # run_function_test="true" + # else + # run_function_test="false" + # fi + # echo "run function test: ${run_function_test}" + + run_tdgpt_test="true" + run_function_test="true" # Output the results for GitHub Actions echo "run_function_test=$run_function_test" >> $GITHUB_OUTPUT echo "run_tdgpt_test=$run_tdgpt_test" >> $GITHUB_OUTPUT @@ -101,33 +81,24 @@ jobs: echo ${{ github.event.pull_request.number }} run-tests-on-linux: - uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@ci/test/workflow + uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@main needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false run_function_test: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} run_tdgpt_test: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' }} - specified_source_branch: ${{ github.event.inputs.specified_source_branch }} - specified_target_branch: ${{ github.event.inputs.specified_target_branch }} - specified_pr_number: ${{ github.event.inputs.specified_pr_number }} run-tests-on-mac: - uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml@ci/test/workflow + uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml@main needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false - specified_source_branch: ${{ github.event.inputs.specified_source_branch }} - specified_target_branch: ${{ github.event.inputs.specified_target_branch }} - specified_pr_number: ${{ github.event.inputs.specified_pr_number }} run-tests-on-windows: - uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml@ci/test/workflow + uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml@main needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false - specified_source_branch: ${{ github.event.inputs.specified_source_branch }} - specified_target_branch: ${{ github.event.inputs.specified_target_branch }} - specified_pr_number: ${{ github.event.inputs.specified_pr_number }} From e00422751e179b6d3e27531fcba9ce44664e03cb Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 12 Mar 2025 20:39:17 +0800 Subject: [PATCH 18/44] ci: simplify parameters in tdengine-ci-test workflow --- .github/workflows/tdengine-ci-test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tdengine-ci-test.yml b/.github/workflows/tdengine-ci-test.yml index 2f761b9501..43944a71c3 100644 --- a/.github/workflows/tdengine-ci-test.yml +++ b/.github/workflows/tdengine-ci-test.yml @@ -102,7 +102,4 @@ jobs: needs: fetch-parameters if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} with: - tdinternal: false - specified_source_branch: ${{ github.event.inputs.specified_source_branch }} - specified_target_branch: ${{ github.event.inputs.specified_target_branch }} - specified_pr_number: ${{ github.event.inputs.specified_pr_number }} + tdinternal: false \ No newline at end of file From a89460190750f6c801944cc4af15693a6232fd4d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 12 Mar 2025 20:46:05 +0800 Subject: [PATCH 19/44] ci: update concurrency group to use pull request number --- .github/workflows/tdengine-ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tdengine-ci-test.yml b/.github/workflows/tdengine-ci-test.yml index 43944a71c3..08310c91ff 100644 --- a/.github/workflows/tdengine-ci-test.yml +++ b/.github/workflows/tdengine-ci-test.yml @@ -11,7 +11,7 @@ on: - 'docs/**' concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-TDengine + group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-TDengine cancel-in-progress: true env: From 4adee83552901478e69f9afd7aac503406663c80 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 12 Mar 2025 20:58:05 +0800 Subject: [PATCH 20/44] ci: update workflow to cancel on pull request close in addition to merge --- .github/workflows/cancel-workflow-on-merge.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 4cd915b526..5d4a5000a2 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -1,4 +1,4 @@ -name: Cancel Workflow on Merge +name: Cancel Workflow on Merge or Close on: pull_request: @@ -8,8 +8,8 @@ jobs: cancel-workflow: runs-on: ubuntu-latest steps: - - name: Cancel Workflow if Merged - if: ${{ github.event.pull_request.merged }} + - name: Cancel Workflow if Merged or Closed + if: ${{ github.event.pull_request.merged || github.event.pull_request.state == 'closed' }} run: | - echo "PR has been merged, cancelling workflow..." - gh run cancel --repo ${{ github.repository }} --workflow "TDengine Test" --branch ${{ github.event.pull_request.base.ref }} \ No newline at end of file + echo "PR has been merged or closed, cancelling workflow..." + gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel {} \ No newline at end of file From c400322fc0cbfbdaaca4573a61b4171f94bdefdf Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 12 Mar 2025 21:00:36 +0800 Subject: [PATCH 21/44] ci: add ci manual workflow --- .github/workflows/taosd-ci-manual.yml | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/taosd-ci-manual.yml diff --git a/.github/workflows/taosd-ci-manual.yml b/.github/workflows/taosd-ci-manual.yml new file mode 100644 index 0000000000..7cb3688c46 --- /dev/null +++ b/.github/workflows/taosd-ci-manual.yml @@ -0,0 +1,54 @@ +name: TDengine CI Test Manually + +on: + workflow_dispatch: + inputs: + specified_source_branch: + description: 'Enter the source branch name of TDengine' + required: false + default: 'unavailable' + type: string + specified_target_branch: + description: 'Enter the target branch name of TDengine' + required: false + default: 'unavailable' + type: string + specified_pr_number: + description: 'Enter the PR number of TDengine' + required: false + default: 'unavailable' + type: string + +concurrency: + group: ${{ github.event.inputs.specified_target_branch }}-${{ github.event.inputs.specified_pr_number }}-TDengine + cancel-in-progress: true + +env: + WKC: '/var/lib/jenkins/workspace/TDinternal/community' + +jobs: + run-tests-on-linux: + uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@ci/test/workflow + with: + tdinternal: false + run_function_test: true + run_tdgpt_test: false + specified_source_branch: ${{ github.event.inputs.specified_source_branch }} + specified_target_branch: ${{ github.event.inputs.specified_target_branch }} + specified_pr_number: ${{ github.event.inputs.specified_pr_number }} + + run-tests-on-mac: + uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml@ci/test/workflow + with: + tdinternal: false + specified_source_branch: ${{ github.event.inputs.specified_source_branch }} + specified_target_branch: ${{ github.event.inputs.specified_target_branch }} + specified_pr_number: ${{ github.event.inputs.specified_pr_number }} + + run-tests-on-windows: + uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml@ci/test/workflow + with: + tdinternal: false + specified_source_branch: ${{ github.event.inputs.specified_source_branch }} + specified_target_branch: ${{ github.event.inputs.specified_target_branch }} + specified_pr_number: ${{ github.event.inputs.specified_pr_number }} From 3cd15a4434899685b18d4ef4dfd6c0b962488e71 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Wed, 12 Mar 2025 23:37:33 +0800 Subject: [PATCH 22/44] fix: typo, replace radio to ratio --- source/libs/function/src/builtinsimpl.c | 8 ++++---- source/util/test/decompressTest.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c81f05ea0b..2acea3d80f 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -6615,12 +6615,12 @@ int32_t blockDBUsageFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { uint64_t totalDiskSize = pData->dataInDiskSize; uint64_t rawDataSize = pData->rawDataSize; - double compressRadio = 0; + double compressRatio = 0; if (rawDataSize != 0) { - compressRadio = totalDiskSize * 100 / (double)rawDataSize; - len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[%.2f%]", compressRadio); + compressRatio = totalDiskSize * 100 / (double)rawDataSize; + len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_ratio=[%.2f%]", compressRatio); } else { - len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_radio=[NULL]"); + len = tsnprintf(varDataVal(st), sizeof(st) - VARSTR_HEADER_SIZE, "Compress_ratio=[NULL]"); } varDataSetLen(st, len); diff --git a/source/util/test/decompressTest.cpp b/source/util/test/decompressTest.cpp index b1f7f7e85c..c067831e88 100644 --- a/source/util/test/decompressTest.cpp +++ b/source/util/test/decompressTest.cpp @@ -182,7 +182,7 @@ void compressImplTestByAlg(void* pVal, int8_t type, int32_t num, uint32_t DEFINE_VAR(cmprAlg) int32_t len = compres.compFunc(pVal, bytes, num, px, externalSize, cmprAlg, pBuf, externalSize); - printf("encode:%s, compress alg:%s, type:%s, compresess size: %d, actual size: %d, radio: %f\n", end[l1], alg[l2], + printf("encode:%s, compress alg:%s, type:%s, compresess size: %d, actual size: %d, ratio: %f\n", end[l1], alg[l2], compres.name, len, bytes, (float)len / bytes); char* pOutput = static_cast(taosMemoryCalloc(1, externalSize)); memset(pBuf, 0, externalSize); From ad7afcc437d73ee541b91cc1d4cd98b17f1d3ec1 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 13 Mar 2025 09:39:06 +0800 Subject: [PATCH 23/44] ci: enhance workflow to set up GitHub CLI and cancel on PR merge or close --- .github/workflows/cancel-workflow-on-merge.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 5d4a5000a2..0885fbfdd8 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -8,8 +8,15 @@ jobs: cancel-workflow: runs-on: ubuntu-latest steps: + - name: Set up GitHub CLI + run: | + echo "${{ secrets.GITHUB_TOKEN }}" > ~/.gh_token + gh auth login --with-token < ~/.gh_token + - name: Cancel Workflow if Merged or Closed if: ${{ github.event.pull_request.merged || github.event.pull_request.state == 'closed' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "PR has been merged or closed, cancelling workflow..." gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel {} \ No newline at end of file From cf3582b9641b8fa25e15339bc78fd3da039190cc Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 13 Mar 2025 09:53:16 +0800 Subject: [PATCH 24/44] ci: add GH_TOKEN environment variable for GitHub CLI authentication in workflow --- .github/workflows/cancel-workflow-on-merge.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 0885fbfdd8..68c87cf24d 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -9,6 +9,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up GitHub CLI + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "${{ secrets.GITHUB_TOKEN }}" > ~/.gh_token gh auth login --with-token < ~/.gh_token @@ -19,4 +21,4 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "PR has been merged or closed, cancelling workflow..." - gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel {} \ No newline at end of file + gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel --repo ${{ github.repository }} {} \ No newline at end of file From b721ebed76699cb4cdbeaf469c37eaa8994ceb25 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 13 Mar 2025 10:44:58 +0800 Subject: [PATCH 25/44] ci: streamline GitHub CLI authentication in cancel workflow --- .github/workflows/cancel-workflow-on-merge.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 68c87cf24d..5147a5b204 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -8,17 +8,11 @@ jobs: cancel-workflow: runs-on: ubuntu-latest steps: - - name: Set up GitHub CLI - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "${{ secrets.GITHUB_TOKEN }}" > ~/.gh_token - gh auth login --with-token < ~/.gh_token - - name: Cancel Workflow if Merged or Closed if: ${{ github.event.pull_request.merged || github.event.pull_request.state == 'closed' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "PR has been merged or closed, cancelling workflow..." + gh auth login --with-token $GH_TOKEN gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel --repo ${{ github.repository }} {} \ No newline at end of file From 86f4f6571de27fb78a79875ee3dd7b5069baacf6 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 13 Mar 2025 11:00:57 +0800 Subject: [PATCH 26/44] ci: update authentication method in cancel workflow to check status --- .github/workflows/cancel-workflow-on-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 5147a5b204..235ac2e7b1 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -14,5 +14,5 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "PR has been merged or closed, cancelling workflow..." - gh auth login --with-token $GH_TOKEN + gh auth status gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel --repo ${{ github.repository }} {} \ No newline at end of file From 8c8b013565ff06f0d545d34340fc63f701d571ac Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 13 Mar 2025 11:11:37 +0800 Subject: [PATCH 27/44] ci: update cancel workflow to include queued status in run cancellation --- .github/workflows/cancel-workflow-on-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 235ac2e7b1..44b3abdd49 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -15,4 +15,4 @@ jobs: run: | echo "PR has been merged or closed, cancelling workflow..." gh auth status - gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel --repo ${{ github.repository }} {} \ No newline at end of file + gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --status queued --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel --repo ${{ github.repository }} {} \ No newline at end of file From 26618f01414730e40fd8defa6a0de216935a64c2 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 13 Mar 2025 11:27:35 +0800 Subject: [PATCH 28/44] docs: update prerequisites header Signed-off-by: WANG Xu --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f7db2a7ea2..d532046ce7 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,9 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine 1. [Introduction](#1-introduction) 1. [Documentation](#2-documentation) 1. [Prerequisites](#3-prerequisites) - - [3.1 Prerequisites On Linux](#31-on-linux) - - [3.2 Prerequisites On macOS](#32-on-macos) - - [3.3 Prerequisites On Windows](#33-on-windows) + - [3.1 Prerequisites On Linux](#31-prerequisites-on-linux) + - [3.2 Prerequisites On macOS](#32-prerequisites-on-macos) + - [3.3 Prerequisites On Windows](#33-prerequisites-on-windows) - [3.4 Clone the repo](#34-clone-the-repo) 1. [Building](#4-building) - [4.1 Build on Linux](#41-build-on-linux) @@ -84,7 +84,7 @@ At the moment, TDengine server supports running on Linux/Windows/MacOS systems. If you want to compile taosAdapter or taosKeeper, you need to install Go 1.18 or above. -## 3.1 On Linux +## 3.1 Prerequisites on Linux
@@ -109,7 +109,7 @@ yum install -y zlib-static xz-devel snappy-devel jansson-devel pkgconfig libatom
-## 3.2 On macOS +## 3.2 Prerequisites on macOS
@@ -123,7 +123,7 @@ brew install argp-standalone gflags pkgconfig
-## 3.3 On Windows +## 3.3 Prerequisites on Windows
From 593af04b844e57aa6df24f93d306613bf4d5c7ed Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:42:39 +0800 Subject: [PATCH 29/44] docs: lowercase anchors --- docs/en/10-third-party/05-bi/03-powerbi.md | 4 ++-- docs/en/10-third-party/05-bi/12-tableau.md | 4 ++-- docs/en/10-third-party/05-bi/13-excel.md | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/10-third-party/05-bi/03-powerbi.md b/docs/en/10-third-party/05-bi/03-powerbi.md index 85746da45b..dac8ee2b62 100644 --- a/docs/en/10-third-party/05-bi/03-powerbi.md +++ b/docs/en/10-third-party/05-bi/03-powerbi.md @@ -14,8 +14,8 @@ Power BI is a business analytics tool provided by Microsoft. By configuring the ## Configure Data Source -**Step 1**, Search and open the [ODBC Data Source (64 bit)] management tool in the Start menu of the Windows operating system and configure it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#Installation). - +**Step 1**, Search and open the [ODBC Data Source (64 bit)] management tool in the Start menu of the Windows operating system and configure it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#installation). + **Step 2**, Open Power BI and log in, click [Home] -> [Get Data] -> [Other] -> [ODBC] -> [Connect], add data source. **Step 3**, Select the data source name just created, such as [MyTDengine], if you need to enter SQL, you can click the [Advanced options] tab, in the expanded dialog box enter the SQL statement. Click the [OK] button to connect to the configured data source. diff --git a/docs/en/10-third-party/05-bi/12-tableau.md b/docs/en/10-third-party/05-bi/12-tableau.md index d871451798..cacfaf94c0 100644 --- a/docs/en/10-third-party/05-bi/12-tableau.md +++ b/docs/en/10-third-party/05-bi/12-tableau.md @@ -13,11 +13,11 @@ Prepare the following environment: - TDengine 3.3.5.8 and above version is installed and running normally (both Enterprise and Community versions are available). - taosAdapter is running normally, refer to [taosAdapter Reference](../../../tdengine-reference/components/taosadapter/). - Install and run Tableau Desktop (if not installed, please download and install Windows operating system 64-bit [Download Tableau Desktop](https://www.tableau.com/products/desktop/download)). Install Tableau please refer to [Tableau Desktop](https://www.tableau.com). -- Download the latest Windows operating system X64 client driver from the TDengine official website and install it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#Installation). +- Download the latest Windows operating system X64 client driver from the TDengine official website and install it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#installation). ## Configure Data Source -**Step 1**, Search and open the "ODBC Data Source (64 bit)" management tool in the Start menu of the Windows operating system and configure it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#Installation). +**Step 1**, Search and open the "ODBC Data Source (64 bit)" management tool in the Start menu of the Windows operating system and configure it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#installation). :::tip It should be noted that when configuring the ODBC data source for Tableau, the [Database] configuration item on the TDengine ODBC data source configuration page is required. You need to select a database that can be successfully connected. diff --git a/docs/en/10-third-party/05-bi/13-excel.md b/docs/en/10-third-party/05-bi/13-excel.md index 47ee780c02..286c64bca2 100644 --- a/docs/en/10-third-party/05-bi/13-excel.md +++ b/docs/en/10-third-party/05-bi/13-excel.md @@ -13,12 +13,12 @@ Prepare the following environment: - TDengine 3.3.5.8 and above version is installed and running normally (both Enterprise and Community versions are available). - taosAdapter is running normally, refer to [taosAdapter Reference](../../../tdengine-reference/components/taosadapter/). - Install and run Excel. If not installed, please download and install it. For specific instructions, please refer to Microsoft's official documentation. -- Download the latest Windows operating system X64 client driver from the TDengine official website and install it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#Installation). +- Download the latest Windows operating system X64 client driver from the TDengine official website and install it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#installation). ## Configure Data Source -**Step 1**, Search and open the [ODBC Data Source (64 bit)] management tool in the Start menu of the Windows operating system and configure it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#Installation). - +**Step 1**, Search and open the [ODBC Data Source (64 bit)] management tool in the Start menu of the Windows operating system and configure it, refer to [Install ODBC Driver](../../../tdengine-reference/client-libraries/odbc/#installation). + **Step 2**, Start Excel in the Windows system environment, then select [Data] -> [Get Data] -> [From Other Sources] -> [From ODBC]. ![excel-odbc](./excel/odbc-menu.webp) From 83cc432bf9d2f4a783ba5d586696063b0ccc6fef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Mar 2025 14:55:22 +0800 Subject: [PATCH 30/44] test(stream): add status check before pause/resume streams. --- tests/system-test/8-stream/checkpoint_info2.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/system-test/8-stream/checkpoint_info2.py b/tests/system-test/8-stream/checkpoint_info2.py index 3dc57477f7..0f18b04feb 100644 --- a/tests/system-test/8-stream/checkpoint_info2.py +++ b/tests/system-test/8-stream/checkpoint_info2.py @@ -8,6 +8,7 @@ # expressly provided by the written permission from Jianhui Tao # ################################################################### +import time # -*- coding: utf-8 -*- @@ -88,6 +89,17 @@ class TDTestCase: return True def restart_stream(self): + st = time.time() + while True: + sql = 'select status from information_schema.ins_stream_tasks where status<>"ready" ' + if len(tdSql.getResult(sql)) != 0: + time.sleep(1) + tdLog.info("wait for task to be ready, 1s") + else: + et = time.time() + tdLog.info(f"wait for tasks to be ready: {et-st}s") + break + tdLog.debug("========restart stream========") for i in range(5): tdSql.execute("pause stream s1") From d6d839a4307665ea3e8834800da94323e29f20dc Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 13 Mar 2025 17:12:15 +0800 Subject: [PATCH 31/44] update doc-build and ci workflow --- .github/workflows/taosd-ci.yml | 79 +++------------------------ .github/workflows/taosd-doc-build.yml | 1 - 2 files changed, 9 insertions(+), 71 deletions(-) diff --git a/.github/workflows/taosd-ci.yml b/.github/workflows/taosd-ci.yml index 23c0e47ebb..7f6dc5d3d7 100644 --- a/.github/workflows/taosd-ci.yml +++ b/.github/workflows/taosd-ci.yml @@ -9,6 +9,13 @@ on: paths-ignore: - 'packaging/**' - 'docs/**' + - 'tools/tdgpt/**' + - 'source/libs/executor/src/forecastoperator.c' + - 'source/libs/executor/src/anomalywindowoperator.c' + - 'include/common/tanalytics.h' + - 'source/common/src/tanalytics.c' + - 'tests/parallel/tdgpt_cases.task' + - 'tests/script/tsim/analytics' concurrency: group: ${{ github.workflow }}-${{ github.ref }}-TDengine @@ -18,87 +25,19 @@ env: WKC: '/var/lib/jenkins/workspace/TDinternal/community' jobs: - fetch-parameters: - runs-on: - group: CI - labels: [self-hosted, Linux, X64, testing] - outputs: - run_function_test: ${{ steps.parameters.outputs.run_function_test }} - run_tdgpt_test: ${{ steps.parameters.outputs.run_tdgpt_test }} - steps: - - name: Determine trigger source and fetch parameters - id: parameters - run: | - set -euo pipefail - # target_branch=${{ github.event.pull_request.base.ref }} - - # # Fetch the latest code from the target branch - # cd ${{ env.WKC }} - # git reset --hard - # git clean -f - # git remote prune origin - # git fetch - # git checkout "$target_branch" - # git remote prune origin - # git pull >/dev/null - - # # Check whether to run tdgpt test cases - # changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | grep -v "^docs/en/" | grep -v "^docs/zh/" | grep -v ".md$" | tr '\n' ' ' || :) - # echo "changed files exclude doc: ${changed_files_non_doc}" - - # if [[ -n "$changed_files_non_doc" && "$changed_files_non_doc" =~ (forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt) ]]; then - # run_tdgpt_test="true" - # else - # run_tdgpt_test="false" - # fi - # echo "run tdgpt test: ${run_tdgpt_test}" - - # # Check whether to run function test cases - # changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD $target_branch) | \ - # grep -v "^docs/en/" | \ - # grep -v "^docs/zh/" | \ - # grep -v ".md$" | \ - # grep -Ev "forecastoperator\.c|anomalywindowoperator\.c|tanalytics\.h|tanalytics\.c|tdgpt_cases\.task|analytics|tdgpt" | \ - # tr '\n' ' ' || :) - # echo "changed files exclude tdgpt: ${changed_files_non_tdgpt}" - - # if [ -n "$changed_files_non_tdgpt" ]; then - # run_function_test="true" - # else - # run_function_test="false" - # fi - - # echo "run function test: ${run_function_test}" - - run_tdgpt_test="true" - run_function_test="true" - # Output the results for GitHub Actions - echo "run_function_test=$run_function_test" >> $GITHUB_OUTPUT - echo "run_tdgpt_test=$run_tdgpt_test" >> $GITHUB_OUTPUT - - echo ${{ github.event.pull_request.head.ref }} - echo ${{ github.event.pull_request.base.ref }} - echo ${{ github.event.pull_request.number }} - run-tests-on-linux: uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@main - needs: fetch-parameters - if: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' || needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false - run_function_test: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} - run_tdgpt_test: ${{ needs.fetch-parameters.outputs.run_tdgpt_test == 'true' }} + run_function_test: true + run_tdgpt_test: false run-tests-on-mac: uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml@main - needs: fetch-parameters - if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false run-tests-on-windows: uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml@main - needs: fetch-parameters - if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }} with: tdinternal: false diff --git a/.github/workflows/taosd-doc-build.yml b/.github/workflows/taosd-doc-build.yml index 28f1f10835..3ac922820a 100644 --- a/.github/workflows/taosd-doc-build.yml +++ b/.github/workflows/taosd-doc-build.yml @@ -7,7 +7,6 @@ on: - '3.0' paths: - 'docs/**' - - '*.md' env: DOC_WKC: "/root/doc_ci_work" From ab8e2b7590091df93c0fbe38b34dafdd2fec83c7 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 13 Mar 2025 17:29:13 +0800 Subject: [PATCH 32/44] refactor: break long line into multiple lines for readability Signed-off-by: WANG Xu --- .github/workflows/cancel-workflow-on-merge.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 44b3abdd49..e9a0403fa3 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -1,4 +1,5 @@ name: Cancel Workflow on Merge or Close +description: Cancels the workflow if the PR is merged or closed to avoid unnecessary runs. on: pull_request: @@ -15,4 +16,11 @@ jobs: run: | echo "PR has been merged or closed, cancelling workflow..." gh auth status - gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --status queued --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel --repo ${{ github.repository }} {} \ No newline at end of file + gh run list \ + --repo ${{ github.repository }} \ + --branch ${{ github.event.pull_request.head.ref }} \ + --workflow "TDengine Test" \ + --status in_progress \ + --status queued + --json databaseId --jq '.[].databaseId' | \ + xargs -I {} gh run cancel --repo ${{ github.repository }} {} \ No newline at end of file From 15146174aa5a24b0425a2335667397b4b73538c1 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 13 Mar 2025 18:43:31 +0800 Subject: [PATCH 33/44] docs: interp --- docs/en/14-reference/03-taos-sql/06-select.md | 9 +++ .../14-reference/03-taos-sql/10-function.md | 69 +++++++++---------- docs/zh/14-reference/03-taos-sql/06-select.md | 9 +++ .../14-reference/03-taos-sql/10-function.md | 66 ++++++++---------- 4 files changed, 79 insertions(+), 74 deletions(-) diff --git a/docs/en/14-reference/03-taos-sql/06-select.md b/docs/en/14-reference/03-taos-sql/06-select.md index 3b89c4fe5a..98559104c0 100644 --- a/docs/en/14-reference/03-taos-sql/06-select.md +++ b/docs/en/14-reference/03-taos-sql/06-select.md @@ -277,6 +277,15 @@ TDengine supports INNER JOIN based on the timestamp primary key, with the follow 5. Both sides of JOIN support subqueries. 6. Does not support mixing with the FILL clause. +## INTERP +The INTERP clause is used in conjunction with the [INTERP](../function/#interp) function to determine the rules for capturing time slices and interpolation when the INTERP function is executed. +- The output time range for INTERP is specified by the RANGE(timestamp1, timestamp2) field, which must satisfy timestamp1 \<= timestamp2. Here, timestamp1 is the start value of the output time range, i.e., if the conditions for interpolation are met at timestamp1, then timestamp1 is the first record output, and timestamp2 is the end value of the output time range, i.e., the timestamp of the last record output cannot be greater than timestamp2. +- INTERP determines the number of results within the output time range based on the EVERY(time_unit) field, starting from timestamp1 and interpolating at fixed intervals of time (time_unit value), where time_unit can be time units: 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), 1w (weeks). For example, EVERY(500a) will interpolate the specified data every 500 milliseconds. +- INTERP determines how to interpolate at each time point that meets the output conditions based on the FILL field. For how to use the FILL clause, refer to [FILL Clause](../time-series-extensions/) +- INTERP can interpolate at a single time point specified in the RANGE field, in which case the EVERY field can be omitted. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear). +- INTERP query supports NEAR FILL mode, i.e., when FILL is needed, it uses the data closest to the current time point for interpolation. When the timestamps before and after are equally close to the current time slice, FILL the previous row's value. This mode is not supported in stream computing and window queries. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR).(Supported from version 3.3.4.9). +- INTERP `RANGE` clause supports the expansion of the time range (supported from version 3.3.4.9), such as `RANGE('2023-01-01 00:00:00', 10s)` means to find data 10s before and after the time point '2023-01-01 00:00:00' for interpolation, FILL PREV/NEXT/NEAR respectively means to look for data forward/backward/around the time point, if there is no data around the time point, then use the value specified by FILL for interpolation, therefore the FILL clause must specify a value at this time. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1). Currently, only the combination of time point and time range is supported, not the combination of time interval and time range, i.e., RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h) is not supported. The specified time range rules are similar to EVERY, the unit cannot be year or month, the value cannot be 0, and cannot have quotes. When using this extension, other FILL modes except FILL PREV/NEXT/NEAR are not supported, and the EVERY clause cannot be specified. + ## GROUP BY If a GROUP BY clause is specified in the statement, the SELECT list can only contain the following expressions: diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index 8397c59177..27b5d2a577 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -1932,43 +1932,6 @@ FIRST(expr) - If all values in a column in the result set are NULL, the return for that column is also NULL; - If all columns in the result set are NULL, no results are returned. - For tables with composite primary keys, if there are multiple entries with the smallest timestamp, only the data with the smallest composite primary key is returned. - -### INTERP - -```sql -INTERP(expr [, ignore_null_values]) - -ignore_null_values: { - 0 - | 1 -} -``` - -**Function Description**: Returns the record value or interpolated value of a specified column at a specified time slice. The ignore_null_values parameter can be 0 or 1, where 1 means to ignore NULL values, default is 0. - -**Return Data Type**: Same as the field type. - -**Applicable Data Types**: Numeric types. - -**Applicable to**: Tables and supertables. - -**Usage Instructions** - -- INTERP is used to obtain the record value of a specified column at a specified time slice. If there is no row data that meets the conditions at that time slice, interpolation will be performed according to the settings of the FILL parameter. -- The input data for INTERP is the data of the specified column, which can be filtered through conditional statements (where clause). If no filtering condition is specified, the input is all data. -- INTERP SQL queries need to be used together with the RANGE, EVERY, and FILL keywords; stream computing cannot use RANGE, needs EVERY and FILL keywords together. -- The output time range for INTERP is specified by the RANGE(timestamp1, timestamp2) field, which must satisfy timestamp1 \<= timestamp2. Here, timestamp1 is the start value of the output time range, i.e., if the conditions for interpolation are met at timestamp1, then timestamp1 is the first record output, and timestamp2 is the end value of the output time range, i.e., the timestamp of the last record output cannot be greater than timestamp2. -- INTERP determines the number of results within the output time range based on the EVERY(time_unit) field, starting from timestamp1 and interpolating at fixed intervals of time (time_unit value), where time_unit can be time units: 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), 1w (weeks). For example, EVERY(500a) will interpolate the specified data every 500 milliseconds. -- INTERP determines how to interpolate at each time point that meets the output conditions based on the FILL field. For how to use the FILL clause, refer to [FILL Clause](../time-series-extensions/) -- INTERP can interpolate at a single time point specified in the RANGE field, in which case the EVERY field can be omitted. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear). -- When INTERP is applied to a supertable, it will sort all the subtable data under that supertable by primary key column and perform interpolation calculations, and can also be used with PARTITION BY tbname to force the results to a single timeline. -- INTERP can be used with the pseudocolumn _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0). -- INTERP can be used with the pseudocolumn _isfilled to display whether the return result is from the original record or generated by the interpolation algorithm (supported from version 3.0.3.0). -- For queries on tables with composite primary keys, if there are data with the same timestamp, only the data with the smallest composite primary key participates in the calculation. -- INTERP query supports NEAR FILL mode, i.e., when FILL is needed, it uses the data closest to the current time point for interpolation. When the timestamps before and after are equally close to the current time slice, FILL the previous row's value. This mode is not supported in stream computing and window queries. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR).(Supported from version 3.3.4.9). -- INTERP can only use the pseudocolumn `_irowts_origin` when using FILL PREV/NEXT/NEAR modes. `_irowts_origin` is supported from version 3.3.4.9. -- INTERP `RANGE` clause supports the expansion of the time range (supported from version 3.3.4.9), such as `RANGE('2023-01-01 00:00:00', 10s)` means to find data 10s before and after the time point '2023-01-01 00:00:00' for interpolation, FILL PREV/NEXT/NEAR respectively means to look for data forward/backward/around the time point, if there is no data around the time point, then use the value specified by FILL for interpolation, therefore the FILL clause must specify a value at this time. For example: SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1). Currently, only the combination of time point and time range is supported, not the combination of time interval and time range, i.e., RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h) is not supported. The specified time range rules are similar to EVERY, the unit cannot be year or month, the value cannot be 0, and cannot have quotes. When using this extension, other FILL modes except FILL PREV/NEXT/NEAR are not supported, and the EVERY clause cannot be specified. - ### LAST ```sql @@ -2212,6 +2175,38 @@ ignore_option: { - When there is no composite primary key, if different subtables have data with the same timestamp, a "Duplicate timestamps not allowed" message will be displayed - When using composite primary keys, the timestamp and primary key combinations of different subtables may be the same, which row is used depends on which one is found first, meaning that the results of running diff() multiple times in this situation may vary. +### INTERP + +```sql +INTERP(expr [, ignore_null_values]) + +ignore_null_values: { + 0 + | 1 +} +``` + +**Function Description**: Returns the record value or interpolated value of a specified column at a specified time slice. The ignore_null_values parameter can be 0 or 1, where 1 means to ignore NULL values, default is 0. + +**Return Data Type**: Same as the field type. + +**Applicable Data Types**: Numeric types. + +**Applicable to**: Tables and supertables. + +**Usage Instructions** + +- INTERP is used to obtain the record value of a specified column at a specified time slice. If there is no row data that meets the conditions at that time slice, interpolation will be performed according to the settings of the [FILL](../distinguished/#FILL-Clause) parameter. +- The input data for INTERP is the data of the specified column, which can be filtered through conditional statements (where clause). If no filtering condition is specified, the input is all data. +- INTERP SQL queries need to be used together with the RANGE, EVERY, and FILL keywords; stream computing cannot use RANGE, needs EVERY and FILL keywords together. + +- When INTERP is applied to a supertable, it will sort all the subtable data under that supertable by primary key column and perform interpolation calculations, and can also be used with PARTITION BY tbname to force the results to a single timeline. +- INTERP can be used with the pseudocolumn _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0). +- INTERP can be used with the pseudocolumn _isfilled to display whether the return result is from the original record or generated by the interpolation algorithm (supported from version 3.0.3.0). +- For queries on tables with composite primary keys, if there are data with the same timestamp, only the data with the smallest composite primary key participates in the calculation. + +- INTERP can only use the pseudocolumn `_irowts_origin` when using FILL PREV/NEXT/NEAR modes. `_irowts_origin` is supported from version 3.3.4.9. + ### IRATE ```sql diff --git a/docs/zh/14-reference/03-taos-sql/06-select.md b/docs/zh/14-reference/03-taos-sql/06-select.md index 696fe03d41..32a89c8cdb 100644 --- a/docs/zh/14-reference/03-taos-sql/06-select.md +++ b/docs/zh/14-reference/03-taos-sql/06-select.md @@ -277,6 +277,15 @@ TDengine 支持基于时间戳主键的 INNER JOIN，规则如下： 5. JOIN 两侧均支持子查询。 6. 不支持与 FILL 子句混合使用。 +## INTERP +interp 子句配合 [INTERP](../function/#interp) 函数使用, 用于确定 INTERP 函数执行时，时间断面的截取规则和插值规则。 +- INTERP 的输出时间范围根据 RANGE(timestamp1, timestamp2) 字段来指定，需满足 timestamp1 \<= timestamp2。其中 timestamp1 为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2 为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 +- INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（time_unit 值）进行插值，time_unit 可取值时间单位：1a(毫秒)、1s(秒)、1m(分)、1h(小时)、1d(天)、1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值。 +- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](./distinguished#fill-子句) +- INTERP 可以在 RANGE 字段中只指定唯一的时间戳对单个时间点进行插值，在这种情况下，EVERY 字段可以省略。例如 `SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear)`。 +- INTERP 查询支持 NEAR FILL 模式，即当需要 FILL 时，使用距离当前时间点最近的数据进行插值，当前后时间戳与当前时间断面一样近时，FILL 前一行的值. 此模式在流计算中和窗口查询中不支持。例如 `SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)` (v3.3.4.9 及以后支持)。 +- INTERP `RANGE`子句支持时间范围的扩展(v3.3.4.9 及以后支持)，如`RANGE('2023-01-01 00:00:00', 10s)`表示在时间点 '2023-01-01 00:00:00' 查找前后 10s 的数据进行插值，FILL PREV/NEXT/NEAR 分别表示从时间点向前/向后/前后查找数据，若时间点周围没有数据，则使用 FILL 指定的值进行插值，因此此时 FILL 子句必须指定值。例如 `SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1)`。目前仅支持时间点和时间范围的组合，不支持时间区间和时间范围的组合，即不支持 `RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h)`。所指定的时间范围规则与 EVERY 类似，单位不能是年或月，值不能为 0，不能带引号。使用该扩展时，不支持除 `FILL PREV/NEXT/NEAR` 外的其他 FILL 模式，且不能指定 EVERY 子句。 + ## GROUP BY 如果在语句中同时指定了 GROUP BY 子句，那么 SELECT 列表只能包含如下表达式： diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 882d567c48..9602e7fa5c 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -1861,43 +1861,6 @@ FIRST(expr) - 如果结果集中的某列全部为 NULL 值，则该列的返回结果也是 NULL； - 如果结果集中所有列全部为 NULL 值，则不返回结果。 - 对于存在复合主键的表的查询，若最小时间戳的数据有多条，则只有对应的复合主键最小的数据被返回。 - -### INTERP - -```sql -INTERP(expr [, ignore_null_values]) - -ignore_null_values: { - 0 - | 1 -} -``` - -**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值，缺省值为 0。 - -**返回数据类型**：同字段类型。 - -**适用数据类型**：数值类型。 - -**适用于**：表和超级表。 - -**使用说明** - -- INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。 -- INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 -- INTERP SQL 查询需要同时与 RANGE、EVERY 和 FILL 关键字一起使用；流计算不能使用 RANGE，需要 EVERY 和 FILL 关键字一起使用。 -- INTERP 的输出时间范围根据 RANGE(timestamp1, timestamp2) 字段来指定，需满足 timestamp1 \<= timestamp2。其中 timestamp1 为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2 为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 -- INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（time_unit 值）进行插值，time_unit 可取值时间单位：1a(毫秒)、1s(秒)、1m(分)、1h(小时)、1d(天)、1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值。 -- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](../distinguished/#fill-子句) -- INTERP 可以在 RANGE 字段中只指定唯一的时间戳对单个时间点进行插值，在这种情况下，EVERY 字段可以省略。例如 `SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear)`。 -- INTERP 作用于超级表时，会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。 -- INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(v3.0.2.0 以后支持)。 -- INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(v3.0.3.0 以后支持)。 -- INTERP 对于带复合主键的表的查询，若存在相同时间戳的数据，则只有对应的复合主键最小的数据参与运算。 -- INTERP 查询支持 NEAR FILL 模式，即当需要 FILL 时，使用距离当前时间点最近的数据进行插值，当前后时间戳与当前时间断面一样近时，FILL 前一行的值. 此模式在流计算中和窗口查询中不支持。例如 `SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', '2023-01-01 00:10:00') FILL(NEAR)` (v3.3.4.9 及以后支持)。 -- INTERP 只有在使用 FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`。`_irowts_origin`在 v3.3.4.9 及以后支持。 -- INTERP `RANGE`子句支持时间范围的扩展(v3.3.4.9 及以后支持)，如`RANGE('2023-01-01 00:00:00', 10s)`表示在时间点 '2023-01-01 00:00:00' 查找前后 10s 的数据进行插值，FILL PREV/NEXT/NEAR 分别表示从时间点向前/向后/前后查找数据，若时间点周围没有数据，则使用 FILL 指定的值进行插值，因此此时 FILL 子句必须指定值。例如 `SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00', 10s) FILL(PREV, 1)`。目前仅支持时间点和时间范围的组合，不支持时间区间和时间范围的组合，即不支持 `RANGE('2023-01-01 00:00:00', '2023-02-01 00:00:00', 1h)`。所指定的时间范围规则与 EVERY 类似，单位不能是年或月，值不能为 0，不能带引号。使用该扩展时，不支持除 `FILL PREV/NEXT/NEAR` 外的其他 FILL 模式，且不能指定 EVERY 子句。 - ### LAST ```sql @@ -2139,6 +2102,35 @@ ignore_option: { - 当没有复合主键时，如果不同的子表有相同时间戳的数据，会提示 "Duplicate timestamps not allowed" - 当使用复合主键时，不同子表的时间戳和主键组合可能相同，使用哪一行取决于先找到哪一行，这意味着在这种情况下多次运行 diff() 的结果可能会不同。 +### INTERP + +```sql +INTERP(expr [, ignore_null_values]) + +ignore_null_values: { + 0 + | 1 +} +``` + +**功能说明**：返回指定时间截面指定列的记录值或插值。ignore_null_values 参数的值可以是 0 或 1，为 1 时表示忽略 NULL 值，缺省值为 0。 + +**返回数据类型**：同字段类型。 + +**适用数据类型**：数值类型。 + +**适用于**：表和超级表。 + +**使用说明** +- INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 [FILL](../distinguished/#fill-子句) 参数的设定进行插值。 +- INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 +- INTERP SQL 查询需要同时与 RANGE、EVERY 和 FILL 关键字一起使用；流计算不能使用 RANGE，需要 EVERY 和 FILL 关键字一起使用。使用规则见 [INTERP](../select/#interp) 子句。 +- INTERP 作用于超级表时，会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。 +- INTERP 可以与伪列 `_irowts` 一起使用，返回插值点所对应的时间戳(v3.0.2.0 以后支持)。 +- INTERP 可以与伪列 `_isfilled` 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(v3.0.3.0 以后支持)。 +- INTERP 只有在使用 [FILL](../distinguished/#fill-子句) PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`, 用于返回 `interp` 函数所使用的原始数据的时间戳列。若范围内无值, 则返回 NULL。`_irowts_origin` 在 v3.3.4.9 以后支持。 +- INTERP 对于带复合主键的表的查询，若存在相同时间戳的数据，则只有对应的复合主键最小的数据参与运算。 + ### IRATE ```sql From 1ccaa8048434f28306ce15864801a6e9e00c6337 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 13 Mar 2025 17:29:13 +0800 Subject: [PATCH 34/44] refactor: break long line into multiple lines for readability Signed-off-by: WANG Xu --- .github/workflows/cancel-workflow-on-merge.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 44b3abdd49..5c45dbc6fe 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -15,4 +15,11 @@ jobs: run: | echo "PR has been merged or closed, cancelling workflow..." gh auth status - gh run list --repo ${{ github.repository }} --branch ${{ github.event.pull_request.head.ref }} --workflow "TDengine Test" --status in_progress --status queued --json databaseId --jq '.[].databaseId' | xargs -I {} gh run cancel --repo ${{ github.repository }} {} \ No newline at end of file + gh run list \ + --repo ${{ github.repository }} \ + --branch ${{ github.event.pull_request.head.ref }} \ + --workflow "TDengine Test" \ + --status in_progress \ + --status queued \ + --json databaseId --jq '.[].databaseId' | \ + xargs -I {} gh run cancel --repo ${{ github.repository }} {} \ No newline at end of file From 98bed1c2a63a236c7c6b1d5ee9d96259eac3de2a Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 13 Mar 2025 19:44:17 +0800 Subject: [PATCH 35/44] fix: remove invalid description Signed-off-by: WANG Xu --- .github/workflows/cancel-workflow-on-merge.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 38a18ec87f..5c45dbc6fe 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -1,5 +1,4 @@ name: Cancel Workflow on Merge or Close -description: Cancels the workflow if the PR is merged or closed to avoid unnecessary runs. on: pull_request: From 32505dacec7cc0dc6ac642952a4a6280c091ef39 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 13 Mar 2025 21:04:08 +0800 Subject: [PATCH 36/44] docs(opc): support request_ts#TS-5728 --- docs/en/06-advanced/05-data-in/05-opcua.md | 66 ++++++++++++---------- docs/en/06-advanced/05-data-in/06-opcda.md | 46 ++++++++------- docs/zh/06-advanced/05-data-in/05-opcua.md | 66 ++++++++++++---------- docs/zh/06-advanced/05-data-in/06-opcda.md | 66 ++++++++++++---------- 4 files changed, 130 insertions(+), 114 deletions(-) diff --git a/docs/en/06-advanced/05-data-in/05-opcua.md b/docs/en/06-advanced/05-data-in/05-opcua.md index d4bdb1d2f5..f2215dfb24 100644 --- a/docs/en/06-advanced/05-data-in/05-opcua.md +++ b/docs/en/06-advanced/05-data-in/05-opcua.md @@ -107,21 +107,23 @@ The header is the first line of the CSV file, with the following rules: (1) The header of the CSV can configure the following columns: -| Number | Column Name | Description | Required | Default Behavior | -| ------ | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | point_id | The id of the data point on the OPC UA server | Yes | None | -| 2 | stable | The corresponding supertable for the data point in TDengine | Yes | None | -| 3 | tbname | The corresponding subtable for the data point in TDengine | Yes | None | -| 4 | enable | Whether to collect data from this point | No | Use the unified default value `1` for enable | -| 5 | value_col | The column name in TDengine corresponding to the collected value of the data point | No | Use the unified default value `val` as the value_col | -| 6 | value_transform | The transformation function executed in taosX for the collected value of the data point | No | Do not transform the collected value uniformly | -| 7 | type | The data type of the collected value of the data point | No | Use the original type of the collected value as the data type in TDengine | -| 8 | quality_col | The column name in TDengine corresponding to the quality of the collected value | No | Do not add a quality column in TDengine uniformly | -| 9 | ts_col | The original timestamp column of the data point in TDengine | No | If both ts_col and received_ts_col are non-empty, use the former as the timestamp column; if one of ts_col or received_ts_col is non-empty, use the non-empty column as the timestamp column; if both are empty, use the original timestamp of the data point as the timestamp column with the default name `ts`. | -| 10 | received_ts_col | The timestamp column in TDengine when the data point value is received | No | Same as above | -| 11 | ts_transform | The transformation function executed in taosX for the original timestamp of the data point | No | Do not transform the original timestamp of the data point uniformly | -| 12 | received_ts_transform | The transformation function executed in taosX for the received timestamp of the data point | No | Do not transform the received timestamp of the data point uniformly | -| 13 | tag::VARCHAR(200)::name | The Tag column corresponding to the data point in TDengine. Here `tag` is a reserved keyword indicating that this column is a tag; `VARCHAR(200)` indicates the type of tag; `name` is the actual name of the tag. | No | If 1 or more tag columns are configured, use the configured tag columns; if no tag columns are configured and stable exists in TDengine, use the tags of the stable in TDengine; if no tag columns are configured and stable does not exist in TDengine, automatically add the following 2 tag columns: tag::VARCHAR(256)::point_id and tag::VARCHAR(256)::point_name | +| Number | Column Name | Description | Required | Default Behavior | +|--------|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1 | point_id | The id of the data point on the OPC UA server | Yes | None | +| 2 | stable | The corresponding supertable for the data point in TDengine | Yes | None | +| 3 | tbname | The corresponding subtable for the data point in TDengine | Yes | None | +| 4 | enable | Whether to collect data from this point | No | Use the unified default value `1` for enable | +| 5 | value_col | The column name in TDengine corresponding to the collected value of the data point | No | Use the unified default value `val` as the value_col | +| 6 | value_transform | The transformation function executed in taosX for the collected value of the data point | No | Do not transform the collected value uniformly | +| 7 | type | The data type of the collected value of the data point | No | Use the original type of the collected value as the data type in TDengine | +| 8 | quality_col | The column name in TDengine corresponding to the quality of the collected value | No | Do not add a quality column in TDengine uniformly | +| 9 | ts_col | The original timestamp column of the data point in TDengine | No | ts_col, request_ts, received_ts these 3 columns, when there are more than 2 columns, the leftmost column is used as the primary key in TDengine. | +| 10 | request_ts_col | The timestamp column in TDengine when the data point value is request | No | Same as above | +| 11 | received_ts_col | The timestamp column in TDengine when the data point value is received | No | Same as above | +| 12 | ts_transform | The transformation function executed in taosX for the original timestamp of the data point | No | Do not transform the original timestamp of the data point uniformly | +| 13 | request_ts_transform | The transformation function executed in taosX for the request timestamp of the data point | No | Do not transform the original timestamp of the data point uniformly | +| 14 | received_ts_transform | The transformation function executed in taosX for the received timestamp of the data point | No | Do not transform the received timestamp of the data point uniformly | +| 15 | tag::VARCHAR(200)::name | The Tag column corresponding to the data point in TDengine. Here `tag` is a reserved keyword indicating that this column is a tag; `VARCHAR(200)` indicates the type of tag; `name` is the actual name of the tag. | No | If 1 or more tag columns are configured, use the configured tag columns; if no tag columns are configured and stable exists in TDengine, use the tags of the stable in TDengine; if no tag columns are configured and stable does not exist in TDengine, automatically add the following 2 tag columns: tag::VARCHAR(256)::point_id and tag::VARCHAR(256)::point_name | (2) In the CSV Header, there cannot be duplicate columns; @@ -137,21 +139,23 @@ Each Row in the CSV file configures an OPC data point. The rules for Rows are as (1) Correspondence with columns in the Header -| Number | Column in Header | Type of Value | Value Range | Mandatory | Default Value | -| ------ | ----------------------- | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------ | -| 1 | point_id | String | Strings like `ns=3;i=1005`, must meet the OPC UA ID specification, i.e., include ns and id parts | Yes | | -| 2 | enable | int | 0: Do not collect this point, and delete the corresponding subtable in TDengine before the OPC DataIn task starts; 1: Collect this point, do not delete the subtable before the OPC DataIn task starts. | No | 1 | -| 3 | stable | String | Any string that meets the TDengine supertable naming convention; if special character `.` exists, replace with underscore if `{type}` exists: if type in CSV file is not empty, replace with the value of type if type is empty, replace with the original type of the collected value | Yes | | -| 4 | tbname | String | Any string that meets the TDengine subtable naming convention; for OPC UA: if `{ns}` exists, replace with ns from point_id if `{id}` exists, replace with id from point_id for OPC DA: if `{tag_name}` exists, replace with tag_name | Yes | | -| 5 | value_col | String | Column name that meets TDengine naming convention | No | val | -| 6 | value_transform | String | Expressions that meet the Rhai engine, for example: `(val + 10) / 1000 * 2.0`, `log(val) + 10`, etc.; | No | None | -| 7 | type | String | Supported types include: b/bool/i8/tinyint/i16/small/inti32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/float/f64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | No | Original type of the data point value | -| 8 | quality_col | String | Column name that meets TDengine naming convention | No | None | -| 9 | ts_col | String | Column name that meets TDengine naming convention | No | ts | -| 10 | received_ts_col | String | Column name that meets TDengine naming convention | No | rts | -| 11 | ts_transform | String | Supports +, -, *, /, % operators, for example: ts / 1000* 1000, sets the last 3 digits of a timestamp in ms to 0; ts + 8 *3600* 1000, adds 8 hours to a timestamp in ms; ts - 8 *3600* 1000, subtracts 8 hours from a timestamp in ms; | No | None | -| 12 | received_ts_transform | String | No | None | | -| 13 | tag::VARCHAR(200)::name | String | The value inside a tag, when the tag type is VARCHAR, can be in Chinese | No | NULL | +| Number | Column in Header | Type of Value | Value Range | Mandatory | Default Value | +|--------|-------------------------| ------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --------- |---------------------------------------| +| 1 | point_id | String | Strings like `ns=3;i=1005`, must meet the OPC UA ID specification, i.e., include ns and id parts | Yes | | +| 2 | enable | int | 0: Do not collect this point, and delete the corresponding subtable in TDengine before the OPC DataIn task starts; 1: Collect this point, do not delete the subtable before the OPC DataIn task starts. | No | 1 | +| 3 | stable | String | Any string that meets the TDengine supertable naming convention; if special character `.` exists, replace with underscore if `{type}` exists: if type in CSV file is not empty, replace with the value of type if type is empty, replace with the original type of the collected value | Yes | | +| 4 | tbname | String | Any string that meets the TDengine subtable naming convention; for OPC UA: if `{ns}` exists, replace with ns from point_id if `{id}` exists, replace with id from point_id for OPC DA: if `{tag_name}` exists, replace with tag_name | Yes | | +| 5 | value_col | String | Column name that meets TDengine naming convention | No | val | +| 6 | value_transform | String | Expressions that meet the Rhai engine, for example: `(val + 10) / 1000 * 2.0`, `log(val) + 10`, etc.; | No | None | +| 7 | type | String | Supported types include: b/bool/i8/tinyint/i16/small/inti32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/float/f64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | No | Original type of the data point value | +| 8 | quality_col | String | Column name that meets TDengine naming convention | No | None | +| 9 | ts_col | String | Column name that meets TDengine naming convention | No | ts | +| 10 | request_ts_col | String | Column name that meets TDengine naming convention | No | qts | +| 11 | received_ts_col | String | Column name that meets TDengine naming convention | No | rts | +| 12 | ts_transform | String | Supports +, -, *, /, % operators, for example: ts / 1000* 1000, sets the last 3 digits of a timestamp in ms to 0; ts + 8 *3600* 1000, adds 8 hours to a timestamp in ms; ts - 8 *3600* 1000, subtracts 8 hours from a timestamp in ms; | No | None | +| 13 | request_ts_transform | String | Supports +, -, *, /, % operators, for example: qts / 1000* 1000, sets the last 3 digits of a timestamp in ms to 0; qts + 8 *3600* 1000, adds 8 hours to a timestamp in ms; qts - 8 *3600* 1000, subtracts 8 hours from a timestamp in ms; | No | None | +| 14 | received_ts_transform | String | Supports +, -, *, /, % operators, for example: qts / 1000* 1000, sets the last 3 digits of a timestamp in ms to 0; qts + 8 *3600* 1000, adds 8 hours to a timestamp in ms; qts - 8 *3600* 1000, subtracts 8 hours from a timestamp in ms; | None | None | +| 15 | tag::VARCHAR(200)::name | String | The value inside a tag, when the tag type is VARCHAR, can be in Chinese | No | NULL | (2) `point_id` is unique throughout the DataIn task, meaning: in an OPC DataIn task, a data point can only be written to one subtable in TDengine. If you need to write a data point to multiple subtables, you need to create multiple OPC DataIn tasks; @@ -171,7 +175,7 @@ Data points can be filtered by configuring **Root Node ID**, **Namespace**, **Re Configure **Supertable Name**, **Table Name** to specify the supertable and subtable where the data will be written. -Configure **Primary Key Column**, choose `origin_ts` to use the original timestamp of the OPC data point as the primary key in TDengine; choose `received_ts` to use the data's reception timestamp as the primary key in TDengine. Configure **Primary Key Alias** to specify the name of the TDengine timestamp column. +Configure **Primary Key Column**, choose `origin_ts` to use the original timestamp of the OPC data point as the primary key in TDengine; choose `request_ts` to use the data's request timestamp as the primary key in TDengine; choose `received_ts` to use the data's reception timestamp as the primary key in TDengine. Configure **Primary Key Alias** to specify the name of the TDengine timestamp column.
diff --git a/docs/en/06-advanced/05-data-in/06-opcda.md b/docs/en/06-advanced/05-data-in/06-opcda.md index 4d692f413f..84de34c7ac 100644 --- a/docs/en/06-advanced/05-data-in/06-opcda.md +++ b/docs/en/06-advanced/05-data-in/06-opcda.md @@ -81,21 +81,23 @@ The header is the first line of the CSV file, with the following rules: (1) The header of the CSV can configure the following columns: -| No. | Column Name | Description | Required | Default Behavior | -| ---- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | tag_name | The id of the data point on the OPC DA server | Yes | None | -| 2 | stable | The supertable in TDengine corresponding to the data point | Yes | None | -| 3 | tbname | The subtable in TDengine corresponding to the data point | Yes | None | -| 4 | enable | Whether to collect data from this point | No | Use a unified default value `1` for enable | -| 5 | value_col | The column name in TDengine corresponding to the collected value of the data point | No | Use a unified default value `val` as the value_col | -| 6 | value_transform | The transform function executed in taosX for the collected value of the data point | No | Do not perform a transform on the collected value | -| 7 | type | The data type of the collected value of the data point | No | Use the original type of the collected value as the data type in TDengine | -| 8 | quality_col | The column name in TDengine corresponding to the quality of the collected value | No | Do not add a quality column in TDengine | -| 9 | ts_col | The timestamp column in TDengine corresponding to the original timestamp of the data point | No | If both ts_col and received_ts_col are non-empty, use the former as the timestamp column; if one of ts_col or received_ts_col is non-empty, use the non-empty column as the timestamp column; if both are empty, use the original timestamp of the data point as the timestamp column in TDengine, with the default column name ts. | -| 10 | received_ts_col | The timestamp column in TDengine corresponding to the timestamp when the data point value was received | No | | -| 11 | ts_transform | The transform function executed in taosX for the original timestamp of the data point | No | Do not perform a transform on the original timestamp of the data point | -| 12 | received_ts_transform | The transform function executed in taosX for the received timestamp of the data point | No | Do not perform a transform on the received timestamp of the data point | -| 13 | tag::VARCHAR(200)::name | The Tag column in TDengine corresponding to the data point. Where `tag` is a reserved keyword, indicating that this column is a tag column; `VARCHAR(200)` indicates the type of this tag, which can also be other legal types; `name` is the actual name of this tag. | No | If configuring more than one tag column, use the configured tag columns; if no tag columns are configured, and stable exists in TDengine, use the tags of the stable in TDengine; if no tag columns are configured, and stable does not exist in TDengine, automatically add the following two tag columns by default: tag::VARCHAR(256)::point_idtag::VARCHAR(256)::point_name | +| No. | Column Name | Description | Required | Default Behavior | +|-----|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 1 | tag_name | The id of the data point on the OPC DA server | Yes | None | +| 2 | stable | The supertable in TDengine corresponding to the data point | Yes | None | +| 3 | tbname | The subtable in TDengine corresponding to the data point | Yes | None | +| 4 | enable | Whether to collect data from this point | No | Use a unified default value `1` for enable | +| 5 | value_col | The column name in TDengine corresponding to the collected value of the data point | No | Use a unified default value `val` as the value_col | +| 6 | value_transform | The transform function executed in taosX for the collected value of the data point | No | Do not perform a transform on the collected value | +| 7 | type | The data type of the collected value of the data point | No | Use the original type of the collected value as the data type in TDengine | +| 8 | quality_col | The column name in TDengine corresponding to the quality of the collected value | No | Do not add a quality column in TDengine | +| 9 | ts_col | The timestamp column in TDengine corresponding to the original timestamp of the data point | No | ts_col, request_ts, received_ts these 3 columns, when there are more than 2 columns, the leftmost column is used as the primary key in TDengine. | +| 10 | request_ts_col | The timestamp column in TDengine corresponding to the timestamp when the data point value was request | No | Same as above | +| 11 | received_ts_col | The timestamp column in TDengine corresponding to the timestamp when the data point value was received | No | Same as above | +| 12 | ts_transform | The transform function executed in taosX for the original timestamp of the data point | No | Do not perform a transform on the original timestamp of the data point | +| 13 | request_ts_transform | The transform function executed in taosX for the request timestamp of the data point | No | Do not perform a transform on the received timestamp of the data point | +| 14 | received_ts_transform | The transform function executed in taosX for the received timestamp of the data point | No | Do not perform a transform on the received timestamp of the data point | +| 15 | tag::VARCHAR(200)::name | The Tag column in TDengine corresponding to the data point. Where `tag` is a reserved keyword, indicating that this column is a tag column; `VARCHAR(200)` indicates the type of this tag, which can also be other legal types; `name` is the actual name of this tag. | No | If configuring more than one tag column, use the configured tag columns; if no tag columns are configured, and stable exists in TDengine, use the tags of the stable in TDengine; if no tag columns are configured, and stable does not exist in TDengine, automatically add the following two tag columns by default: tag::VARCHAR(256)::point_idtag::VARCHAR(256)::point_name | (2) In the CSV Header, there cannot be duplicate columns; @@ -112,7 +114,7 @@ Each Row in the CSV file configures an OPC data point. The rules for Rows are as (1) Correspondence with columns in the Header | Number | Column in Header | Type of Value | Range of Values | Mandatory | Default Value | -| ------ | ----------------------- | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------ | +|--------|-------------------------| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | ------------------------ | | 1 | tag_name | String | Strings like `root.parent.temperature`, must meet the OPC DA ID specification | Yes | | | 2 | enable | int | 0: Do not collect this point, and delete the corresponding subtable in TDengine before the OPC DataIn task starts; 1: Collect this point, do not delete the subtable before the OPC DataIn task starts. | No | 1 | | 3 | stable | String | Any string that meets the TDengine supertable naming convention; if there are special characters `.`, replace with underscore. If `{type}` exists: if type in CSV file is not empty, replace with the value of type; if empty, replace with the original type of the collected value | Yes | | @@ -122,10 +124,12 @@ Each Row in the CSV file configures an OPC data point. The rules for Rows are as | 7 | type | String | Supported types include: b/bool/i8/tinyint/i16/smallint/i32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/floatf64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | No | Original type of data point value | | 8 | quality_col | String | Column name that meets TDengine naming convention | No | None | | 9 | ts_col | String | Column name that meets TDengine naming convention | No | ts | -| 10 | received_ts_col | String | Column name that meets TDengine naming convention | No | rts | -| 11 | ts_transform | String | Supports +, -, *, /, % operators, for example: ts / 1000* 1000, sets the last 3 digits of a ms unit timestamp to 0; ts + 8 *3600* 1000, adds 8 hours to a ms precision timestamp; ts - 8 *3600* 1000, subtracts 8 hours from a ms precision timestamp; | No | None | -| 12 | received_ts_transform | String | No | None | | -| 13 | tag::VARCHAR(200)::name | String | The value in tag, when the tag type is VARCHAR, it can be in Chinese | No | NULL | +| 10 | request_ts_col | String | Column name that meets TDengine naming convention | No | rts | +| 11 | received_ts_col | String | Column name that meets TDengine naming convention | No | rts | +| 12 | ts_transform | String | Supports +, -, *, /, % operators, for example: ts / 1000* 1000, sets the last 3 digits of a ms unit timestamp to 0; ts + 8 *3600* 1000, adds 8 hours to a ms precision timestamp; ts - 8 *3600* 1000, subtracts 8 hours from a ms precision timestamp; | No | None | +| 13 | request_ts_transform | String | No | None | | +| 14 | received_ts_transform | String | No | None | | +| 15 | tag::VARCHAR(200)::name | String | The value in tag, when the tag type is VARCHAR, it can be in Chinese | No | NULL | (2) `tag_name` is unique throughout the DataIn task, that is: in an OPC DataIn task, a data point can only be written to one subtable in TDengine. If you need to write a data point to multiple subtables, you need to create multiple OPC DataIn tasks; @@ -145,7 +149,7 @@ Data points can be filtered by configuring the **Root Node ID** and **Regular Ex Configure **Supertable Name** and **Table Name** to specify the supertable and subtable where the data will be written. -Configure **Primary Key Column**, choosing `origin_ts` to use the original timestamp of the OPC data point as the primary key in TDengine; choosing `received_ts` to use the timestamp when the data is received as the primary key. Configure **Primary Key Alias** to specify the name of the TDengine timestamp column. +Configure **Primary Key Column**, choosing `origin_ts` to use the original timestamp of the OPC data point as the primary key in TDengine; choosing `request_ts` to use the timestamp when the data is request as the primary key; choosing `received_ts` to use the timestamp when the data is received as the primary key. Configure **Primary Key Alias** to specify the name of the TDengine timestamp column.
diff --git a/docs/zh/06-advanced/05-data-in/05-opcua.md b/docs/zh/06-advanced/05-data-in/05-opcua.md index 5123dacd1b..fbb025dfc1 100644 --- a/docs/zh/06-advanced/05-data-in/05-opcua.md +++ b/docs/zh/06-advanced/05-data-in/05-opcua.md @@ -89,21 +89,23 @@ Header 是 CSV 文件的第一行，规则如下： (1) CSV 的 Header 中可以配置以下列： -| 序号 | 列名 | 描述 | 是否必填 | 默认行为 | -| ---- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | point_id | 数据点位在 OPC UA 服务器上的 id | 是 | 无 | -| 2 | stable | 数据点位在 TDengine 中对应的超级表 | 是 | 无 | -| 3 | tbname | 数据点位在 TDengine 中对应的子表 | 是 | 无 | -| 4 | enable | 是否采集该点位的数据 | 否 | 使用统一的默认值`1`作为 enable 的值 | -| 5 | value_col | 数据点位采集值在 TDengine 中对应的列名 | 否 | 使用统一的默认值`val` 作为 value_col 的值 | -| 6 | value_transform | 数据点位采集值在 taosX 中执行的变换函数 | 否 | 统一不进行采集值的 transform | -| 7 | type | 数据点位采集值的数据类型 | 否 | 统一使用采集值的原始类型作为 TDengine 中的数据类型 | -| 8 | quality_col | 数据点位采集值质量在 TDengine 中对应的列名 | 否 | 统一不在 TDengine 添加 quality 列 | -| 9 | ts_col | 数据点位的原始时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col 和 received_ts_col 都非空时，使用前者作为时间戳列；ts_col 和 received_ts_col 有一列非空时，使用不为空的列作时间戳列；ts_col 和 received_ts_col 都为空时，使用数据点位原始时间戳作 TDengine 中的时间戳列，且列名为默认值`ts`。 | -| 10 | received_ts_col | 接收到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | 同上 | -| 11 | ts_transform | 数据点位时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform | -| 12 | received_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform | -| 13 | tag::VARCHAR(200)::name | 数据点位在 TDengine 中对应的 Tag 列。其中`tag` 为保留关键字，表示该列为一个 tag 列；`VARCHAR(200)` 表示该 tag 的类型，也可以是其它合法的类型；`name` 是该 tag 的实际名称。 | 否 | 配置 1 个以上的 tag 列，则使用配置的 tag 列；没有配置任何 tag 列，且 stable 在 TDengine 中存在，使用 TDengine 中的 stable 的 tag；没有配置任何 tag 列，且 stable 在 TDengine 中不存在，则默认自动添加以下 2 个 tag 列：tag::VARCHAR(256)::point_id 和 tag::VARCHAR(256)::point_name | +| 序号 | 列名 | 描述 | 是否必填 | 默认行为 | +|----|-------------------------|--------------------------------------------------------------------------------------------------------------------| -------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1 | point_id | 数据点位在 OPC UA 服务器上的 id | 是 | 无 | +| 2 | stable | 数据点位在 TDengine 中对应的超级表 | 是 | 无 | +| 3 | tbname | 数据点位在 TDengine 中对应的子表 | 是 | 无 | +| 4 | enable | 是否采集该点位的数据 | 否 | 使用统一的默认值`1`作为 enable 的值 | +| 5 | value_col | 数据点位采集值在 TDengine 中对应的列名 | 否 | 使用统一的默认值`val` 作为 value_col 的值 | +| 6 | value_transform | 数据点位采集值在 taosX 中执行的变换函数 | 否 | 统一不进行采集值的 transform | +| 7 | type | 数据点位采集值的数据类型 | 否 | 统一使用采集值的原始类型作为 TDengine 中的数据类型 | +| 8 | quality_col | 数据点位采集值质量在 TDengine 中对应的列名 | 否 | 统一不在 TDengine 添加 quality 列 | +| 9 | ts_col | 数据点位的原始时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键。 | +| 10 | request_ts_col | 请求到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键。 | +| 11 | received_ts_col | 接收到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键。 | +| 12 | ts_transform | 数据点位时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform | +| 13 | request_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform | +| 14 | received_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform | +| 15 | tag::VARCHAR(200)::name | 数据点位在 TDengine 中对应的 Tag 列。其中`tag` 为保留关键字，表示该列为一个 tag 列；`VARCHAR(200)` 表示该 tag 的类型，也可以是其它合法的类型；`name` 是该 tag 的实际名称。 | 否 | 配置 1 个以上的 tag 列，则使用配置的 tag 列；没有配置任何 tag 列，且 stable 在 TDengine 中存在，使用 TDengine 中的 stable 的 tag；没有配置任何 tag 列，且 stable 在 TDengine 中不存在，则默认自动添加以下 2 个 tag 列：tag::VARCHAR(256)::point_id 和 tag::VARCHAR(256)::point_name | (2) CSV Header 中，不能有重复的列； @@ -120,21 +122,23 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 (1) 与 Header 中的列有如下对应关系 -| 序号 | Header 中的列 | 值的类型 | 值的范围 | 是否必填 | 默认值 | -| ---- | ----------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------ | -| 1 | point_id | String | 类似`ns=3;i=1005`这样的字符串，要满足 OPC UA 的 ID 的规范，即：包含 ns 和 id 部分 | 是 | | -| 2 | enable | int | 0：不采集该点位，且在 OPC DataIn 任务开始前，删除 TDengine 中点位对应的子表；1：采集该点位，在 OPC DataIn 任务开始前，不删除子表。 | 否 | 1 | -| 3 | stable | String | 符合 TDengine 超级表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换如果存在`{type}`，则：CSV 文件的 type 不为空，使用 type 的值进行替换CSV 文件的 type 为空，使用采集值的原始类型进行替换 | 是 | | -| 4 | tbname | String | 符合 TDengine 子表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换对于 OPC UA：如果存在`{ns}`，使用 point_id 中的 ns 替换如果存在`{id}`，使用 point_id 中的 id 替换对于 OPC DA：如果存在`{tag_name}`，使用 tag_name 替换 | 是 | | -| 5 | value_col | String | 符合 TDengine 命名规范的列名 | 否 | val | -| 6 | value_transform | String | 符合 Rhai 引擎的计算表达式，例如：`(val + 10) / 1000 * 2.0`，`log(val) + 10`等； | 否 | None | -| 7 | type | String | 支持类型包括：b/bool/i8/tinyint/i16/small/inti32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/float/f64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | 否 | 数据点位采集值的原始类型 | -| 8 | quality_col | String | 符合 TDengine 命名规范的列名 | 否 | None | -| 9 | ts_col | String | 符合 TDengine 命名规范的列名 | 否 | ts | -| 10 | received_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts | -| 11 | ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；ts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；ts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None | -| 12 | received_ts_transform | String | 否 | None | | -| 13 | tag::VARCHAR(200)::name | String | tag 里的值，当 tag 的类型是 VARCHAR 时，可以是中文 | 否 | NULL | +| 序号 | Header 中的列 | 值的类型 | 值的范围 | 是否必填 | 默认值 | +|----|-------------------------| -------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------| ------------------------ | +| 1 | point_id | String | 类似`ns=3;i=1005`这样的字符串，要满足 OPC UA 的 ID 的规范，即：包含 ns 和 id 部分 | 是 | | +| 2 | enable | int | 0：不采集该点位，且在 OPC DataIn 任务开始前，删除 TDengine 中点位对应的子表；1：采集该点位，在 OPC DataIn 任务开始前，不删除子表。 | 否 | 1 | +| 3 | stable | String | 符合 TDengine 超级表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换如果存在`{type}`，则：CSV 文件的 type 不为空，使用 type 的值进行替换CSV 文件的 type 为空，使用采集值的原始类型进行替换 | 是 | | +| 4 | tbname | String | 符合 TDengine 子表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换对于 OPC UA：如果存在`{ns}`，使用 point_id 中的 ns 替换如果存在`{id}`，使用 point_id 中的 id 替换对于 OPC DA：如果存在`{tag_name}`，使用 tag_name 替换 | 是 | | +| 5 | value_col | String | 符合 TDengine 命名规范的列名 | 否 | val | +| 6 | value_transform | String | 符合 Rhai 引擎的计算表达式，例如：`(val + 10) / 1000 * 2.0`，`log(val) + 10`等； | 否 | None | +| 7 | type | String | 支持类型包括：b/bool/i8/tinyint/i16/small/inti32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/float/f64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | 否 | 数据点位采集值的原始类型 | +| 8 | quality_col | String | 符合 TDengine 命名规范的列名 | 否 | None | +| 9 | ts_col | String | 符合 TDengine 命名规范的列名 | 否 | ts | +| 10 | request_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts | +| 11 | received_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts | +| 12 | ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；ts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；ts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None | +| 13 | request_ts_transform | String | 支持 +、-、*、/、% 操作符，例如：qts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；qts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；qts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None | +| 14 | received_ts_transform | String | 支持 +、-、*、/、% 操作符，例如：rts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；rts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；rts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None | +| 15 | tag::VARCHAR(200)::name | String | tag 里的值，当 tag 的类型是 VARCHAR 时，可以是中文 | 否 | NULL | (2) point_id 在整个 DataIn 任务中是唯一的，即：在一个 OPC DataIn 任务中，一个数据点位只能被写入到 TDengine 的一张子表。如果需要将一个数据点位写入多张子表，需要建多个 OPC DataIn 任务； @@ -154,7 +158,7 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 通过配置 **超级表名**、**表名称**，指定数据要写入的超级表、子表。 -配置**主键列**，选择 origin_ts 表示使用 OPC 点位数据的原始时间戳作 TDengine 中的主键；选择 received_ts 表示使用数据的接收时间戳作 TDengine 中的主键。配置**主键别名**，指定 TDengine 时间戳列的名称。 +配置**主键列**，选择 origin_ts 表示使用 OPC 点位数据的原始时间戳作 TDengine 中的主键；选择 request_ts 表示使用数据的请求时间戳作 TDengine 中的主键；选择 received_ts 表示使用数据的接收时间戳作 TDengine 中的主键。配置**主键别名**，指定 TDengine 时间戳列的名称。 ![point.png](./pic/opcua-06-point.png) diff --git a/docs/zh/06-advanced/05-data-in/06-opcda.md b/docs/zh/06-advanced/05-data-in/06-opcda.md index 32ac1c1f8a..93ce0ba15a 100644 --- a/docs/zh/06-advanced/05-data-in/06-opcda.md +++ b/docs/zh/06-advanced/05-data-in/06-opcda.md @@ -65,21 +65,23 @@ Header 是 CSV 文件的第一行，规则如下： (1) CSV 的 Header 中可以配置以下列： -| 序号 | 列名 | 描述 | 是否必填 | 默认行为 | -| ---- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | tag_name | 数据点位在 OPC DA 服务器上的 id | 是 | 无 | -| 2 | stable | 数据点位在 TDengine 中对应的超级表 | 是 | 无 | -| 3 | tbname | 数据点位在 TDengine 中对应的子表 | 是 | 无 | -| 4 | enable | 是否采集该点位的数据 | 否 | 使用统一的默认值`1`作为 enable 的值 | -| 5 | value_col | 数据点位采集值在 TDengine 中对应的列名 | 否 | 使用统一的默认值`val` 作为 value_col 的值 | -| 6 | value_transform | 数据点位采集值在 taosX 中执行的变换函数 | 否 | 统一不进行采集值的 transform | -| 7 | type | 数据点位采集值的数据类型 | 否 | 统一使用采集值的原始类型作为 TDengine 中的数据类型 | -| 8 | quality_col | 数据点位采集值质量在 TDengine 中对应的列名 | 否 | 统一不在 TDengine 添加 quality 列 | -| 9 | ts_col | 数据点位的原始时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col 和 received_ts_col 都非空时，使用前者作为时间戳列；ts_col 和 received_ts_col 有一列非空时，使用不为空的列作时间戳列；ts_col 和 received_ts_col 都为空时，使用数据点位原始时间戳作 TDengine 中的时间戳列，且列名为默认值ts。 | -| 10 | received_ts_col | 接收到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | | -| 11 | ts_transform | 数据点位时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform | -| 12 | received_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform | -| 13 | tag::VARCHAR(200)::name | 数据点位在 TDengine 中对应的 Tag 列。其中`tag` 为保留关键字，表示该列为一个 tag 列；`VARCHAR(200)` 表示该 tag 的类型，也可以是其它合法的类型；`name` 是该 tag 的实际名称。 | 否 | 配置 1 个以上的 tag 列，则使用配置的 tag 列；没有配置任何 tag 列，且 stable 在 TDengine 中存在，使用 TDengine 中的 stable 的 tag；没有配置任何 tag 列，且 stable 在 TDengine 中不存在，则默认自动添加以下 2 个 tag 列：tag::VARCHAR(256)::point_idtag::VARCHAR(256)::point_name | +| 序号 | 列名 | 描述 | 是否必填 | 默认行为 | +|----|-------------------------|-------------------------------------------------------------------------------------------------------------------| -------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1 | tag_name | 数据点位在 OPC DA 服务器上的 id | 是 | 无 | +| 2 | stable | 数据点位在 TDengine 中对应的超级表 | 是 | 无 | +| 3 | tbname | 数据点位在 TDengine 中对应的子表 | 是 | 无 | +| 4 | enable | 是否采集该点位的数据 | 否 | 使用统一的默认值`1`作为 enable 的值 | +| 5 | value_col | 数据点位采集值在 TDengine 中对应的列名 | 否 | 使用统一的默认值`val` 作为 value_col 的值 | +| 6 | value_transform | 数据点位采集值在 taosX 中执行的变换函数 | 否 | 统一不进行采集值的 transform | +| 7 | type | 数据点位采集值的数据类型 | 否 | 统一使用采集值的原始类型作为 TDengine 中的数据类型 | +| 8 | quality_col | 数据点位采集值质量在 TDengine 中对应的列名 | 否 | 统一不在 TDengine 添加 quality 列 | +| 9 | ts_col | 数据点位的原始时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键 | +| 10 | request_ts_col | 请求该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键 | +| 11 | received_ts_col | 接收到该点位采集值时的时间戳在 TDengine 中对应的时间戳列 | 否 | ts_col，request_ts，received_ts 这 3 列，当有 2 列以上存在时，以最左侧的列作为 TDengine 中的主键 | +| 12 | ts_transform | 数据点位时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform | +| 13 | request_ts_transform | 数据点位请求时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位原始时间戳的 transform | +| 14 | received_ts_transform | 数据点位接收时间戳在 taosX 中执行的变换函数 | 否 | 统一不进行数据点位接收时间戳的 transform | +| 15 | tag::VARCHAR(200)::name | 数据点位在 TDengine 中对应的 Tag 列。其中`tag` 为保留关键字，表示该列为一个 tag 列；`VARCHAR(200)` 表示该 tag 的类型，也可以是其它合法的类型；`name` 是该 tag 的实际名称。 | 否 | 配置 1 个以上的 tag 列，则使用配置的 tag 列；没有配置任何 tag 列，且 stable 在 TDengine 中存在，使用 TDengine 中的 stable 的 tag；没有配置任何 tag 列，且 stable 在 TDengine 中不存在，则默认自动添加以下 2 个 tag 列：tag::VARCHAR(256)::point_idtag::VARCHAR(256)::point_name | (2) CSV Header 中，不能有重复的列； @@ -96,21 +98,23 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 (1) 与 Header 中的列有如下对应关系 -| 序号 | Header 中的列 | 值的类型 | 值的范围 | 是否必填 | 默认值 | -| ---- | ----------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------ | -| 1 | tag_name | String | 类似`root.parent.temperature`这样的字符串，要满足 OPC DA 的 ID 规范 | 是 | | -| 2 | enable | int | 0：不采集该点位，且在 OPC DataIn 任务开始前，删除 TDengine 中点位对应的子表；1：采集该点位，在 OPC DataIn 任务开始前，不删除子表。 | 否 | 1 | -| 3 | stable | String | 符合 TDengine 超级表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换如果存在`{type}`，则：CSV 文件的 type 不为空，使用 type 的值进行替换CSV 文件的 type 为空，使用采集值的原始类型进行替换 | 是 | | -| 4 | tbname | String | 符合 TDengine 子表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换对于 OPC UA：如果存在`{ns}`，使用 point_id 中的 ns 替换如果存在`{id}`，使用 point_id 中的 id 替换对于 OPC DA：如果存在`{tag_name}`，使用 tag_name 替换 | 是 | | -| 5 | value_col | String | 符合 TDengine 命名规范的列名 | 否 | val | -| 6 | value_transform | String | 符合 Rhai 引擎的计算表达式，例如：`(val + 10) / 1000 * 2.0`，`log(val) + 10`等； | 否 | None | -| 7 | type | String | 支持类型包括：b/bool/i8/tinyint/i16/smallint/i32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/floatf64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | 否 | 数据点位采集值的原始类型 | -| 8 | quality_col | String | 符合 TDengine 命名规范的列名 | 否 | None | -| 9 | ts_col | String | 符合 TDengine 命名规范的列名 | 否 | ts | -| 10 | received_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts | -| 11 | ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；ts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；ts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None | -| 12 | received_ts_transform | String | 否 | None | | -| 13 | tag::VARCHAR(200)::name | String | tag 里的值，当 tag 的类型是 VARCHAR 时，可以是中文 | 否 | NULL | +| 序号 | Header 中的列 | 值的类型 | 值的范围 | 是否必填 | 默认值 | +|----|-------------------------| -------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------|--------------| +| 1 | tag_name | String | 类似`root.parent.temperature`这样的字符串，要满足 OPC DA 的 ID 规范 | 是 | | +| 2 | enable | int | 0：不采集该点位，且在 OPC DataIn 任务开始前，删除 TDengine 中点位对应的子表；1：采集该点位，在 OPC DataIn 任务开始前，不删除子表。 | 否 | 1 | +| 3 | stable | String | 符合 TDengine 超级表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换如果存在`{type}`，则：CSV 文件的 type 不为空，使用 type 的值进行替换CSV 文件的 type 为空，使用采集值的原始类型进行替换 | 是 | | +| 4 | tbname | String | 符合 TDengine 子表命名规范的任何字符串；如果存在特殊字符`.`，使用下划线替换对于 OPC UA：如果存在`{ns}`，使用 point_id 中的 ns 替换如果存在`{id}`，使用 point_id 中的 id 替换对于 OPC DA：如果存在`{tag_name}`，使用 tag_name 替换 | 是 | | +| 5 | value_col | String | 符合 TDengine 命名规范的列名 | 否 | val | +| 6 | value_transform | String | 符合 Rhai 引擎的计算表达式，例如：`(val + 10) / 1000 * 2.0`，`log(val) + 10`等； | 否 | None | +| 7 | type | String | 支持类型包括：b/bool/i8/tinyint/i16/smallint/i32/int/i64/bigint/u8/tinyint unsigned/u16/smallint unsigned/u32/int unsigned/u64/bigint unsigned/f32/floatf64/double/timestamp/timestamp(ms)/timestamp(us)/timestamp(ns)/json | 否 | 数据点位采集值的原始类型 | +| 8 | quality_col | String | 符合 TDengine 命名规范的列名 | 否 | None | +| 9 | ts_col | String | 符合 TDengine 命名规范的列名 | 否 | ts | +| 10 | request_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | qts | +| 11 | received_ts_col | String | 符合 TDengine 命名规范的列名 | 否 | rts | +| 12 | ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；ts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；ts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None | +| 13 | request_ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；qts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；qts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None | +| 14 | received_ts_transform | String | 支持 +、-、*、/、% 操作符，例如：ts / 1000 * 1000，将一个 ms 单位的时间戳的最后 3 位置为 0；rts + 8 * 3600 * 1000，将一个 ms 精度的时间戳，增加 8 小时；rts - 8 * 3600 * 1000，将一个 ms 精度的时间戳，减去 8 小时； | 否 | None | +| 15 | tag::VARCHAR(200)::name | String | tag 里的值，当 tag 的类型是 VARCHAR 时，可以是中文 | 否 | NULL | (2) tag_name 在整个 DataIn 任务中是唯一的，即：在一个 OPC DataIn 任务中，一个数据点位只能被写入到 TDengine 的一张子表。如果需要将一个数据点位写入多张子表，需要建多个 OPC DataIn 任务； @@ -130,7 +134,7 @@ CSV 文件中的每个 Row 配置一个 OPC 数据点位。Row 的规则如下 通过配置 **超级表名**、**表名称**，指定数据要写入的超级表、子表。 -配置**主键列**，选择 origin_ts 表示使用 OPC 点位数据的原始时间戳作 TDengine 中的主键；选择 received_ts 表示使用数据的接收时间戳作 TDengine 中的主键。配置**主键别名**，指定 TDengine 时间戳列的名称。 +配置**主键列**，选择 origin_ts 表示使用 OPC 点位数据的原始时间戳作 TDengine 中的主键；选择 request_ts 表示使用数据的请求时间戳作 TDengine 中的主键；选择 received_ts 表示使用数据的接收时间戳作 TDengine 中的主键。配置**主键别名**，指定 TDengine 时间戳列的名称。 ![point.png](./pic/opcda-06-point.png) From b69faedd7b4a5db356d10324a49e0fbeb64742f5 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Fri, 14 Mar 2025 07:48:15 +0800 Subject: [PATCH 37/44] =?UTF-8?q?=E6=96=B0=E5=A2=9E=20TDgpt=20CI=20?= =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E4=BB=A5=E6=94=AF=E6=8C=81=E5=8D=95?= =?UTF-8?q?=E5=85=83=E6=B5=8B=E8=AF=95=E5=92=8C=E4=BB=A3=E7=A0=81=E6=A3=80?= =?UTF-8?q?=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tdgpt-ci.yml | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/tdgpt-ci.yml diff --git a/.github/workflows/tdgpt-ci.yml b/.github/workflows/tdgpt-ci.yml new file mode 100644 index 0000000000..7c579e1692 --- /dev/null +++ b/.github/workflows/tdgpt-ci.yml @@ -0,0 +1,51 @@ +name: TDgpt CI + +on: + pull_request: + branches: + - '3.0' + paths: + - 'tools/tdgpt/**' + +jobs: + unit-test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + + defaults: + run: + working-directory: ${{ github.workspace }}/tools/tdgpt + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest pylint + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Checking the code with pylint + run: | + pylint $(git ls-files '*.py') --exit-zero + + - name: Checking the code with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Run test cases with pytest + run: | + pytest From 44c1643ce2ccd69cb5a469864a34bc2a3feb1a94 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Fri, 14 Mar 2025 08:05:35 +0800 Subject: [PATCH 38/44] ci: Delete .github/workflows/tdgpt-ci.yml --- .github/workflows/tdgpt-ci.yml | 51 ---------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 .github/workflows/tdgpt-ci.yml diff --git a/.github/workflows/tdgpt-ci.yml b/.github/workflows/tdgpt-ci.yml deleted file mode 100644 index 7c579e1692..0000000000 --- a/.github/workflows/tdgpt-ci.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: TDgpt CI - -on: - pull_request: - branches: - - '3.0' - paths: - - 'tools/tdgpt/**' - -jobs: - unit-test: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: ["3.10"] - - defaults: - run: - working-directory: ${{ github.workspace }}/tools/tdgpt - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest pylint - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - - name: Checking the code with pylint - run: | - pylint $(git ls-files '*.py') --exit-zero - - - name: Checking the code with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Run test cases with pytest - run: | - pytest From 08a85d05f0e6f772cd549169da6c1e47f69aef07 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 14 Mar 2025 09:40:11 +0800 Subject: [PATCH 39/44] docs: interp ins --- docs/en/14-reference/03-taos-sql/06-select.md | 2 +- docs/en/14-reference/03-taos-sql/10-function.md | 8 +++----- docs/zh/14-reference/03-taos-sql/06-select.md | 3 ++- docs/zh/14-reference/03-taos-sql/10-function.md | 8 ++++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/en/14-reference/03-taos-sql/06-select.md b/docs/en/14-reference/03-taos-sql/06-select.md index 98559104c0..2c2b0de8fb 100644 --- a/docs/en/14-reference/03-taos-sql/06-select.md +++ b/docs/en/14-reference/03-taos-sql/06-select.md @@ -278,7 +278,7 @@ TDengine supports INNER JOIN based on the timestamp primary key, with the follow 6. Does not support mixing with the FILL clause. ## INTERP -The INTERP clause is used in conjunction with the [INTERP](../function/#interp) function to determine the rules for capturing time slices and interpolation when the INTERP function is executed. +The INTERP clause is a dedicated syntax for the INTERP function (../function/#interp). When an SQL statement contains an INTERP clause, it can only query the INTERP function and cannot be used with other functions. Additionally, the INTERP clause cannot be used simultaneously with window clauses (window_clause) or group by clauses (group_by_clause). The INTERP function must be used with the RANGE, EVERY, and FILL clauses; stream computing does not support the use of RANGE but requires the use of the EVERY and FILL keywords. - The output time range for INTERP is specified by the RANGE(timestamp1, timestamp2) field, which must satisfy timestamp1 \<= timestamp2. Here, timestamp1 is the start value of the output time range, i.e., if the conditions for interpolation are met at timestamp1, then timestamp1 is the first record output, and timestamp2 is the end value of the output time range, i.e., the timestamp of the last record output cannot be greater than timestamp2. - INTERP determines the number of results within the output time range based on the EVERY(time_unit) field, starting from timestamp1 and interpolating at fixed intervals of time (time_unit value), where time_unit can be time units: 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), 1w (weeks). For example, EVERY(500a) will interpolate the specified data every 500 milliseconds. - INTERP determines how to interpolate at each time point that meets the output conditions based on the FILL field. For how to use the FILL clause, refer to [FILL Clause](../time-series-extensions/) diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index 27b5d2a577..cf143ce759 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -2196,16 +2196,14 @@ ignore_null_values: { **Usage Instructions** -- INTERP is used to obtain the record value of a specified column at a specified time slice. If there is no row data that meets the conditions at that time slice, interpolation will be performed according to the settings of the [FILL](../distinguished/#FILL-Clause) parameter. +- INTERP is used to obtain the record value of a specified column at the specified time slice. It has a dedicated syntax (interp_clause) when used. For syntax introduction, see [reference link](../select/#interp). +- When there is no row data that meets the conditions at the specified time slice, the INTERP function will interpolate according to the settings of the [FILL](../distinguished/#FILL-Clause) parameter. - The input data for INTERP is the data of the specified column, which can be filtered through conditional statements (where clause). If no filtering condition is specified, the input is all data. -- INTERP SQL queries need to be used together with the RANGE, EVERY, and FILL keywords; stream computing cannot use RANGE, needs EVERY and FILL keywords together. - - When INTERP is applied to a supertable, it will sort all the subtable data under that supertable by primary key column and perform interpolation calculations, and can also be used with PARTITION BY tbname to force the results to a single timeline. - INTERP can be used with the pseudocolumn _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0). - INTERP can be used with the pseudocolumn _isfilled to display whether the return result is from the original record or generated by the interpolation algorithm (supported from version 3.0.3.0). -- For queries on tables with composite primary keys, if there are data with the same timestamp, only the data with the smallest composite primary key participates in the calculation. - - INTERP can only use the pseudocolumn `_irowts_origin` when using FILL PREV/NEXT/NEAR modes. `_irowts_origin` is supported from version 3.3.4.9. +- For queries on tables with composite primary keys, if there are data with the same timestamp, only the data with the smallest composite primary key participates in the calculation. ### IRATE diff --git a/docs/zh/14-reference/03-taos-sql/06-select.md b/docs/zh/14-reference/03-taos-sql/06-select.md index 32a89c8cdb..189faf8884 100644 --- a/docs/zh/14-reference/03-taos-sql/06-select.md +++ b/docs/zh/14-reference/03-taos-sql/06-select.md @@ -278,7 +278,8 @@ TDengine 支持基于时间戳主键的 INNER JOIN，规则如下： 6. 不支持与 FILL 子句混合使用。 ## INTERP -interp 子句配合 [INTERP](../function/#interp) 函数使用, 用于确定 INTERP 函数执行时，时间断面的截取规则和插值规则。 + +interp 子句是 INTERP 函数(../function/#interp)的专用语法，当 SQL 语句中存在 interp 子句时，只能查询 INTERP 函数而不能与其他函数一起查询，同时 interp 子句与窗口子句(window_clause)、分组子句(group_by_clause)也不能同时使用。INTERP 函数在使用时需要与 RANGE、EVERY 和 FILL 子句一起使用；流计算不支持使用 RANGE，但需要与 EVERY 和 FILL 关键字一起使用。 - INTERP 的输出时间范围根据 RANGE(timestamp1, timestamp2) 字段来指定，需满足 timestamp1 \<= timestamp2。其中 timestamp1 为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2 为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 - INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（time_unit 值）进行插值，time_unit 可取值时间单位：1a(毫秒)、1s(秒)、1m(分)、1h(小时)、1d(天)、1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](./distinguished#fill-子句) diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 9602e7fa5c..8271a91721 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -2122,14 +2122,14 @@ ignore_null_values: { **适用于**：表和超级表。 **使用说明** -- INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 [FILL](../distinguished/#fill-子句) 参数的设定进行插值。 +- INTERP 用于在指定时间断面获取指定列的记录值，使用时有专用语法(interp_clause)，语法介绍[参考链接](../select/#interp) 。 +- 当指定时间断面不存在符合条件的行数据时，INTERP 函数会根据 [FILL](../distinguished/#fill-子句) 参数的设定进行插值。 - INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 -- INTERP SQL 查询需要同时与 RANGE、EVERY 和 FILL 关键字一起使用；流计算不能使用 RANGE，需要 EVERY 和 FILL 关键字一起使用。使用规则见 [INTERP](../select/#interp) 子句。 - INTERP 作用于超级表时，会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。 - INTERP 可以与伪列 `_irowts` 一起使用，返回插值点所对应的时间戳(v3.0.2.0 以后支持)。 - INTERP 可以与伪列 `_isfilled` 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(v3.0.3.0 以后支持)。 -- INTERP 只有在使用 [FILL](../distinguished/#fill-子句) PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`, 用于返回 `interp` 函数所使用的原始数据的时间戳列。若范围内无值, 则返回 NULL。`_irowts_origin` 在 v3.3.4.9 以后支持。 -- INTERP 对于带复合主键的表的查询，若存在相同时间戳的数据，则只有对应的复合主键最小的数据参与运算。 +- 只有在使用 FILL PREV/NEXT/NEAR 模式时才可以使用伪列 `_irowts_origin`, 用于返回 `interp` 函数所使用的原始数据的时间戳列。若范围内无值, 则返回 NULL。`_irowts_origin` 在 v3.3.4.9 以后支持。 +- 对于带复合主键的表的查询，若存在相同时间戳的数据，则只有对应的复合主键最小的数据参与运算。 ### IRATE From a19c4993dd070409a8f5a4d51930f6e413efa6f4 Mon Sep 17 00:00:00 2001 From: Pan Wei <72057773+dapan1121@users.noreply.github.com> Date: Fri, 14 Mar 2025 11:12:24 +0800 Subject: [PATCH 40/44] Update 10-function.md --- docs/zh/14-reference/03-taos-sql/10-function.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index 8271a91721..2ac299b0b1 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -2124,7 +2124,6 @@ ignore_null_values: { **使用说明** - INTERP 用于在指定时间断面获取指定列的记录值，使用时有专用语法(interp_clause)，语法介绍[参考链接](../select/#interp) 。 - 当指定时间断面不存在符合条件的行数据时，INTERP 函数会根据 [FILL](../distinguished/#fill-子句) 参数的设定进行插值。 -- INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 - INTERP 作用于超级表时，会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。 - INTERP 可以与伪列 `_irowts` 一起使用，返回插值点所对应的时间戳(v3.0.2.0 以后支持)。 - INTERP 可以与伪列 `_isfilled` 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(v3.0.3.0 以后支持)。 From b4a64236dbd4c929eb0b2e4c0081d566bc9a6869 Mon Sep 17 00:00:00 2001 From: Pan Wei <72057773+dapan1121@users.noreply.github.com> Date: Fri, 14 Mar 2025 11:14:45 +0800 Subject: [PATCH 41/44] Update 10-function.md --- docs/en/14-reference/03-taos-sql/10-function.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index cf143ce759..4e60f3284f 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -2198,7 +2198,6 @@ ignore_null_values: { - INTERP is used to obtain the record value of a specified column at the specified time slice. It has a dedicated syntax (interp_clause) when used. For syntax introduction, see [reference link](../select/#interp). - When there is no row data that meets the conditions at the specified time slice, the INTERP function will interpolate according to the settings of the [FILL](../distinguished/#FILL-Clause) parameter. -- The input data for INTERP is the data of the specified column, which can be filtered through conditional statements (where clause). If no filtering condition is specified, the input is all data. - When INTERP is applied to a supertable, it will sort all the subtable data under that supertable by primary key column and perform interpolation calculations, and can also be used with PARTITION BY tbname to force the results to a single timeline. - INTERP can be used with the pseudocolumn _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0). - INTERP can be used with the pseudocolumn _isfilled to display whether the return result is from the original record or generated by the interpolation algorithm (supported from version 3.0.3.0). From d3fca89dc975010074987971e3842f01eb4a73a1 Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Fri, 14 Mar 2025 12:32:18 +0800 Subject: [PATCH 42/44] docs: fix two links in function doc --- docs/en/14-reference/03-taos-sql/10-function.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index 4e60f3284f..b711fe095c 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -2196,8 +2196,8 @@ ignore_null_values: { **Usage Instructions** -- INTERP is used to obtain the record value of a specified column at the specified time slice. It has a dedicated syntax (interp_clause) when used. For syntax introduction, see [reference link](../select/#interp). -- When there is no row data that meets the conditions at the specified time slice, the INTERP function will interpolate according to the settings of the [FILL](../distinguished/#FILL-Clause) parameter. +- INTERP is used to obtain the record value of a specified column at the specified time slice. It has a dedicated syntax (interp_clause) when used. For syntax introduction, see [reference link](../query-data/#interp). +- When there is no row data that meets the conditions at the specified time slice, the INTERP function will interpolate according to the settings of the [FILL](../time-series-extensions/#fill-clause) parameter. - When INTERP is applied to a supertable, it will sort all the subtable data under that supertable by primary key column and perform interpolation calculations, and can also be used with PARTITION BY tbname to force the results to a single timeline. - INTERP can be used with the pseudocolumn _irowts to return the timestamp corresponding to the interpolation point (supported from version 3.0.2.0). - INTERP can be used with the pseudocolumn _isfilled to display whether the return result is from the original record or generated by the interpolation algorithm (supported from version 3.0.3.0). From 103e48f65b7726a7450f3e6ae5c580261b43bf66 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Fri, 14 Mar 2025 14:09:22 +0800 Subject: [PATCH 43/44] ci: modify workflow name Signed-off-by: WANG Xu --- .github/workflows/cancel-workflow-on-merge.yml | 2 +- .../{taoskeeper-ci-build.yml => taoskeeper-build.yml} | 7 +++++-- .github/workflows/tdengine-build.yml | 3 +-- .../{taosd-doc-build.yml => tdengine-doc-build.yml} | 10 +++++----- .../{tdengine-ci-test.yml => tdengine-test.yml} | 1 + .../{taosd-ci-manual.yml => tdenginetest-manual.yml} | 2 +- 6 files changed, 14 insertions(+), 11 deletions(-) rename .github/workflows/{taoskeeper-ci-build.yml => taoskeeper-build.yml} (94%) rename .github/workflows/{taosd-doc-build.yml => tdengine-doc-build.yml} (86%) rename .github/workflows/{tdengine-ci-test.yml => tdengine-test.yml} (98%) rename .github/workflows/{taosd-ci-manual.yml => tdenginetest-manual.yml} (98%) diff --git a/.github/workflows/cancel-workflow-on-merge.yml b/.github/workflows/cancel-workflow-on-merge.yml index 5c45dbc6fe..1c00c4fe9b 100644 --- a/.github/workflows/cancel-workflow-on-merge.yml +++ b/.github/workflows/cancel-workflow-on-merge.yml @@ -1,4 +1,4 @@ -name: Cancel Workflow on Merge or Close +name: Cancel Workflow on Merge on: pull_request: diff --git a/.github/workflows/taoskeeper-ci-build.yml b/.github/workflows/taoskeeper-build.yml similarity index 94% rename from .github/workflows/taoskeeper-ci-build.yml rename to .github/workflows/taoskeeper-build.yml index 71bb1c9a51..cd1205dee5 100644 --- a/.github/workflows/taoskeeper-ci-build.yml +++ b/.github/workflows/taoskeeper-build.yml @@ -1,9 +1,12 @@ name: taosKeeper Build on: - push: + pull_request: + branches: + - 'main' + - '3.0' paths: - - tools/keeper/** + - 'tools/keeper/**' jobs: build: diff --git a/.github/workflows/tdengine-build.yml b/.github/workflows/tdengine-build.yml index 1e89a51c04..2a71cae71c 100644 --- a/.github/workflows/tdengine-build.yml +++ b/.github/workflows/tdengine-build.yml @@ -8,12 +8,11 @@ on: - '3.1' - '3.3.6' - 'enh/cmake-TD-33848' - paths-ignore: - 'docs/**' - 'packaging/**' - 'tests/**' - - '*.md' + - '**/*.md' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/taosd-doc-build.yml b/.github/workflows/tdengine-doc-build.yml similarity index 86% rename from .github/workflows/taosd-doc-build.yml rename to .github/workflows/tdengine-doc-build.yml index 3ac922820a..63266161f4 100644 --- a/.github/workflows/taosd-doc-build.yml +++ b/.github/workflows/tdengine-doc-build.yml @@ -9,11 +9,11 @@ on: - 'docs/**' env: - DOC_WKC: "/root/doc_ci_work" - ZH_DOC_REPO: "docs.taosdata.com" - EN_DOC_REPO: "docs.tdengine.com" - TD_REPO: "TDengine" - TOOLS_REPO: "taos-tools" + DOC_WKC: '/root/doc_ci_work' + ZH_DOC_REPO: 'docs.taosdata.com' + EN_DOC_REPO: 'docs.tdengine.com' + TD_REPO: 'TDengine' + TOOLS_REPO: 'taos-tools' jobs: build-doc: diff --git a/.github/workflows/tdengine-ci-test.yml b/.github/workflows/tdengine-test.yml similarity index 98% rename from .github/workflows/tdengine-ci-test.yml rename to .github/workflows/tdengine-test.yml index bfc0f0a24f..1178e393e9 100644 --- a/.github/workflows/tdengine-ci-test.yml +++ b/.github/workflows/tdengine-test.yml @@ -16,6 +16,7 @@ on: - 'source/common/src/tanalytics.c' - 'tests/parallel/tdgpt_cases.task' - 'tests/script/tsim/analytics' + - '**/*.md' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }}-TDengine diff --git a/.github/workflows/taosd-ci-manual.yml b/.github/workflows/tdenginetest-manual.yml similarity index 98% rename from .github/workflows/taosd-ci-manual.yml rename to .github/workflows/tdenginetest-manual.yml index 7cb3688c46..be14a6e908 100644 --- a/.github/workflows/taosd-ci-manual.yml +++ b/.github/workflows/tdenginetest-manual.yml @@ -1,4 +1,4 @@ -name: TDengine CI Test Manually +name: TDengine Test (Manual) on: workflow_dispatch: From 0a6e5c7fca2d9955e86ac3930f4bf57a7a32389a Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Fri, 14 Mar 2025 14:41:21 +0800 Subject: [PATCH 44/44] fix: bad file name Signed-off-by: WANG Xu --- .../{tdenginetest-manual.yml => tdengine-test-manual.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{tdenginetest-manual.yml => tdengine-test-manual.yml} (100%) diff --git a/.github/workflows/tdenginetest-manual.yml b/.github/workflows/tdengine-test-manual.yml similarity index 100% rename from .github/workflows/tdenginetest-manual.yml rename to .github/workflows/tdengine-test-manual.yml