From ddb4cd6592e4f15a413819a22ab954167dfa6cf0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Mar 2025 13:14:01 +0800 Subject: [PATCH 1/9] doc: add error code information about tdgpt (#30338) * Update 09-error-code.md * Update 09-error-code.md * Update 09-error-code.md --- docs/en/14-reference/09-error-code.md | 14 ++++++++++++++ docs/zh/14-reference/09-error-code.md | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 139f8d38d7..3305eb18eb 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -558,6 +558,20 @@ This document details the server error codes that may be encountered when using | 0x80004017 | Invalid status, please subscribe topic first | tmq status invalidate | Without calling subscribe, directly poll data | | 0x80004100 | Stream task not exist | The stream computing task does not exist | Check the server-side error logs | +## TDgpt + +| Error Code | Description | Possible Error Scenarios or Reasons | Recommanded Actions for Users | +| ---------- | --------------------- | -------------------------------------------------------------------------------- | ------------------------------ | +| 0x80000440 | Analysis service response is NULL | The response content is empty | Check the taosanode.app.log for detailed response information | +| 0x80000441 | Analysis service can't access | Service is not work currectly, or network is broken | Check the status of taosanode and network status | +| 0x80000442 | Analysis algorithm is missing | Algorithm used in analysis is not specified | Add the "algo" parameter in forecast function or anomaly_window clause | +| 0x80000443 | Analysis algorithm not loaded | The specified algorithm is not available | Check for the specified algorithm | +| 0x80000444 | Analysis invalid buffer type | The bufferred data type is invalid | Check the taosanode.app.log for more details | +| 0x80000445 | Analysis failed since anode return error | The responses from anode with error message | Check the taosanode.app.log for more details | +| 0x80000446 | Analysis failed since too many input rows for anode | Input data is too many | Reduce the rows of input data to below than the threshold | +| 0x80000447 | white-noise data not processed | white noise data is not processed | Ignore the white noise check or use another input data | +| 0x80000448 | Analysis internal error, not processed | Internal error occurs | Check the taosanode.app.log for more details | + ## virtual table diff --git a/docs/zh/14-reference/09-error-code.md b/docs/zh/14-reference/09-error-code.md index 517658482c..ef585cf706 100644 --- a/docs/zh/14-reference/09-error-code.md +++ b/docs/zh/14-reference/09-error-code.md @@ -604,4 +604,4 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x80000445 | Analysis failed since anode return error | anode 返回错误信息 | 请检查服务端日志确认问题原因 | | 0x80000446 | Analysis failed since too many input rows for anode | 输入数据太多 | 减小分析数据输入规模 | | 0x80000447 | white-noise data not processed | 白噪声数据不分析 | | -| 0x80000448 | tdgpt internal error, not processed | anode 出现内部错误 | 具体查看server端的错误日志 | +| 0x80000448 | Analysis internal error, not processed | anode 出现内部错误 | 具体查看server端的日志 (taosanode.app.log) | From 0ee9e3db566a05b8dec95d57d83bfe6d6ea61fce Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Sat, 22 Mar 2025 13:56:21 +0800 Subject: [PATCH 2/9] refactor: streamline library removal and linking in installation scripts --- packaging/tools/install.sh | 6 +++++- packaging/tools/make_install.sh | 34 ++++++++++++++++++++++++++------ packaging/tools/remove.sh | 14 ++++++------- packaging/tools/remove_client.sh | 17 ++++++++-------- 4 files changed, 49 insertions(+), 22 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 8fd483b6a7..56a18c64c6 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -273,9 +273,12 @@ function install_lib() { ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : ${csudo}rm -f ${lib_link_dir}/libtaosnative.* || : ${csudo}rm -f ${lib64_link_dir}/libtaosnative.* || : + ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* + #link lib/link_dir ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so ${csudo}ln -sf ${install_main_dir}/driver/libtaosnative.* ${lib_link_dir}/libtaosnative.so.1 @@ -283,13 +286,14 @@ function install_lib() { [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : + #link lib64/link_dir if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : ${csudo}ln -sf ${install_main_dir}/driver/libtaosnative.* ${lib64_link_dir}/libtaosnative.so.1 || : ${csudo}ln -sf ${lib64_link_dir}/libtaosnative.so.1 ${lib64_link_dir}/libtaosnative.so || : - [ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi ${csudo}ldconfig diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 4a694f9841..de7dd2d2cb 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -312,15 +312,37 @@ function install_avro() { function install_lib() { # Remove links - ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - ${csudo}rm -f ${lib_link_dir}/libtaosnative.* || : - [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : + remove_links() { + local dir=$1 + find ${dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || : + find ${dir} -name "libtaosnative.so" -exec ${csudo}rm -f {} \; || : + find ${dir} -name "libtaosws.so" -exec ${csudo}rm -f {} \; || : + } + + remove_links ${lib_link_dir} + if [ "$osType" != "Darwin" ]; then - ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : - ${csudo}rm -f ${lib64_link_dir}/libtaosnative.* || : - [ -f ${lib64_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.so || : + remove_links ${lib64_link_dir} fi + # Copy and set permissions for libraries + copy_and_set_permissions() { + local src=$1 + local dest=$2 + if [ "$osType" != "Darwin" ]; then + ${csudo}cp ${src} ${dest} && ${csudo}chmod 777 ${dest} + else + ${csudo}cp -Rf ${src} ${dest} && ${csudo}chmod 777 ${dest} + fi + } + + # Create symbolic links + create_symlink() { + local target=$1 + local link_name=$2 + ${csudo}ln -sf ${target} ${link_name} + } + if [ "$osType" != "Darwin" ]; then ${csudo}cp ${binary_dir}/build/lib/libtaos.so.${verNumber} \ ${install_main_dir}/driver && diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index d62aa5a564..91b1dfc461 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -5,7 +5,7 @@ set -e #set -x -verMode=edge +verMode=cluster osType=`uname` RED='\033[0;31m' @@ -179,13 +179,13 @@ remove_bin() { function clean_lib() { # Remove link - ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - [ -f ${lib_link_dir}/libtaosnative.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosnative.* || : - [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : + ${csudo}find ${lib_link_dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || : + ${csudo}find ${lib_link_dir} -name "libtaosnative.*" -exec ${csudo}rm -f {} \; || : + ${csudo}find ${lib_link_dir} -name "libtaosws.*" -exec ${csudo}rm -f {} \; || : - ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : - [ -f ${lib64_link_dir}/libtaosnative.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosnative.* || : - [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : + ${csudo}find ${lib64_link_dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || : + ${csudo}find ${lib64_link_dir} -name "libtaosnative.*" -exec ${csudo}rm -f {} \; || : + ${csudo}find ${lib64_link_dir} -name "libtaosws.*" -exec ${csudo}rm -f {} \; || : #${csudo}rm -rf ${v15_java_app_dir} || : } diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index e6ec9c3768..e5c3842a5e 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -70,15 +70,16 @@ function clean_bin() { } function clean_lib() { - # Remove link - ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : - [ -f ${lib_link_dir}/libtaosnative.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosnative.* || : + # Remove link + ${csudo}find ${lib_link_dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || : + ${csudo}find ${lib_link_dir} -name "libtaosnative.*" -exec ${csudo}rm -f {} \; || : + ${csudo}find ${lib_link_dir} -name "libtaosws.*" -exec ${csudo}rm -f {} \; || : + + ${csudo}find ${lib64_link_dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || : + ${csudo}find ${lib64_link_dir} -name "libtaosnative.*" -exec ${csudo}rm -f {} \; || : + ${csudo}find ${lib64_link_dir} -name "libtaosws.*" -exec ${csudo}rm -f {} \; || : + #${csudo}rm -rf ${v15_java_app_dir} || : - ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : - [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : - [ -f ${lib64_link_dir}/libtaosnative.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosnative.* || : - #${csudo}rm -rf ${v15_java_app_dir} || : } function clean_header() { From 9c07a28e6573b0e32dab04b8eff14c5cc0bb6798 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Sat, 22 Mar 2025 14:35:05 +0800 Subject: [PATCH 3/9] Update remove.sh --- packaging/tools/remove.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 91b1dfc461..1f75f00d5d 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -5,7 +5,7 @@ set -e #set -x -verMode=cluster +verMode=edge osType=`uname` RED='\033[0;31m' @@ -338,4 +338,4 @@ fi command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true echo echo "${productName} is removed successfully!" -echo \ No newline at end of file +echo From 3a29a1daf12dcfe156066d82d99c8edeb39b9f5e Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Sat, 22 Mar 2025 13:36:29 +0800 Subject: [PATCH 4/9] ci: add release build Signed-off-by: WANG Xu --- .github/workflows/tdengine-release-build.yml | 114 +++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 .github/workflows/tdengine-release-build.yml diff --git a/.github/workflows/tdengine-release-build.yml b/.github/workflows/tdengine-release-build.yml new file mode 100644 index 0000000000..ef708a0ac9 --- /dev/null +++ b/.github/workflows/tdengine-release-build.yml @@ -0,0 +1,114 @@ +name: TDengine Release Build + +on: + push: + branches: + - 'main' + - '3.*' + paths-ignore: + - 'docs/**' + - 'packaging/**' + - 'tests/**' + - '**/*.md' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Run on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-22.04 + - macos-14 + + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.18 + + - name: Install dependencies on Linux + if: runner.os == 'Linux' + run: | + sudo apt update -y + sudo apt install -y \ + build-essential \ + cmake \ + gawk \ + libgeos-dev \ + libjansson-dev \ + liblzma-dev \ + libsnappy-dev \ + libssl-dev \ + libz-dev \ + pkg-config \ + zlib1g + + - name: Install dependencies on macOS + if: runner.os == 'macOS' + run: | + brew update + brew install \ + argp-standalone \ + gawk \ + gflags \ + geos \ + jansson \ + openssl \ + pkg-config \ + snappy \ + zlib + + - name: Build and install TDengine + run: | + mkdir debug && cd debug + cmake .. -DBUILD_TOOLS=true \ + -DBUILD_KEEPER=true \ + -DBUILD_HTTP=false \ + -DBUILD_TEST=true \ + -DWEBSOCKET=true \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_DEPENDENCY_TESTS=false + make -j 4 + sudo make install + which taosd + which taosadapter + which taoskeeper + + - name: Statistics ldd + run: | + find ${{ github.workspace }}/debug/build/lib -type f -name "*.so" -print0 | xargs -0 ldd || true + find ${{ github.workspace }}/debug/build/bin -type f -print0 | xargs -0 ldd || true + + - name: Statistics size + run: | + find ${{ github.workspace }}/debug/build/lib -type f -print0 | xargs -0 ls -lhrS + find ${{ github.workspace }}/debug/build/bin -type f -print0 | xargs -0 ls -lhrS + + - name: Start taosd + run: | + cp /etc/taos/taos.cfg ./ + sudo echo "supportVnodes 256" >> taos.cfg + nohup sudo taosd -c taos.cfg & + + - name: Start taosadapter + run: nohup sudo taosadapter & + + - name: Run tests with taosBenchmark + run: | + taosBenchmark -t 10 -n 10 -y + taos -s "select count(*) from test.meters" + + - name: Clean up + if: always() + run: | + if pgrep taosd; then sudo pkill taosd; fi + if pgrep taosadapter; then sudo pkill taosadapter; fi From 523b7003a4d007c0a0132d0e54c706956ec586bf Mon Sep 17 00:00:00 2001 From: haoranchen Date: Sat, 22 Mar 2025 15:31:16 +0800 Subject: [PATCH 5/9] fix: add requests pkg in tdgpt install.sh --- tools/tdgpt/script/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/tdgpt/script/install.sh b/tools/tdgpt/script/install.sh index 9952b7f0af..e5bf87740b 100755 --- a/tools/tdgpt/script/install.sh +++ b/tools/tdgpt/script/install.sh @@ -400,6 +400,7 @@ function install_anode_venv() { ${csudo}${venvDir}/bin/pip3 install uwsgi ${csudo}${venvDir}/bin/pip3 install torch --index-url https://download.pytorch.org/whl/cpu ${csudo}${venvDir}/bin/pip3 install --upgrade keras + ${csudo}${venvDir}/bin/pip3 install requests echo -e "Install python library for venv completed!" } From 11f3700a899e8307d45f01e63c2544be418b41f4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Mar 2025 16:51:27 +0800 Subject: [PATCH 6/9] Update 03-preprocess.md --- docs/zh/06-advanced/06-TDgpt/03-preprocess.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md index b63cae0740..8cc207943a 100644 --- a/docs/zh/06-advanced/06-TDgpt/03-preprocess.md +++ b/docs/zh/06-advanced/06-TDgpt/03-preprocess.md @@ -9,14 +9,14 @@ import wndata from './pic/white-noise-data.png' ### 分析流程 时序数据分析之前需要有预处理的过程，为减轻分析算法的负担，TDgpt 在将时序数据发给具体分析算法进行分析时，已经对数据做了预处理，整体的流程如下图所示。 -预处理流程 +预处理流程 TDgpt 首先对输入数据进行白噪声检查（White Noise Data check）, 检查通过以后针对预测分析，还要进行输入（历史）数据的重采样和时间戳对齐处理（异常检测跳过数据重采样和时间戳对齐步骤）。 预处理完成以后，再进行预测或异常检测操作。预处理过程不属于预测或异常检测处理逻辑的一部分。 ### 白噪声检查 -white-noise-data +white-noise-data 白噪声时序数据可以简单地认为是随机数构成的时间数据序列（如上图所示的正态分布随机数序列），随机数构成的时间序列没有分析的价值，因此会直接返回。白噪声检查采用经典的 `Ljung-Box` 统计量检验，计算 `Ljung-Box` 统计量需遍历整个输入时间序列。如果用户能够明确输入序列一定不是白噪声序列，那么可以在参数列表中增加参数 `wncheck=0` 强制要求分析平台忽略白噪声检查，从而节省计算资源。 TDgpt 暂不提供独立的时间序列白噪声检测功能。 From f5c0acc52b9c1f778c432c196e4bb35f0c614856 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Mar 2025 16:53:48 +0800 Subject: [PATCH 7/9] Update index.md --- docs/zh/06-advanced/06-TDgpt/04-forecast/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md index 560e6cdcc0..9c01c1e0e6 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md @@ -136,10 +136,10 @@ gen_figure = true 算法对比分析运行完成以后，生成 fc-results.xlsx 文件，其中包含了调用算法的预测分析误差、执行时间、调用参数等信息。如下图所示： -预测对比结果 +预测对比结果 如果设置了 `gen_figure` 为 true，分析结果中还会有绘制的分析预测结果图（如下图所示）。 -预测对比结果 +预测对比结果 From ab954cc0c776e0263f40ac04c1e8f4350aaf3258 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Mar 2025 17:16:23 +0800 Subject: [PATCH 8/9] Update index.md --- docs/zh/06-advanced/06-TDgpt/04-forecast/index.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md index 9c01c1e0e6..8648a16a7d 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md @@ -139,6 +139,12 @@ gen_figure = true 预测对比结果 +|algorithm|params|MSE|elapsed_time(ms.)| +|---|---|---|---| +|holtwinters|{"trend":"add", "seasonal":"add"}|351.622| 125.1721| +|arima|{"time_step":3600000, "start_p":0, "max_p":10, "start_q":0, "max_q":10}|433.709| 45577.9187| + + 如果设置了 `gen_figure` 为 true，分析结果中还会有绘制的分析预测结果图（如下图所示）。 预测对比结果 From bff673ac8529a392d81bfaca70fdf872aed01191 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 22 Mar 2025 17:17:03 +0800 Subject: [PATCH 9/9] Update index.md --- docs/zh/06-advanced/06-TDgpt/04-forecast/index.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md index 8648a16a7d..6aa66d3142 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md @@ -134,9 +134,7 @@ res_start_time = 1730000000000 gen_figure = true ``` -算法对比分析运行完成以后，生成 fc-results.xlsx 文件，其中包含了调用算法的预测分析误差、执行时间、调用参数等信息。如下图所示： - -预测对比结果 +算法对比分析运行完成以后，生成 fc-results.xlsx 文件，其中包含了调用算法的预测分析误差、执行时间、调用参数等信息，如下表： |algorithm|params|MSE|elapsed_time(ms.)|