Merge remote-tracking branch 'origin/doc/analysis' into doc/analysis

This commit is contained in:
Haojun Liao 2025-03-22 17:46:34 +08:00
commit 37da08ca71
10 changed files with 188 additions and 28 deletions

View File

@ -0,0 +1,114 @@
name: TDengine Release Build
on:
push:
branches:
- 'main'
- '3.*'
paths-ignore:
- 'docs/**'
- 'packaging/**'
- 'tests/**'
- '**/*.md'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
name: Run on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-22.04
- macos-14
steps:
- name: Checkout the repository
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.18
- name: Install dependencies on Linux
if: runner.os == 'Linux'
run: |
sudo apt update -y
sudo apt install -y \
build-essential \
cmake \
gawk \
libgeos-dev \
libjansson-dev \
liblzma-dev \
libsnappy-dev \
libssl-dev \
libz-dev \
pkg-config \
zlib1g
- name: Install dependencies on macOS
if: runner.os == 'macOS'
run: |
brew update
brew install \
argp-standalone \
gawk \
gflags \
geos \
jansson \
openssl \
pkg-config \
snappy \
zlib
- name: Build and install TDengine
run: |
mkdir debug && cd debug
cmake .. -DBUILD_TOOLS=true \
-DBUILD_KEEPER=true \
-DBUILD_HTTP=false \
-DBUILD_TEST=true \
-DWEBSOCKET=true \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_DEPENDENCY_TESTS=false
make -j 4
sudo make install
which taosd
which taosadapter
which taoskeeper
- name: Statistics ldd
run: |
find ${{ github.workspace }}/debug/build/lib -type f -name "*.so" -print0 | xargs -0 ldd || true
find ${{ github.workspace }}/debug/build/bin -type f -print0 | xargs -0 ldd || true
- name: Statistics size
run: |
find ${{ github.workspace }}/debug/build/lib -type f -print0 | xargs -0 ls -lhrS
find ${{ github.workspace }}/debug/build/bin -type f -print0 | xargs -0 ls -lhrS
- name: Start taosd
run: |
cp /etc/taos/taos.cfg ./
sudo echo "supportVnodes 256" >> taos.cfg
nohup sudo taosd -c taos.cfg &
- name: Start taosadapter
run: nohup sudo taosadapter &
- name: Run tests with taosBenchmark
run: |
taosBenchmark -t 10 -n 10 -y
taos -s "select count(*) from test.meters"
- name: Clean up
if: always()
run: |
if pgrep taosd; then sudo pkill taosd; fi
if pgrep taosadapter; then sudo pkill taosadapter; fi

View File

@ -558,6 +558,20 @@ This document details the server error codes that may be encountered when using
| 0x80004017 | Invalid status, please subscribe topic first | tmq status invalidate | Without calling subscribe, directly poll data |
| 0x80004100 | Stream task not exist | The stream computing task does not exist | Check the server-side error logs |
## TDgpt
| Error Code | Description | Possible Error Scenarios or Reasons | Recommanded Actions for Users |
| ---------- | --------------------- | -------------------------------------------------------------------------------- | ------------------------------ |
| 0x80000440 | Analysis service response is NULL | The response content is empty | Check the taosanode.app.log for detailed response information |
| 0x80000441 | Analysis service can't access | Service is not work currectly, or network is broken | Check the status of taosanode and network status |
| 0x80000442 | Analysis algorithm is missing | Algorithm used in analysis is not specified | Add the "algo" parameter in forecast function or anomaly_window clause |
| 0x80000443 | Analysis algorithm not loaded | The specified algorithm is not available | Check for the specified algorithm |
| 0x80000444 | Analysis invalid buffer type | The bufferred data type is invalid | Check the taosanode.app.log for more details |
| 0x80000445 | Analysis failed since anode return error | The responses from anode with error message | Check the taosanode.app.log for more details |
| 0x80000446 | Analysis failed since too many input rows for anode | Input data is too many | Reduce the rows of input data to below than the threshold |
| 0x80000447 | white-noise data not processed | white noise data is not processed | Ignore the white noise check or use another input data |
| 0x80000448 | Analysis internal error, not processed | Internal error occurs | Check the taosanode.app.log for more details |
## virtual table

View File

@ -9,14 +9,14 @@ import wndata from './pic/white-noise-data.png'
### 分析流程
时序数据分析之前需要有预处理的过程为减轻分析算法的负担TDgpt 在将时序数据发给具体分析算法进行分析时,已经对数据做了预处理,整体的流程如下图所示。
<img src={activity} width="560" alt="预处理流程" />
<img src={activity} width="320" alt="预处理流程" />
TDgpt 首先对输入数据进行白噪声检查White Noise Data check, 检查通过以后针对预测分析,还要进行输入(历史)数据的重采样和时间戳对齐处理(异常检测跳过数据重采样和时间戳对齐步骤)。
预处理完成以后,再进行预测或异常检测操作。预处理过程不属于预测或异常检测处理逻辑的一部分。
### 白噪声检查
<img src={wndata} width="430" alt="white-noise-data"/>
<img src={wndata} width="344" alt="white-noise-data"/>
白噪声时序数据可以简单地认为是随机数构成的时间数据序列(如上图所示的正态分布随机数序列),随机数构成的时间序列没有分析的价值,因此会直接返回。白噪声检查采用经典的 `Ljung-Box` 统计量检验,计算 `Ljung-Box` 统计量需遍历整个输入时间序列。如果用户能够明确输入序列一定不是白噪声序列,那么可以在参数列表中增加参数 `wncheck=0` 强制要求分析平台忽略白噪声检查,从而节省计算资源。
TDgpt 暂不提供独立的时间序列白噪声检测功能。

View File

@ -134,12 +134,16 @@ res_start_time = 1730000000000
gen_figure = true
```
算法对比分析运行完成以后,生成 fc-results.xlsx 文件,其中包含了调用算法的预测分析误差、执行时间、调用参数等信息。如下图所示
算法对比分析运行完成以后,生成 fc-results.xlsx 文件,其中包含了调用算法的预测分析误差、执行时间、调用参数等信息,如下表
<img src={fc_result} width="760" alt="预测对比结果" />
|algorithm|params|MSE|elapsed_time(ms.)|
|---|---|---|---|
|holtwinters|{"trend":"add", "seasonal":"add"}|351.622| 125.1721|
|arima|{"time_step":3600000, "start_p":0, "max_p":10, "start_q":0, "max_q":10}|433.709| 45577.9187|
如果设置了 `gen_figure` 为 true分析结果中还会有绘制的分析预测结果图如下图所示
<img src={fc_result_figure} width="540" alt="预测对比结果" />
<img src={fc_result_figure} width="360" alt="预测对比结果" />

View File

@ -604,4 +604,4 @@ description: TDengine 服务端的错误码列表和详细说明
| 0x80000445 | Analysis failed since anode return error | anode 返回错误信息 | 请检查服务端日志确认问题原因 |
| 0x80000446 | Analysis failed since too many input rows for anode | 输入数据太多 | 减小分析数据输入规模 |
| 0x80000447 | white-noise data not processed | 白噪声数据不分析 | |
| 0x80000448 | tdgpt internal error, not processed | anode 出现内部错误 | 具体查看server端的错误日志 |
| 0x80000448 | Analysis internal error, not processed | anode 出现内部错误 | 具体查看server端的日志 (taosanode.app.log) |

View File

@ -273,9 +273,12 @@ function install_lib() {
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
${csudo}rm -f ${lib_link_dir}/libtaosnative.* || :
${csudo}rm -f ${lib64_link_dir}/libtaosnative.* || :
${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/*
#link lib/link_dir
${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
${csudo}ln -sf ${install_main_dir}/driver/libtaosnative.* ${lib_link_dir}/libtaosnative.so.1
@ -283,13 +286,14 @@ function install_lib() {
[ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || :
#link lib64/link_dir
if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
${csudo}ln -sf ${install_main_dir}/driver/libtaosnative.* ${lib64_link_dir}/libtaosnative.so.1 || :
${csudo}ln -sf ${lib64_link_dir}/libtaosnative.so.1 ${lib64_link_dir}/libtaosnative.so || :
[ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
[ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib64_link_dir}/libtaosws.so || :
fi
${csudo}ldconfig

View File

@ -312,15 +312,37 @@ function install_avro() {
function install_lib() {
# Remove links
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
${csudo}rm -f ${lib_link_dir}/libtaosnative.* || :
[ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || :
remove_links() {
local dir=$1
find ${dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || :
find ${dir} -name "libtaosnative.so" -exec ${csudo}rm -f {} \; || :
find ${dir} -name "libtaosws.so" -exec ${csudo}rm -f {} \; || :
}
remove_links ${lib_link_dir}
if [ "$osType" != "Darwin" ]; then
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
${csudo}rm -f ${lib64_link_dir}/libtaosnative.* || :
[ -f ${lib64_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.so || :
remove_links ${lib64_link_dir}
fi
# Copy and set permissions for libraries
copy_and_set_permissions() {
local src=$1
local dest=$2
if [ "$osType" != "Darwin" ]; then
${csudo}cp ${src} ${dest} && ${csudo}chmod 777 ${dest}
else
${csudo}cp -Rf ${src} ${dest} && ${csudo}chmod 777 ${dest}
fi
}
# Create symbolic links
create_symlink() {
local target=$1
local link_name=$2
${csudo}ln -sf ${target} ${link_name}
}
if [ "$osType" != "Darwin" ]; then
${csudo}cp ${binary_dir}/build/lib/libtaos.so.${verNumber} \
${install_main_dir}/driver &&

View File

@ -179,13 +179,13 @@ remove_bin() {
function clean_lib() {
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosnative.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosnative.* || :
[ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}find ${lib_link_dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || :
${csudo}find ${lib_link_dir} -name "libtaosnative.*" -exec ${csudo}rm -f {} \; || :
${csudo}find ${lib_link_dir} -name "libtaosws.*" -exec ${csudo}rm -f {} \; || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosnative.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosnative.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
${csudo}find ${lib64_link_dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || :
${csudo}find ${lib64_link_dir} -name "libtaosnative.*" -exec ${csudo}rm -f {} \; || :
${csudo}find ${lib64_link_dir} -name "libtaosws.*" -exec ${csudo}rm -f {} \; || :
#${csudo}rm -rf ${v15_java_app_dir} || :
}
@ -338,4 +338,4 @@ fi
command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true
echo
echo "${productName} is removed successfully!"
echo
echo

View File

@ -70,15 +70,16 @@ function clean_bin() {
}
function clean_lib() {
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
[ -f ${lib_link_dir}/libtaosnative.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosnative.* || :
# Remove link
${csudo}find ${lib_link_dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || :
${csudo}find ${lib_link_dir} -name "libtaosnative.*" -exec ${csudo}rm -f {} \; || :
${csudo}find ${lib_link_dir} -name "libtaosws.*" -exec ${csudo}rm -f {} \; || :
${csudo}find ${lib64_link_dir} -name "libtaos.*" -exec ${csudo}rm -f {} \; || :
${csudo}find ${lib64_link_dir} -name "libtaosnative.*" -exec ${csudo}rm -f {} \; || :
${csudo}find ${lib64_link_dir} -name "libtaosws.*" -exec ${csudo}rm -f {} \; || :
#${csudo}rm -rf ${v15_java_app_dir} || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
[ -f ${lib64_link_dir}/libtaosnative.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosnative.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
}
function clean_header() {

View File

@ -400,6 +400,7 @@ function install_anode_venv() {
${csudo}${venvDir}/bin/pip3 install uwsgi
${csudo}${venvDir}/bin/pip3 install torch --index-url https://download.pytorch.org/whl/cpu
${csudo}${venvDir}/bin/pip3 install --upgrade keras
${csudo}${venvDir}/bin/pip3 install requests
echo -e "Install python library for venv completed!"
}