From 4ac3d55709b190776c089d57f70a51bf1553f661 Mon Sep 17 00:00:00 2001 From: freemine Date: Wed, 1 Jan 2025 16:15:37 +0800 Subject: [PATCH 01/87] bypass some specific test case which believes to fail in community version of taos --- source/libs/parser/test/parAlterToBalanceTest.cpp | 14 +++++++++++++- source/libs/parser/test/parInitialCTest.cpp | 8 ++++++++ source/libs/tcs/test/tcsTest.cpp | 8 ++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index d25435913f..7b5817d9f1 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -196,14 +196,24 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbFsync(200); setAlterDbWal(1); setAlterDbCacheModel(TSDB_CACHE_MODEL_LAST_ROW); +#ifdef TD_ENTERPRISE setAlterDbSttTrigger(16); +#else + setAlterDbSttTrigger(1); +#endif setAlterDbBuffer(16); setAlterDbPages(128); setAlterDbReplica(3); setAlterDbWalRetentionPeriod(10); setAlterDbWalRetentionSize(20); run("ALTER DATABASE test BUFFER 16 CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 PAGES 128 " - "REPLICA 3 WAL_LEVEL 1 STT_TRIGGER 16 WAL_RETENTION_PERIOD 10 WAL_RETENTION_SIZE 20"); + "REPLICA 3 WAL_LEVEL 1 " +#ifdef TD_ENTERPRISE + "STT_TRIGGER 16 " +#else + "STT_TRIGGER 1 " +#endif + "WAL_RETENTION_PERIOD 10 WAL_RETENTION_SIZE 20"); clearAlterDbReq(); initAlterDb("test"); @@ -289,10 +299,12 @@ TEST_F(ParserInitialATest, alterDatabase) { initAlterDb("test"); setAlterDbSttTrigger(1); run("ALTER DATABASE test STT_TRIGGER 1"); +#ifdef TD_ENTERPRISE setAlterDbSttTrigger(4); run("ALTER DATABASE test STT_TRIGGER 4"); setAlterDbSttTrigger(16); run("ALTER DATABASE test STT_TRIGGER 16"); +#endif clearAlterDbReq(); initAlterDb("test"); diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 3422ebe028..6ded5696f9 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -292,7 +292,11 @@ TEST_F(ParserInitialCTest, createDatabase) { setDbWalRetentionSize(-1); setDbWalRollPeriod(10); setDbWalSegmentSize(20); +#ifdef TD_ENTERPRISE setDbSstTrigger(16); +#else + setDbSstTrigger(1); +#endif setDbHashPrefix(3); setDbHashSuffix(4); setDbTsdbPageSize(32); @@ -320,7 +324,11 @@ TEST_F(ParserInitialCTest, createDatabase) { "WAL_RETENTION_SIZE -1 " "WAL_ROLL_PERIOD 10 " "WAL_SEGMENT_SIZE 20 " +#ifdef TD_ENTERPRISE "STT_TRIGGER 16 " +#else + "STT_TRIGGER 1 " +#endif "TABLE_PREFIX 3 " "TABLE_SUFFIX 4 " "TSDB_PAGESIZE 32"); diff --git a/source/libs/tcs/test/tcsTest.cpp b/source/libs/tcs/test/tcsTest.cpp index 4b5afc5b85..5b5a1bd1d2 100644 --- a/source/libs/tcs/test/tcsTest.cpp +++ b/source/libs/tcs/test/tcsTest.cpp @@ -234,6 +234,13 @@ TEST(TcsTest, InterfaceTest) { // TEST(TcsTest, DISABLED_InterfaceNonBlobTest) { TEST(TcsTest, InterfaceNonBlobTest) { +#ifndef TD_ENTERPRISE + // NOTE: this test case will coredump for community edition of taos + // thus we bypass this test case for the moment + // code = tcsGetObjectBlock(object_name, 0, size, check, &pBlock); + // tcsGetObjectBlock succeeded but pBlock is nullptr + // which results in nullptr-access-coredump shortly after +#else int code = 0; bool check = false; bool withcp = false; @@ -348,4 +355,5 @@ TEST(TcsTest, InterfaceNonBlobTest) { GTEST_ASSERT_EQ(code, 0); tcsUninit(); +#endif } From c4b522ff09e80356c3a522788c5fc83ec81607c5 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Fri, 3 Jan 2025 14:15:29 +0800 Subject: [PATCH 02/87] update README structure for support linux/win/mac platforms by charles --- README.md | 337 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 198 insertions(+), 139 deletions(-) diff --git a/README.md b/README.md index e4814cee67..dfd1be23c7 100644 --- a/README.md +++ b/README.md @@ -26,24 +26,51 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine # Table of Contents -1. [What is TDengine?](#1-what-is-tdengine) -2. [Documentation](#2-documentation) -3. [Building](#3-building) - 1. [Install build tools](#31-install-build-tools) - 1. [Get the source codes](#32-get-the-source-codes) - 1. [Special Note](#33-special-note) - 1. [Build TDengine](#34-build-tdengine) -4. [Installing](#4-installing) - 1. [On Linux platform](#41-on-linux-platform) - 1. [On Windows platform](#42-on-windows-platform) - 1. [On macOS platform](#43-on-macos-platform) - 1. [Quick Run](#44-quick-run) -5. [Try TDengine](#5-try-tdengine) -6. [Developing with TDengine](#6-developing-with-tdengine) -7. [Contribute to TDengine](#7-contribute-to-tdengine) -8. [Join the TDengine Community](#8-join-the-tdengine-community) +- [Table of Contents](#table-of-contents) +- [1. Introduction](#1-introduction) +- [2. Documentation](#2-documentation) +- [3. Prerequisites](#3-prerequisites) + - [3.1 Install dependencies tools](#31-install-dependencies-tools) + - [3.1.1 Linux platform](#311-linux-platform) + - [Install the required package for linux](#install-the-required-package-for-linux) + - [Install Go for linux](#install-go-for-linux) + - [Install node for linux](#install-node-for-linux) + - [Install Python-connector for linux](#install-python-connector-for-linux) + - [3.1.2 Windows platform](#312-windows-platform) + - [Install the required package for windows](#install-the-required-package-for-windows) + - [3.1.3 MacOS platform](#313-macos-platform) + - [Install the required package for macOS](#install-the-required-package-for-macos) + - [3.2 Get the source codes](#32-get-the-source-codes) + - [3.3 Special Note](#33-special-note) +- [4. Building](#4-building) + - [4.1 Linux platform building](#41-linux-platform-building) + - [4.2 Windows platform building](#42-windows-platform-building) + - [4.3 MacOS platform building](#43-macos-platform-building) +- [5. Packaging](#5-packaging) + - [5.1 Linux platform packaging](#51-linux-platform-packaging) + - [5.2 Windows platform packaging](#52-windows-platform-packaging) + - [5.3 MacOS platform packaging](#53-macos-platform-packaging) +- [6. Installation](#6-installation) + - [6.1 Linux platform installation](#61-linux-platform-installation) + - [6.2 Windows platform installation](#62-windows-platform-installation) + - [6.3 MacOS platform installation](#63-macos-platform-installation) +- [7. Running](#7-running) + - [7.1 Linux platform](#71-linux-platform) + - [7.2 Windows platform](#72-windows-platform) + - [7.3 MacOS platform](#73-macos-platform) +- [8. Testing](#8-testing) + - [8.1 Linux platform testing](#81-linux-platform-testing) + - [8.2 Windows platform testing](#82-windows-platform-testing) + - [8.3 MacOS platform testing](#83-macos-platform-testing) +- [9. Releasing](#9-releasing) + - [9.1 Linux platform releasing](#91-linux-platform-releasing) + - [9.2 Windows platform realeasing](#92-windows-platform-realeasing) + - [9.3 MacOS platform releasing](#93-macos-platform-releasing) +- [10. CI/CD](#10-cicd) +- [11. Coverage](#11-coverage) +- [12. Contributing](#12-contributing) -# 1. What is TDengine？ +# 1. Introduction TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-series databases with the following advantages: @@ -65,109 +92,127 @@ For a full list of TDengine competitive advantages, please [check here](https:// For user manual, system design and architecture, please refer to [TDengine Documentation](https://docs.tdengine.com) ([TDengine 文档](https://docs.taosdata.com)) -# 3. Building +# 3. Prerequisites -At the moment, TDengine server supports running on Linux/Windows/macOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service . TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. Right now we don't support build with cross-compiling environment. +List the software and tools required to work on the project. -You can choose to install through source code, [container](https://docs.tdengine.com/get-started/docker/), [installation package](https://docs.tdengine.com/get-started/package/) or [Kubernetes](https://docs.tdengine.com/deployment/k8s/). This quick guide only applies to installing from source. +- go 1.20+ (for taosadapter) +- node 16.20.2 (for taos-explorer) +- python 3.10.12+ (for test) -TDengine provide a few useful tools such as taosBenchmark (was named taosdemo) and taosdump. They were part of TDengine. By default, TDengine compiling does not include taosTools. You can use `cmake .. -DBUILD_TOOLS=true` to make them be compiled with TDengine. +Step-by-step instructions to set up the prerequisites software. -To build TDengine, use [CMake](https://cmake.org/) 3.13.0 or higher versions in the project directory. +## 3.1 Install dependencies tools -## 3.1 Install build tools +### 3.1.1 Linux platform -### Ubuntu 18.04 and above or Debian +#### Install the required package for linux ```bash -sudo apt-get install -y gcc cmake build-essential git libssl-dev libgflags2.2 libgflags-dev +apt-get install -y llvm gcc make cmake libssl-dev pkg-config perl g++ lzma curl locales psmisc sudo tree libgeos-dev libgflags2.2 libgflags-dev libgoogle-glog-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g build-essential valgrind rsync vim libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common r-base iputils-ping ``` -#### Install build dependencies for taosTools +#### Install Go for linux -To build the [taosTools](https://github.com/taosdata/taos-tools) on Ubuntu/Debian, the following packages need to be installed. +Update the installation package to version 1.23.3. ```bash -sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config +cd /usr/local/ +wget https://golang.google.cn/dl/go1.23.3.linux-amd64.tar.gz +rm -rf /usr/local/go && tar -C /usr/local -xzf go1.23.3.linux-amd64.tar.gz ``` -### CentOS 7.9 +Set up environment variables, first add the following content to the end of the `~/.bashrc` file. ```bash -sudo yum install epel-release -sudo yum update -sudo yum install -y gcc gcc-c++ make cmake3 gflags git openssl-devel -sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake +export GO_HOME=/usr/local/go +export PATH=$GO_HOME/bin:$PATH +export CGO_ENABLED=1 ``` -### CentOS 8/Fedora/Rocky Linux +Then make the environment variables take effect. ```bash -sudo dnf install -y gcc gcc-c++ make cmake epel-release gflags git openssl-devel +source ~/.bashrc ``` -#### Install build dependencies for taosTools on CentOS +Configure proxy to accelerate the download of Go dependencies. -#### CentOS 7.9 - -``` -sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel +```bash +go env -w GOPROXY=https://goproxy.cn,direct +go env -w GO111MODULE=on ``` -#### CentOS 8/Fedora/Rocky Linux +Check if the environment variables have taken effect and if the version is the installed version. -``` -sudo yum install -y epel-release -sudo yum install -y dnf-plugins-core -sudo yum config-manager --set-enabled powertools -sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel +```bash +go env +go version ``` -Note: Since snappy lacks pkg-config support (refer to [link](https://github.com/google/snappy/pull/86)), it leads a cmake prompt libsnappy not found. But snappy still works well. +#### Install node for linux -If the PowerTools installation fails, you can try to use: +Recommend install node using nvm. -``` -sudo yum config-manager --set-enabled powertools +```bash +curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.8/install.sh | bash ``` -#### For CentOS + devtoolset +Set up environment variables, add the following content to the end of the `~/.bashrc` file. -Besides above dependencies, please run following commands: - -``` -sudo yum install centos-release-scl -sudo yum install devtoolset-9 devtoolset-9-libatomic-devel -scl enable devtoolset-9 -- bash +```bash +export NVM_DIR="$HOME/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" ``` -### macOS +Then make the environment variables take effect. + +```bash +source ~/.bashrc +``` + +Finally, Install node and yarn. + +```bash +nvm install 16.20.2 +npm config set registry=https://registry.npmmirror.com +npm install -g yarn +``` + +#### Install Python-connector for linux + +Install Python3. + +```bash +apt install python3 +apt install python3-pip +``` + +Install the dependent Python components. + +```bash +pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog +``` + +Install the Python connector for TDengine. + +```bash +pip3 install taospy taos-ws-py +``` + +### 3.1.2 Windows platform + +#### Install the required package for windows + + +### 3.1.3 MacOS platform + +#### Install the required package for macOS ``` brew install argp-standalone gflags pkgconfig ``` -### Setup golang environment - -TDengine includes a few components like taosAdapter developed by Go language. Please refer to golang.org official documentation for golang environment setup. - -Please use version 1.20+. For the user in China, we recommend using a proxy to accelerate package downloading. - -``` -go env -w GO111MODULE=on -go env -w GOPROXY=https://goproxy.cn,direct -``` - -The default will not build taosAdapter, but you can use the following command to build taosAdapter as the service for RESTful interface. - -``` -cmake .. -DBUILD_HTTP=false -``` - -### Setup rust environment - -TDengine includes a few components developed by Rust language. Please refer to rust-lang.org official documentation for rust environment setup. - ## 3.2 Get the source codes First of all, you may clone the source codes from github: @@ -188,9 +233,17 @@ You can modify the file ~/.gitconfig to use ssh protocol instead of https for be [JDBC Connector](https://github.com/taosdata/taos-connector-jdbc)， [Go Connector](https://github.com/taosdata/driver-go)，[Python Connector](https://github.com/taosdata/taos-connector-python)，[Node.js Connector](https://github.com/taosdata/taos-connector-node)，[C# Connector](https://github.com/taosdata/taos-connector-dotnet) ，[Rust Connector](https://github.com/taosdata/taos-connector-rust) and [Grafana plugin](https://github.com/taosdata/grafanaplugin) has been moved to standalone repository. -## 3.4 Build TDengine +# 4. Building -### On Linux platform +At the moment, TDengine server supports running on Linux/Windows/MacOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service. TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. Right now we don't support build with cross-compiling environment. + +You can choose to install through source code, [container](https://docs.tdengine.com/get-started/docker/), [installation package](https://docs.tdengine.com/get-started/package/) or [Kubernetes](https://docs.tdengine.com/deployment/k8s/). This quick guide only applies to installing from source. + +TDengine provide a few useful tools such as taosBenchmark (was named taosdemo) and taosdump. They were part of TDengine. By default, TDengine compiling does not include taosTools. You can use `cmake .. -DBUILD_TOOLS=true` to make them be compiled with TDengine. + +To build TDengine, use [CMake](https://cmake.org/) 3.13.0 or higher versions in the project directory. + +## 4.1 Linux platform building You can run the bash script `build.sh` to build both TDengine and taosTools including taosBenchmark and taosdump as below: @@ -223,7 +276,7 @@ aarch64: cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` -### On Windows platform +## 4.2 Windows platform building If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". Please specify "amd64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. @@ -255,7 +308,7 @@ cmake .. -G "NMake Makefiles" nmake ``` -### On macOS platform +## 4.3 MacOS platform building Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. @@ -264,9 +317,17 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` -# 4. Installing +# 5. Packaging -## 4.1 On Linux platform +## 5.1 Linux platform packaging + +## 5.2 Windows platform packaging + +## 5.3 MacOS platform packaging + +# 6. Installation + +## 6.1 Linux platform installation After building successfully, TDengine can be installed by @@ -278,21 +339,7 @@ Users can find more information about directories installed on the system in the Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/package/) for it. -To start the service after installation, in a terminal, use: - -```bash -sudo systemctl start taosd -``` - -Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: - -```bash -taos -``` - -If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. - -## 4.2 On Windows platform +## 6.2 Windows platform installation After building successfully, TDengine can be installed by: @@ -300,7 +347,7 @@ After building successfully, TDengine can be installed by: nmake install ``` -## 4.3 On macOS platform +## 6.3 MacOS platform installation After building successfully, TDengine can be installed by: @@ -326,7 +373,23 @@ taos If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. -## 4.4 Quick Run +# 7. Running + +## 7.1 Linux platform + +To start the service after installation, in a terminal, use: + +```bash +sudo systemctl start taosd +``` + +Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: + +```bash +taos +``` + +If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: (We take Linux as an example, command on Windows will be `taosd.exe`) @@ -342,48 +405,44 @@ In another terminal, use the TDengine CLI to connect the server: option "-c test/cfg" specifies the system configuration file directory. -# 5. Try TDengine +## 7.2 Windows platform -It is easy to run SQL commands from TDengine CLI which is the same as other SQL databases. +## 7.3 MacOS platform -```sql -CREATE DATABASE demo; -USE demo; -CREATE TABLE t (ts TIMESTAMP, speed INT); -INSERT INTO t VALUES('2019-07-15 00:00:00', 10); -INSERT INTO t VALUES('2019-07-15 01:00:00', 20); -SELECT * FROM t; - ts | speed | -=================================== - 19-07-15 00:00:00.000| 10| - 19-07-15 01:00:00.000| 20| -Query OK, 2 row(s) in set (0.001700s) +# 8. Testing + +## 8.1 Linux platform testing + +## 8.2 Windows platform testing + +## 8.3 MacOS platform testing + +# 9. Releasing + +## 9.1 Linux platform releasing + +## 9.2 Windows platform realeasing + +## 9.3 MacOS platform releasing + +# 10. CI/CD +We use jenkins for CI/CD workflow configuration. See http://ci.bl.taosdata.com:8080/job/NewTest/view/change-requests/ +We can also run ci script locally. + +```bash +cd /root/TDengine/tests +./run_all_ci_cases.sh ``` -# 6. Developing with TDengine +# 11. Coverage +We can see coverage result in https://coveralls.io/github/taosdata/TDengine +We can also run coverage script locally. -## Official Connectors +```bash +cd /root/TDengine/tests +./run_local_coverage.sh +``` -TDengine provides abundant developing tools for users to develop on TDengine. Follow the links below to find your desired connectors and relevant documentation. - -- [Java](https://docs.tdengine.com/reference/connectors/java/) -- [C/C++](https://docs.tdengine.com/reference/connectors/cpp/) -- [Python](https://docs.tdengine.com/reference/connectors/python/) -- [Go](https://docs.tdengine.com/reference/connectors/go/) -- [Node.js](https://docs.tdengine.com/reference/connectors/node/) -- [Rust](https://docs.tdengine.com/reference/connectors/rust/) -- [C#](https://docs.tdengine.com/reference/connectors/csharp/) -- [RESTful API](https://docs.tdengine.com/reference/connectors/rest-api/) - -# 7. Contribute to TDengine +# 12. Contributing Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to the project. - -# 8. Join the TDengine Community - -For more information about TDengine, you can follow us on social media and join our Discord server: - -- [Discord](https://discord.com/invite/VZdSuUg4pS) -- [Twitter](https://twitter.com/TDengineDB) -- [LinkedIn](https://www.linkedin.com/company/tdengine/) -- [YouTube](https://www.youtube.com/@tdengine) From a33c2485a967147666e018b61ffd83d4b2fc0826 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Mon, 6 Jan 2025 14:08:16 +0800 Subject: [PATCH 03/87] Create CODEOWNERS --- .github/CODEOWNERS | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..a7eea818ab --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,16 @@ +# reference +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +# merge team +# @guanshengliang Shengliang Guan +# @zitsen Linhe Huo +# @wingwing2005 Ya Qiang Li +# @feici02 WANG Xu +# @hzcheng Hongze Cheng +# @dapan1121 Pan Wei +# @sheyanjie-qq She Yanjie +# @pigzhou ZacharyZhou + +* @taosdata/merge +/.github/ @feici02 +/packaging/ @feici02 From 42bbc940bf81579c9baa7dc2e463a589e72c710a Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Mon, 6 Jan 2025 14:16:00 +0800 Subject: [PATCH 04/87] Update CODEOWNERS --- .github/CODEOWNERS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a7eea818ab..aa88b3f791 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,13 +2,13 @@ # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners # merge team -# @guanshengliang Shengliang Guan +# @guanshengliang Shengliang Guan # @zitsen Linhe Huo # @wingwing2005 Ya Qiang Li # @feici02 WANG Xu # @hzcheng Hongze Cheng -# @dapan1121 Pan Wei -# @sheyanjie-qq She Yanjie +# @dapan1121 Pan Wei +# @sheyanjie-qq She Yanjie # @pigzhou ZacharyZhou * @taosdata/merge From 5822d7a6b0ce27b662deb4b8e7740753adba847e Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Mon, 6 Jan 2025 17:53:28 +0800 Subject: [PATCH 05/87] doc:add test cases --- README.md | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dfd1be23c7..363194f202 100644 --- a/README.md +++ b/README.md @@ -413,6 +413,64 @@ option "-c test/cfg" specifies the system configuration file directory. ## 8.1 Linux platform testing +### 8.1.1 Run the TSIM test script +Run the TSIM test script to simulate TDengine. +```bash +cd /root/TDengine/tests/script +./test.sh -f tsim/db/basic1.sim +``` + +### 8.1.2 Run the Python test script +Run the Python test script to simulate TDengine. +```bash +cd /root/TDengine/tests/system-test +python3 ./test.py -f 2-query/floor.py +``` + +### 8.1.3 Run unittest +Run the unit test script to simulate TDengine. +```bash +cd /root/TDengine/tests/unit-test/ +bash test.sh +``` + +### 8.1.4 Smoke Testing +Run the smoke test script to simulate TDengine. +```bash +cd /root/TDengine/packaging/smokeTest +./test_smoking_selfhost.sh +``` + +### 8.1.5 TSBS Test --replace +Run the TSBS test script to simulate TDengine. +1. Clone the code +```bash +cd /root && git clone https://github.com/taosdata/tsbs.git && cd tsbs/scripts/tsdbComp +``` +2. Modify IP and host of client and server in `test.ini` +```ini +clientIP="192.168.0.203" # client ip +clientHost="trd03" # client hostname +serverIP="192.168.0.204" # server ip +serverHost="trd04" # server hostname +``` +3. Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: +```ini +serverPass="taosdata123" # server root password +``` +4. Run the following command to start the test: + ```bash +nohup bash tsdbComparison.sh > test.log & +``` +5. When the test is done, the result can be found in `/data2/` directory, which can also be configured in `test.ini`. + +### 8.1.6 Crash_gen Test +Run the crash_gen script to simulate TDengine. +```bash +cd /root/TDengine/tests/pytest/ && ./crash_gen.sh +``` + + ## 8.2 Windows platform testing ## 8.3 MacOS platform testing @@ -425,16 +483,16 @@ option "-c test/cfg" specifies the system configuration file directory. ## 9.3 MacOS platform releasing -# 10. CI/CD +# 10. CI/CD We use jenkins for CI/CD workflow configuration. See http://ci.bl.taosdata.com:8080/job/NewTest/view/change-requests/ We can also run ci script locally. ```bash cd /root/TDengine/tests -./run_all_ci_cases.sh +./run_all_ci_cases.sh -b main ``` -# 11. Coverage +# 11. Coverage --replace We can see coverage result in https://coveralls.io/github/taosdata/TDengine We can also run coverage script locally. From 1e2e526f960ec13103e0909ce72a27f8bcd697e6 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Tue, 7 Jan 2025 14:56:40 +0800 Subject: [PATCH 06/87] update structure of readme to remove the second layer title by charles --- README.md | 188 ++++++++++++++++++++++++++---------------------------- 1 file changed, 89 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 363194f202..62c3d6c5c9 100644 --- a/README.md +++ b/README.md @@ -31,43 +31,22 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine - [2. Documentation](#2-documentation) - [3. Prerequisites](#3-prerequisites) - [3.1 Install dependencies tools](#31-install-dependencies-tools) - - [3.1.1 Linux platform](#311-linux-platform) - - [Install the required package for linux](#install-the-required-package-for-linux) - - [Install Go for linux](#install-go-for-linux) - - [Install node for linux](#install-node-for-linux) - - [Install Python-connector for linux](#install-python-connector-for-linux) - - [3.1.2 Windows platform](#312-windows-platform) - - [Install the required package for windows](#install-the-required-package-for-windows) - - [3.1.3 MacOS platform](#313-macos-platform) - - [Install the required package for macOS](#install-the-required-package-for-macos) - [3.2 Get the source codes](#32-get-the-source-codes) - [3.3 Special Note](#33-special-note) - [4. Building](#4-building) - - [4.1 Linux platform building](#41-linux-platform-building) - - [4.2 Windows platform building](#42-windows-platform-building) - - [4.3 MacOS platform building](#43-macos-platform-building) - [5. Packaging](#5-packaging) - - [5.1 Linux platform packaging](#51-linux-platform-packaging) - - [5.2 Windows platform packaging](#52-windows-platform-packaging) - - [5.3 MacOS platform packaging](#53-macos-platform-packaging) - [6. Installation](#6-installation) - - [6.1 Linux platform installation](#61-linux-platform-installation) - - [6.2 Windows platform installation](#62-windows-platform-installation) - - [6.3 MacOS platform installation](#63-macos-platform-installation) - [7. Running](#7-running) - - [7.1 Linux platform](#71-linux-platform) - - [7.2 Windows platform](#72-windows-platform) - - [7.3 MacOS platform](#73-macos-platform) - [8. Testing](#8-testing) - - [8.1 Linux platform testing](#81-linux-platform-testing) - - [8.2 Windows platform testing](#82-windows-platform-testing) - - [8.3 MacOS platform testing](#83-macos-platform-testing) + - [8.1 Run the TSIM test script](#81-run-the-tsim-test-script) + - [8.2 Run the Python test script](#82-run-the-python-test-script) + - [8.3 Run unittest](#83-run-unittest) + - [8.4 Smoke Testing](#84-smoke-testing) + - [8.5 TSBS Test --replace](#85-tsbs-test---replace) + - [8.6 Crash\_gen Test](#86-crash_gen-test) - [9. Releasing](#9-releasing) - - [9.1 Linux platform releasing](#91-linux-platform-releasing) - - [9.2 Windows platform realeasing](#92-windows-platform-realeasing) - - [9.3 MacOS platform releasing](#93-macos-platform-releasing) - [10. CI/CD](#10-cicd) -- [11. Coverage](#11-coverage) +- [11. Coverage --replace](#11-coverage---replace) - [12. Contributing](#12-contributing) # 1. Introduction @@ -104,15 +83,13 @@ Step-by-step instructions to set up the prerequisites software. ## 3.1 Install dependencies tools -### 3.1.1 Linux platform - -#### Install the required package for linux +Install the required package for linux ```bash apt-get install -y llvm gcc make cmake libssl-dev pkg-config perl g++ lzma curl locales psmisc sudo tree libgeos-dev libgflags2.2 libgflags-dev libgoogle-glog-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g build-essential valgrind rsync vim libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common r-base iputils-ping ``` -#### Install Go for linux +Install Go for linux Update the installation package to version 1.23.3. @@ -150,7 +127,7 @@ go env go version ``` -#### Install node for linux +Install node for linux Recommend install node using nvm. @@ -179,7 +156,7 @@ npm config set registry=https://registry.npmmirror.com npm install -g yarn ``` -#### Install Python-connector for linux +Install Python-connector for linux Install Python3. @@ -200,14 +177,11 @@ Install the Python connector for TDengine. pip3 install taospy taos-ws-py ``` -### 3.1.2 Windows platform +Install the required package for windows -#### Install the required package for windows +to be updated... - -### 3.1.3 MacOS platform - -#### Install the required package for macOS +Install the required package for macOS ``` brew install argp-standalone gflags pkgconfig @@ -243,7 +217,7 @@ TDengine provide a few useful tools such as taosBenchmark (was named taosdemo) a To build TDengine, use [CMake](https://cmake.org/) 3.13.0 or higher versions in the project directory. -## 4.1 Linux platform building +Linux platform building You can run the bash script `build.sh` to build both TDengine and taosTools including taosBenchmark and taosdump as below: @@ -276,7 +250,7 @@ aarch64: cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` -## 4.2 Windows platform building +4.2 Windows platform building If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". Please specify "amd64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. @@ -308,7 +282,7 @@ cmake .. -G "NMake Makefiles" nmake ``` -## 4.3 MacOS platform building +MacOS platform building Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. @@ -319,15 +293,21 @@ cmake .. && cmake --build . # 5. Packaging -## 5.1 Linux platform packaging +Linux platform packaging -## 5.2 Windows platform packaging +to be updated... -## 5.3 MacOS platform packaging +Windows platform packaging + +to be updated... + +MacOS platform packaging + +to be updated... # 6. Installation -## 6.1 Linux platform installation +Linux platform installation After building successfully, TDengine can be installed by @@ -337,9 +317,9 @@ sudo make install Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section. -Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/package/) for it. +Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/deploy-from-package/) for it. -## 6.2 Windows platform installation +Windows platform installation After building successfully, TDengine can be installed by: @@ -347,7 +327,7 @@ After building successfully, TDengine can be installed by: nmake install ``` -## 6.3 MacOS platform installation +MacOS platform installation After building successfully, TDengine can be installed by: @@ -355,29 +335,9 @@ After building successfully, TDengine can be installed by: sudo make install ``` -Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section. - -Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/package/) for it. - -To start the service after installation, double-click the /applications/TDengine to start the program, or in a terminal, use: - -```bash -sudo launchctl start com.tdengine.taosd -``` - -Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: - -```bash -taos -``` - -If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. - # 7. Running -## 7.1 Linux platform - -To start the service after installation, in a terminal, use: +To start the service after installation on linux, in a terminal, use: ```bash sudo systemctl start taosd @@ -405,87 +365,117 @@ In another terminal, use the TDengine CLI to connect the server: option "-c test/cfg" specifies the system configuration file directory. -## 7.2 Windows platform +To start the service after installation on windows, in a terminal, use: -## 7.3 MacOS platform +```bash +to be updated +``` +Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: + +```bash +to be updated +``` + +To start the service after installation on macOS, double-click the /applications/TDengine to start the program, or in a terminal, use: + +```bash +sudo launchctl start com.tdengine.taosd +``` + +Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: + +```bash +taos +``` + +If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. # 8. Testing -## 8.1 Linux platform testing +## 8.1 Run the TSIM test script + +TSIM test framework is developed by c++ in the start-up period of TDengine， the test scripts are basic cases, you can run +the script with below command: -### 8.1.1 Run the TSIM test script -Run the TSIM test script to simulate TDengine. ```bash cd /root/TDengine/tests/script ./test.sh -f tsim/db/basic1.sim ``` -### 8.1.2 Run the Python test script -Run the Python test script to simulate TDengine. +## 8.2 Run the Python test script +Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only +work for TDengine enterprise version, you can run the script with below command: + ```bash cd /root/TDengine/tests/system-test python3 ./test.py -f 2-query/floor.py ``` -### 8.1.3 Run unittest -Run the unit test script to simulate TDengine. +## 8.3 Run unittest +Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run +the script with below command: + ```bash cd /root/TDengine/tests/unit-test/ bash test.sh ``` -### 8.1.4 Smoke Testing -Run the smoke test script to simulate TDengine. +## 8.4 Smoke Testing +Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine, you can run the +script with below command: + ```bash cd /root/TDengine/packaging/smokeTest ./test_smoking_selfhost.sh ``` -### 8.1.5 TSBS Test --replace -Run the TSBS test script to simulate TDengine. +## 8.5 TSBS Test --replace +The Time Series Benchmark Suite(TSBS) test script provides a standardized approach to evaluate the performance of various +time series databases, you can run the script with below command: + 1. Clone the code ```bash cd /root && git clone https://github.com/taosdata/tsbs.git && cd tsbs/scripts/tsdbComp ``` -2. Modify IP and host of client and server in `test.ini` +1. Modify IP and host of client and server in `test.ini` ```ini clientIP="192.168.0.203" # client ip clientHost="trd03" # client hostname serverIP="192.168.0.204" # server ip serverHost="trd04" # server hostname ``` -3. Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: +1. Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: ```ini serverPass="taosdata123" # server root password ``` -4. Run the following command to start the test: +1. Run the following command to start the test: ```bash nohup bash tsdbComparison.sh > test.log & ``` -5. When the test is done, the result can be found in `/data2/` directory, which can also be configured in `test.ini`. +1. When the test is done, the result can be found in `/data2/` directory, which can also be configured in `test.ini`. + +## 8.6 Crash_gen Test +Crash_gen is a chaotic testing tool developed by TDengine, it can evaluate the resilience of TDengine through simulating +normal operation、intentional fault injection and their combination, you can run the script with below command: -### 8.1.6 Crash_gen Test -Run the crash_gen script to simulate TDengine. ```bash cd /root/TDengine/tests/pytest/ && ./crash_gen.sh ``` +Windows platform testing -## 8.2 Windows platform testing +to be updated... -## 8.3 MacOS platform testing +MacOS platform testing + +to be updated... # 9. Releasing -## 9.1 Linux platform releasing - -## 9.2 Windows platform realeasing - -## 9.3 MacOS platform releasing +You can access the [Release](https://github.com/taosdata/TDengine/releases) for TDengine release version list. # 10. CI/CD -We use jenkins for CI/CD workflow configuration. See http://ci.bl.taosdata.com:8080/job/NewTest/view/change-requests/ -We can also run ci script locally. +You can run ci script locally with below commands: ```bash cd /root/TDengine/tests From 16d15427120495d837e508adb18c10ebb60bf4b4 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Tue, 7 Jan 2025 16:44:39 +0800 Subject: [PATCH 07/87] udpate README to remove the second layer section for Building and Testing by charles --- README.md | 528 ++++++++++++++++++++++++++---------------------------- 1 file changed, 259 insertions(+), 269 deletions(-) diff --git a/README.md b/README.md index 62c3d6c5c9..d843f98ffa 100644 --- a/README.md +++ b/README.md @@ -30,20 +30,11 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine - [1. Introduction](#1-introduction) - [2. Documentation](#2-documentation) - [3. Prerequisites](#3-prerequisites) - - [3.1 Install dependencies tools](#31-install-dependencies-tools) - - [3.2 Get the source codes](#32-get-the-source-codes) - - [3.3 Special Note](#33-special-note) - [4. Building](#4-building) - [5. Packaging](#5-packaging) - [6. Installation](#6-installation) - [7. Running](#7-running) - [8. Testing](#8-testing) - - [8.1 Run the TSIM test script](#81-run-the-tsim-test-script) - - [8.2 Run the Python test script](#82-run-the-python-test-script) - - [8.3 Run unittest](#83-run-unittest) - - [8.4 Smoke Testing](#84-smoke-testing) - - [8.5 TSBS Test --replace](#85-tsbs-test---replace) - - [8.6 Crash\_gen Test](#86-crash_gen-test) - [9. Releasing](#9-releasing) - [10. CI/CD](#10-cicd) - [11. Coverage --replace](#11-coverage---replace) @@ -81,131 +72,130 @@ List the software and tools required to work on the project. Step-by-step instructions to set up the prerequisites software. -## 3.1 Install dependencies tools +- Install dependencies tools -Install the required package for linux + - The required package for linux -```bash -apt-get install -y llvm gcc make cmake libssl-dev pkg-config perl g++ lzma curl locales psmisc sudo tree libgeos-dev libgflags2.2 libgflags-dev libgoogle-glog-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g build-essential valgrind rsync vim libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common r-base iputils-ping -``` + ```bash + apt-get install -y llvm gcc make cmake libssl-dev pkg-config perl g++ lzma curl locales psmisc sudo tree libgeos-dev libgflags2.2 libgflags-dev libgoogle-glog-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g build-essential valgrind rsync vim libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common r-base iputils-ping + ``` -Install Go for linux + Install Go for linux -Update the installation package to version 1.23.3. + Update the installation package to version 1.23.3. -```bash -cd /usr/local/ -wget https://golang.google.cn/dl/go1.23.3.linux-amd64.tar.gz -rm -rf /usr/local/go && tar -C /usr/local -xzf go1.23.3.linux-amd64.tar.gz -``` + ```bash + cd /usr/local/ + wget https://golang.google.cn/dl/go1.23.3.linux-amd64.tar.gz + rm -rf /usr/local/go && tar -C /usr/local -xzf go1.23.3.linux-amd64.tar.gz + ``` -Set up environment variables, first add the following content to the end of the `~/.bashrc` file. + Set up environment variables, first add the following content to the end of the `~/.bashrc` file. -```bash -export GO_HOME=/usr/local/go -export PATH=$GO_HOME/bin:$PATH -export CGO_ENABLED=1 -``` + ```bash + export GO_HOME=/usr/local/go + export PATH=$GO_HOME/bin:$PATH + export CGO_ENABLED=1 + ``` -Then make the environment variables take effect. + Then make the environment variables take effect. -```bash -source ~/.bashrc -``` + ```bash + source ~/.bashrc + ``` -Configure proxy to accelerate the download of Go dependencies. + Configure proxy to accelerate the download of Go dependencies. -```bash -go env -w GOPROXY=https://goproxy.cn,direct -go env -w GO111MODULE=on -``` + ```bash + go env -w GOPROXY=https://goproxy.cn,direct + go env -w GO111MODULE=on + ``` -Check if the environment variables have taken effect and if the version is the installed version. + Check if the environment variables have taken effect and if the version is the installed version. -```bash -go env -go version -``` + ```bash + go env + go version + ``` -Install node for linux + Install node for linux -Recommend install node using nvm. + Recommend install node using nvm. -```bash -curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.8/install.sh | bash -``` + ```bash + curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.8/install.sh | bash + ``` -Set up environment variables, add the following content to the end of the `~/.bashrc` file. + Set up environment variables, add the following content to the end of the `~/.bashrc` file. -```bash -export NVM_DIR="$HOME/.nvm" -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" -``` + ```bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + ``` -Then make the environment variables take effect. + Then make the environment variables take effect. -```bash -source ~/.bashrc -``` + ```bash + source ~/.bashrc + ``` -Finally, Install node and yarn. + Finally, Install node and yarn. -```bash -nvm install 16.20.2 -npm config set registry=https://registry.npmmirror.com -npm install -g yarn -``` + ```bash + nvm install 16.20.2 + npm config set registry=https://registry.npmmirror.com + npm install -g yarn + ``` -Install Python-connector for linux + Install Python-connector for linux -Install Python3. + Install Python3. -```bash -apt install python3 -apt install python3-pip -``` + ```bash + apt install python3 + apt install python3-pip + ``` -Install the dependent Python components. + Install the dependent Python components. -```bash -pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog -``` + ```bash + pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog + ``` -Install the Python connector for TDengine. + Install the Python connector for TDengine. -```bash -pip3 install taospy taos-ws-py -``` + ```bash + pip3 install taospy taos-ws-py + ``` -Install the required package for windows + - The required package for windows -to be updated... + to be updated... -Install the required package for macOS + - The required package for macOS -``` -brew install argp-standalone gflags pkgconfig -``` + ``` + brew install argp-standalone gflags pkgconfig + ``` -## 3.2 Get the source codes +- Get the source codes -First of all, you may clone the source codes from github: + First of all, you may clone the source codes from github: -```bash -git clone https://github.com/taosdata/TDengine.git -cd TDengine -``` + ```bash + git clone https://github.com/taosdata/TDengine.git + cd TDengine + ``` -You can modify the file ~/.gitconfig to use ssh protocol instead of https for better download speed. You will need to upload ssh public key to GitHub first. Please refer to GitHub official documentation for detail. + You can modify the file ~/.gitconfig to use ssh protocol instead of https for better download speed. You will need to upload ssh public key to GitHub first. Please refer to GitHub official documentation for detail. -``` -[url "git@github.com:"] - insteadOf = https://github.com/ -``` + ``` + [url "git@github.com:"] + insteadOf = https://github.com/ + ``` -## 3.3 Special Note - -[JDBC Connector](https://github.com/taosdata/taos-connector-jdbc)， [Go Connector](https://github.com/taosdata/driver-go)，[Python Connector](https://github.com/taosdata/taos-connector-python)，[Node.js Connector](https://github.com/taosdata/taos-connector-node)，[C# Connector](https://github.com/taosdata/taos-connector-dotnet) ，[Rust Connector](https://github.com/taosdata/taos-connector-rust) and [Grafana plugin](https://github.com/taosdata/grafanaplugin) has been moved to standalone repository. +- Special Note + [JDBC Connector](https://github.com/taosdata/taos-connector-jdbc)， [Go Connector](https://github.com/taosdata/driver-go)，[Python Connector](https://github.com/taosdata/taos-connector-python)，[Node.js Connector](https://github.com/taosdata/taos-connector-node)，[C# Connector](https://github.com/taosdata/taos-connector-dotnet) ，[Rust Connector](https://github.com/taosdata/taos-connector-rust) and [Grafana plugin](https://github.com/taosdata/grafanaplugin) has been moved to standalone repository. # 4. Building @@ -217,258 +207,258 @@ TDengine provide a few useful tools such as taosBenchmark (was named taosdemo) a To build TDengine, use [CMake](https://cmake.org/) 3.13.0 or higher versions in the project directory. -Linux platform building +- Linux platform building -You can run the bash script `build.sh` to build both TDengine and taosTools including taosBenchmark and taosdump as below: + You can run the bash script `build.sh` to build both TDengine and taosTools including taosBenchmark and taosdump as below: -```bash -./build.sh -``` + ```bash + ./build.sh + ``` -It equals to execute following commands: + It equals to execute following commands: -```bash -mkdir debug -cd debug -cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true -make -``` + ```bash + mkdir debug + cd debug + cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true + make + ``` -You can use Jemalloc as memory allocator instead of glibc: + You can use Jemalloc as memory allocator instead of glibc: -``` -apt install autoconf -cmake .. -DJEMALLOC_ENABLED=true -``` + ``` + apt install autoconf + cmake .. -DJEMALLOC_ENABLED=true + ``` -TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 platform. -You can also specify CPUTYPE option like aarch64 too if the detection result is not correct: + TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 platform. + You can also specify CPUTYPE option like aarch64 too if the detection result is not correct: -aarch64: + aarch64: -```bash -cmake .. -DCPUTYPE=aarch64 && cmake --build . -``` + ```bash + cmake .. -DCPUTYPE=aarch64 && cmake --build . + ``` -4.2 Windows platform building +- Windows platform building -If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". -Please specify "amd64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. + If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". + Please specify "amd64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. -```cmd -mkdir debug && cd debug -"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < amd64 | x86 > -cmake .. -G "NMake Makefiles" -nmake -``` + ```cmd + mkdir debug && cd debug + "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < amd64 | x86 > + cmake .. -G "NMake Makefiles" + nmake + ``` -If you use the Visual Studio 2019 or 2017: + If you use the Visual Studio 2019 or 2017: -please open a command window by executing "cmd.exe". -Please specify "x64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. + please open a command window by executing "cmd.exe". + Please specify "x64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. -```cmd -mkdir debug && cd debug -"c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 > -cmake .. -G "NMake Makefiles" -nmake -``` + ```cmd + mkdir debug && cd debug + "c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 > + cmake .. -G "NMake Makefiles" + nmake + ``` -Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: + Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: -```cmd -mkdir debug && cd debug -cmake .. -G "NMake Makefiles" -nmake -``` + ```cmd + mkdir debug && cd debug + cmake .. -G "NMake Makefiles" + nmake + ``` -MacOS platform building +- MacOS platform building -Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. + Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. -```shell -mkdir debug && cd debug -cmake .. && cmake --build . -``` + ```shell + mkdir debug && cd debug + cmake .. && cmake --build . + ``` # 5. Packaging -Linux platform packaging - -to be updated... - -Windows platform packaging - -to be updated... - -MacOS platform packaging - -to be updated... +To be updated... # 6. Installation -Linux platform installation +- Linux platform installation -After building successfully, TDengine can be installed by + After building successfully, TDengine can be installed by -```bash -sudo make install -``` + ```bash + sudo make install + ``` -Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section. + Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section. -Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/deploy-from-package/) for it. + Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/deploy-from-package/) for it. -Windows platform installation +- Windows platform installation -After building successfully, TDengine can be installed by: + After building successfully, TDengine can be installed by: -```cmd -nmake install -``` + ```cmd + nmake install + ``` -MacOS platform installation +- MacOS platform installation -After building successfully, TDengine can be installed by: + After building successfully, TDengine can be installed by: -```bash -sudo make install -``` + ```bash + sudo make install + ``` # 7. Running -To start the service after installation on linux, in a terminal, use: +- To start the service after installation on linux, in a terminal, use: -```bash -sudo systemctl start taosd -``` + ```bash + sudo systemctl start taosd + ``` -Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: + Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: -```bash -taos -``` + ```bash + taos + ``` -If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. + If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. -If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: (We take Linux as an example, command on Windows will be `taosd.exe`) + If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: (We take Linux as an example, command on Windows will be `taosd.exe`) -```bash -./build/bin/taosd -c test/cfg -``` + ```bash + ./build/bin/taosd -c test/cfg + ``` -In another terminal, use the TDengine CLI to connect the server: + In another terminal, use the TDengine CLI to connect the server: -```bash -./build/bin/taos -c test/cfg -``` + ```bash + ./build/bin/taos -c test/cfg + ``` -option "-c test/cfg" specifies the system configuration file directory. + option "-c test/cfg" specifies the system configuration file directory. -To start the service after installation on windows, in a terminal, use: +- To start the service after installation on windows, in a terminal, use: -```bash -to be updated -``` -Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: + ```bash + to be updated + ``` + Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: -```bash -to be updated -``` + ```bash + to be updated + ``` -To start the service after installation on macOS, double-click the /applications/TDengine to start the program, or in a terminal, use: +- To start the service after installation on macOS, double-click the /applications/TDengine to start the program, or in a terminal, use: -```bash -sudo launchctl start com.tdengine.taosd -``` + ```bash + sudo launchctl start com.tdengine.taosd + ``` -Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: + Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: -```bash -taos -``` + ```bash + taos + ``` If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. # 8. Testing -## 8.1 Run the TSIM test script +- Run the TSIM test script -TSIM test framework is developed by c++ in the start-up period of TDengine， the test scripts are basic cases, you can run -the script with below command: + TSIM test framework is developed by c++ in the start-up period of TDengine， the test scripts are basic cases, you can run + the script with below command: -```bash -cd /root/TDengine/tests/script -./test.sh -f tsim/db/basic1.sim -``` + ```bash + cd /root/TDengine/tests/script + ./test.sh -f tsim/db/basic1.sim + ``` -## 8.2 Run the Python test script -Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only -work for TDengine enterprise version, you can run the script with below command: +- Run the Python test script + + Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only + work for TDengine enterprise version, you can run the script with below command: -```bash -cd /root/TDengine/tests/system-test -python3 ./test.py -f 2-query/floor.py -``` + ```bash + cd /root/TDengine/tests/system-test + python3 ./test.py -f 2-query/floor.py + ``` -## 8.3 Run unittest -Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run -the script with below command: +- Run unittest + + Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run + the script with below command: -```bash -cd /root/TDengine/tests/unit-test/ -bash test.sh -``` + ```bash + cd /root/TDengine/tests/unit-test/ + bash test.sh + ``` -## 8.4 Smoke Testing -Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine, you can run the -script with below command: +- Smoke Testing + + Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine, you can run the + script with below command: -```bash -cd /root/TDengine/packaging/smokeTest -./test_smoking_selfhost.sh -``` + ```bash + cd /root/TDengine/packaging/smokeTest + ./test_smoking_selfhost.sh + ``` -## 8.5 TSBS Test --replace -The Time Series Benchmark Suite(TSBS) test script provides a standardized approach to evaluate the performance of various -time series databases, you can run the script with below command: +- TSBS Test --replace -1. Clone the code -```bash -cd /root && git clone https://github.com/taosdata/tsbs.git && cd tsbs/scripts/tsdbComp -``` -1. Modify IP and host of client and server in `test.ini` -```ini -clientIP="192.168.0.203" # client ip -clientHost="trd03" # client hostname -serverIP="192.168.0.204" # server ip -serverHost="trd04" # server hostname -``` -1. Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: -```ini -serverPass="taosdata123" # server root password -``` -1. Run the following command to start the test: - ```bash -nohup bash tsdbComparison.sh > test.log & -``` -1. When the test is done, the result can be found in `/data2/` directory, which can also be configured in `test.ini`. + The Time Series Benchmark Suite(TSBS) test script provides a standardized approach to evaluate the performance of various + time series databases, you can run the script with below command: -## 8.6 Crash_gen Test -Crash_gen is a chaotic testing tool developed by TDengine, it can evaluate the resilience of TDengine through simulating -normal operation、intentional fault injection and their combination, you can run the script with below command: + - Clone the code -```bash -cd /root/TDengine/tests/pytest/ && ./crash_gen.sh -``` + ```bash + cd /root && git clone https://github.com/taosdata/tsbs.git && cd tsbs/scripts/tsdbComp + ``` + + - Modify IP and host of client and server in `test.ini` + ```ini + clientIP="192.168.0.203" # client ip + clientHost="trd03" # client hostname + serverIP="192.168.0.204" # server ip + serverHost="trd04" # server hostname + ``` + + - Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: + + ```ini + serverPass="taosdata123" # server root password + ``` + + - Run the following command to start the test: + ```bash + nohup bash tsdbComparison.sh > test.log & + ``` + When the test is done, the result can be found in `/data2/` directory, which can also be configured in `test.ini`. + +- Crash_gen Test + + Crash_gen is a chaotic testing tool developed by TDengine, it can evaluate the resilience of TDengine through simulating + normal operation、intentional fault injection and their combination, you can run the script with below command: + + ```bash + cd /root/TDengine/tests/pytest/ && ./crash_gen.sh + ``` Windows platform testing -to be updated... + to be updated... MacOS platform testing -to be updated... + to be updated... # 9. Releasing From 54638bb89095835772678467608d23d22792526a Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Tue, 7 Jan 2025 16:59:19 +0800 Subject: [PATCH 08/87] Update CODEOWNERS --- .github/CODEOWNERS | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aa88b3f791..a95bfd30eb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,4 +13,14 @@ * @taosdata/merge /.github/ @feici02 +/cmake/ @guanshengliang +/contrib/ @guanshengliang +/deps/ @guanshengliang +/docs/ @guanshengliang @zitsen +/examples/ @guanshengliang @zitsen +/include/ @guanshengliang @hzcheng @dapan1121 /packaging/ @feici02 +/source/ @guanshengliang @hzcheng @dapan1121 +/tests/ @guanshengliang @zitsen +/tools/ @guanshengliang @zitsen +/utils/ @guanshengliang From 7ac75a8dba6ffbe01c335a16ce8c8e0900166f15 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Tue, 7 Jan 2025 18:18:14 +0800 Subject: [PATCH 09/87] Update toc --- README.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d843f98ffa..30d8f21d88 100644 --- a/README.md +++ b/README.md @@ -26,19 +26,18 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine # Table of Contents -- [Table of Contents](#table-of-contents) -- [1. Introduction](#1-introduction) -- [2. Documentation](#2-documentation) -- [3. Prerequisites](#3-prerequisites) -- [4. Building](#4-building) -- [5. Packaging](#5-packaging) -- [6. Installation](#6-installation) -- [7. Running](#7-running) -- [8. Testing](#8-testing) -- [9. Releasing](#9-releasing) -- [10. CI/CD](#10-cicd) -- [11. Coverage --replace](#11-coverage---replace) -- [12. Contributing](#12-contributing) +1. [Introduction](#1-introduction) +1. [Documentation](#2-documentation) +1. [Prerequisites](#3-prerequisites) +1. [Building](#4-building) +1. [Packaging](#5-packaging) +1. [Installation](#6-installation) +1. [Running](#7-running) +1. [Testing](#8-testing) +1. [Releasing](#9-releasing) +1. [CI/CD](#10-cicd) +1. [Coverage --replace](#11-coverage---replace) +1. [Contributing](#12-contributing) # 1. Introduction From c85f5833a0742fb74179db1fd98ad61dda602f3c Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 8 Jan 2025 08:18:39 +0800 Subject: [PATCH 10/87] adjust the contents style format by charles --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 30d8f21d88..fc938942d6 100644 --- a/README.md +++ b/README.md @@ -65,13 +65,13 @@ For user manual, system design and architecture, please refer to [TDengine Docum List the software and tools required to work on the project. -- go 1.20+ (for taosadapter) -- node 16.20.2 (for taos-explorer) -- python 3.10.12+ (for test) + - go 1.20+ (for taosadapter) + - node 16.20.2 (for taos-explorer) + - python 3.10.12+ (for test) Step-by-step instructions to set up the prerequisites software. -- Install dependencies tools +#### Install dependencies tools - The required package for linux @@ -177,7 +177,7 @@ Step-by-step instructions to set up the prerequisites software. brew install argp-standalone gflags pkgconfig ``` -- Get the source codes +#### Get the source codes First of all, you may clone the source codes from github: @@ -193,7 +193,7 @@ Step-by-step instructions to set up the prerequisites software. insteadOf = https://github.com/ ``` -- Special Note +#### Special Note [JDBC Connector](https://github.com/taosdata/taos-connector-jdbc)， [Go Connector](https://github.com/taosdata/driver-go)，[Python Connector](https://github.com/taosdata/taos-connector-python)，[Node.js Connector](https://github.com/taosdata/taos-connector-node)，[C# Connector](https://github.com/taosdata/taos-connector-dotnet) ，[Rust Connector](https://github.com/taosdata/taos-connector-rust) and [Grafana plugin](https://github.com/taosdata/grafanaplugin) has been moved to standalone repository. # 4. Building @@ -416,13 +416,13 @@ If TDengine CLI connects the server successfully, welcome messages and version i The Time Series Benchmark Suite(TSBS) test script provides a standardized approach to evaluate the performance of various time series databases, you can run the script with below command: - - Clone the code + Clone the code ```bash cd /root && git clone https://github.com/taosdata/tsbs.git && cd tsbs/scripts/tsdbComp ``` - - Modify IP and host of client and server in `test.ini` + Modify IP and host of client and server in `test.ini` ```ini clientIP="192.168.0.203" # client ip clientHost="trd03" # client hostname @@ -430,13 +430,13 @@ If TDengine CLI connects the server successfully, welcome messages and version i serverHost="trd04" # server hostname ``` - - Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: + Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: ```ini serverPass="taosdata123" # server root password ``` - - Run the following command to start the test: + Run the following command to start the test: ```bash nohup bash tsdbComparison.sh > test.log & ``` @@ -472,8 +472,8 @@ cd /root/TDengine/tests ``` # 11. Coverage --replace -We can see coverage result in https://coveralls.io/github/taosdata/TDengine -We can also run coverage script locally. +You can see coverage result in https://coveralls.io/github/taosdata/TDengine +You can also run coverage script locally. ```bash cd /root/TDengine/tests From a0be3dcaec62f73088b09423dff8823530b0b1bc Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Wed, 8 Jan 2025 10:40:06 +0800 Subject: [PATCH 11/87] update README to add child section number and remove tsbs, crash_gen test from testing part --- README.md | 459 ++++++++++++++++++++---------------------------------- 1 file changed, 165 insertions(+), 294 deletions(-) diff --git a/README.md b/README.md index fc938942d6..8c5ec10df2 100644 --- a/README.md +++ b/README.md @@ -63,138 +63,42 @@ For user manual, system design and architecture, please refer to [TDengine Docum # 3. Prerequisites -List the software and tools required to work on the project. +## 3.1 Install dependencies tools - - go 1.20+ (for taosadapter) - - node 16.20.2 (for taos-explorer) - - python 3.10.12+ (for test) +### The required package for linux -Step-by-step instructions to set up the prerequisites software. +```bash +apt-get install -y llvm gcc make cmake libssl-dev pkg-config perl g++ lzma curl locales psmisc sudo tree libgeos-dev libgflags2.2 libgflags-dev libgoogle-glog-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g build-essential valgrind rsync vim libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common r-base iputils-ping +``` -#### Install dependencies tools +### The required package for windows - - The required package for linux +to be updated... - ```bash - apt-get install -y llvm gcc make cmake libssl-dev pkg-config perl g++ lzma curl locales psmisc sudo tree libgeos-dev libgflags2.2 libgflags-dev libgoogle-glog-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g build-essential valgrind rsync vim libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common r-base iputils-ping - ``` +### The required package for macOS - Install Go for linux +``` +brew install argp-standalone gflags pkgconfig +``` - Update the installation package to version 1.23.3. +## 3.2 Get the source codes - ```bash - cd /usr/local/ - wget https://golang.google.cn/dl/go1.23.3.linux-amd64.tar.gz - rm -rf /usr/local/go && tar -C /usr/local -xzf go1.23.3.linux-amd64.tar.gz - ``` +First of all, you may clone the source codes from github: - Set up environment variables, first add the following content to the end of the `~/.bashrc` file. +```bash +git clone https://github.com/taosdata/TDengine.git +cd TDengine +``` - ```bash - export GO_HOME=/usr/local/go - export PATH=$GO_HOME/bin:$PATH - export CGO_ENABLED=1 - ``` +You can modify the file ~/.gitconfig to use ssh protocol instead of https for better download speed. You will need to upload ssh public key to GitHub first. Please refer to GitHub official documentation for detail. - Then make the environment variables take effect. +``` +[url "git@github.com:"] + insteadOf = https://github.com/ +``` - ```bash - source ~/.bashrc - ``` - - Configure proxy to accelerate the download of Go dependencies. - - ```bash - go env -w GOPROXY=https://goproxy.cn,direct - go env -w GO111MODULE=on - ``` - - Check if the environment variables have taken effect and if the version is the installed version. - - ```bash - go env - go version - ``` - - Install node for linux - - Recommend install node using nvm. - - ```bash - curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.8/install.sh | bash - ``` - - Set up environment variables, add the following content to the end of the `~/.bashrc` file. - - ```bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - ``` - - Then make the environment variables take effect. - - ```bash - source ~/.bashrc - ``` - - Finally, Install node and yarn. - - ```bash - nvm install 16.20.2 - npm config set registry=https://registry.npmmirror.com - npm install -g yarn - ``` - - Install Python-connector for linux - - Install Python3. - - ```bash - apt install python3 - apt install python3-pip - ``` - - Install the dependent Python components. - - ```bash - pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog - ``` - - Install the Python connector for TDengine. - - ```bash - pip3 install taospy taos-ws-py - ``` - - - The required package for windows - - to be updated... - - - The required package for macOS - - ``` - brew install argp-standalone gflags pkgconfig - ``` - -#### Get the source codes - - First of all, you may clone the source codes from github: - - ```bash - git clone https://github.com/taosdata/TDengine.git - cd TDengine - ``` - - You can modify the file ~/.gitconfig to use ssh protocol instead of https for better download speed. You will need to upload ssh public key to GitHub first. Please refer to GitHub official documentation for detail. - - ``` - [url "git@github.com:"] - insteadOf = https://github.com/ - ``` - -#### Special Note - [JDBC Connector](https://github.com/taosdata/taos-connector-jdbc)， [Go Connector](https://github.com/taosdata/driver-go)，[Python Connector](https://github.com/taosdata/taos-connector-python)，[Node.js Connector](https://github.com/taosdata/taos-connector-node)，[C# Connector](https://github.com/taosdata/taos-connector-dotnet) ，[Rust Connector](https://github.com/taosdata/taos-connector-rust) and [Grafana plugin](https://github.com/taosdata/grafanaplugin) has been moved to standalone repository. +## 3.3 Special Note +[JDBC Connector](https://github.com/taosdata/taos-connector-jdbc)， [Go Connector](https://github.com/taosdata/driver-go)，[Python Connector](https://github.com/taosdata/taos-connector-python)，[Node.js Connector](https://github.com/taosdata/taos-connector-node)，[C# Connector](https://github.com/taosdata/taos-connector-dotnet) ，[Rust Connector](https://github.com/taosdata/taos-connector-rust) and [Grafana plugin](https://github.com/taosdata/grafanaplugin) has been moved to standalone repository. # 4. Building @@ -206,79 +110,79 @@ TDengine provide a few useful tools such as taosBenchmark (was named taosdemo) a To build TDengine, use [CMake](https://cmake.org/) 3.13.0 or higher versions in the project directory. -- Linux platform building +## 4.1 Linux platform building - You can run the bash script `build.sh` to build both TDengine and taosTools including taosBenchmark and taosdump as below: +You can run the bash script `build.sh` to build both TDengine and taosTools including taosBenchmark and taosdump as below: - ```bash - ./build.sh - ``` +```bash +./build.sh +``` - It equals to execute following commands: +It equals to execute following commands: - ```bash - mkdir debug - cd debug - cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true - make - ``` +```bash +mkdir debug +cd debug +cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true +make +``` - You can use Jemalloc as memory allocator instead of glibc: +You can use Jemalloc as memory allocator instead of glibc: - ``` - apt install autoconf - cmake .. -DJEMALLOC_ENABLED=true - ``` +``` +apt install autoconf +cmake .. -DJEMALLOC_ENABLED=true +``` - TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 platform. - You can also specify CPUTYPE option like aarch64 too if the detection result is not correct: +TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 platform. +You can also specify CPUTYPE option like aarch64 too if the detection result is not correct: - aarch64: +aarch64: - ```bash - cmake .. -DCPUTYPE=aarch64 && cmake --build . - ``` +```bash +cmake .. -DCPUTYPE=aarch64 && cmake --build . +``` -- Windows platform building +## 4.2 Windows platform building - If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". - Please specify "amd64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. +If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". +Please specify "amd64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. - ```cmd - mkdir debug && cd debug - "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < amd64 | x86 > - cmake .. -G "NMake Makefiles" - nmake - ``` +```cmd +mkdir debug && cd debug +"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < amd64 | x86 > +cmake .. -G "NMake Makefiles" +nmake +``` - If you use the Visual Studio 2019 or 2017: +If you use the Visual Studio 2019 or 2017: - please open a command window by executing "cmd.exe". - Please specify "x64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. +please open a command window by executing "cmd.exe". +Please specify "x64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. - ```cmd - mkdir debug && cd debug - "c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 > - cmake .. -G "NMake Makefiles" - nmake - ``` +```cmd +mkdir debug && cd debug +"c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 > +cmake .. -G "NMake Makefiles" +nmake +``` - Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: +Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: - ```cmd - mkdir debug && cd debug - cmake .. -G "NMake Makefiles" - nmake - ``` +```cmd +mkdir debug && cd debug +cmake .. -G "NMake Makefiles" +nmake +``` -- MacOS platform building +## 4.3 MacOS platform building - Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. +Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. - ```shell - mkdir debug && cd debug - cmake .. && cmake --build . - ``` +```shell +mkdir debug && cd debug +cmake .. && cmake --build . +``` # 5. Packaging @@ -286,176 +190,143 @@ To be updated... # 6. Installation -- Linux platform installation +## 6.1 Linux platform installation - After building successfully, TDengine can be installed by +After building successfully, TDengine can be installed by - ```bash - sudo make install - ``` +```bash +sudo make install +``` - Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section. +Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section. - Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/deploy-from-package/) for it. +Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/deploy-from-package/) for it. -- Windows platform installation +## 6.2 Windows platform installation - After building successfully, TDengine can be installed by: +After building successfully, TDengine can be installed by: - ```cmd - nmake install - ``` +```cmd +nmake install +``` -- MacOS platform installation +## 6.3 MacOS platform installation - After building successfully, TDengine can be installed by: +After building successfully, TDengine can be installed by: - ```bash - sudo make install - ``` +```bash +sudo make install +``` # 7. Running -- To start the service after installation on linux, in a terminal, use: +## 7.1 Run TDengine on linux - ```bash - sudo systemctl start taosd - ``` +To start the service after installation on linux, in a terminal, use: - Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: +```bash +sudo systemctl start taosd +``` - ```bash - taos - ``` +Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: - If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. +```bash +taos +``` - If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: (We take Linux as an example, command on Windows will be `taosd.exe`) +If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. - ```bash - ./build/bin/taosd -c test/cfg - ``` +If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: (We take Linux as an example, command on Windows will be `taosd.exe`) - In another terminal, use the TDengine CLI to connect the server: +```bash +./build/bin/taosd -c test/cfg +``` - ```bash - ./build/bin/taos -c test/cfg - ``` +In another terminal, use the TDengine CLI to connect the server: - option "-c test/cfg" specifies the system configuration file directory. +```bash +./build/bin/taos -c test/cfg +``` -- To start the service after installation on windows, in a terminal, use: +option "-c test/cfg" specifies the system configuration file directory. - ```bash - to be updated - ``` - Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: +## 7.2 Run TDengine on windows - ```bash - to be updated - ``` +To start the service after installation on windows, in a terminal, use: -- To start the service after installation on macOS, double-click the /applications/TDengine to start the program, or in a terminal, use: +```bash +to be updated +``` - ```bash - sudo launchctl start com.tdengine.taosd - ``` +Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: - Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: +```bash +taos +``` - ```bash - taos - ``` +## 7.3 Run TDengine on MacOS + +To start the service after installation on macOS, double-click the /applications/TDengine to start the program, or in a terminal, use: + +```bash +sudo launchctl start com.tdengine.taosd +``` + +Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: + +```bash +taos +``` If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. # 8. Testing -- Run the TSIM test script +## 8.1 Run the TSIM test script - TSIM test framework is developed by c++ in the start-up period of TDengine， the test scripts are basic cases, you can run - the script with below command: +TSIM test framework is developed by c++ in the start-up period of TDengine， the test scripts are basic cases, you can run +the script with below command: - ```bash - cd /root/TDengine/tests/script - ./test.sh -f tsim/db/basic1.sim - ``` +```bash +cd /root/TDengine/tests/script +./test.sh -f tsim/db/basic1.sim +``` -- Run the Python test script - - Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only - work for TDengine enterprise version, you can run the script with below command: +## 8.2 Run the Python test script - ```bash - cd /root/TDengine/tests/system-test - python3 ./test.py -f 2-query/floor.py - ``` +Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only +work for TDengine enterprise version, you can run the script with below command: -- Run unittest - - Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run - the script with below command: +```bash +cd /root/TDengine/tests/system-test +python3 ./test.py -f 2-query/floor.py +``` - ```bash - cd /root/TDengine/tests/unit-test/ - bash test.sh - ``` +## 8.3 Run unittest -- Smoke Testing - - Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine, you can run the - script with below command: +Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run +the script with below command: - ```bash - cd /root/TDengine/packaging/smokeTest - ./test_smoking_selfhost.sh - ``` +```bash +cd /root/TDengine/tests/unit-test/ +bash test.sh +``` -- TSBS Test --replace +## 8.4 Smoke Testing - The Time Series Benchmark Suite(TSBS) test script provides a standardized approach to evaluate the performance of various - time series databases, you can run the script with below command: +Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine, you can run the +script with below command: - Clone the code +```bash +cd /root/TDengine/packaging/smokeTest +./test_smoking_selfhost.sh +``` - ```bash - cd /root && git clone https://github.com/taosdata/tsbs.git && cd tsbs/scripts/tsdbComp - ``` - - Modify IP and host of client and server in `test.ini` - ```ini - clientIP="192.168.0.203" # client ip - clientHost="trd03" # client hostname - serverIP="192.168.0.204" # server ip - serverHost="trd04" # server hostname - ``` - - Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: - - ```ini - serverPass="taosdata123" # server root password - ``` - - Run the following command to start the test: - ```bash - nohup bash tsdbComparison.sh > test.log & - ``` - When the test is done, the result can be found in `/data2/` directory, which can also be configured in `test.ini`. - -- Crash_gen Test - - Crash_gen is a chaotic testing tool developed by TDengine, it can evaluate the resilience of TDengine through simulating - normal operation、intentional fault injection and their combination, you can run the script with below command: - - ```bash - cd /root/TDengine/tests/pytest/ && ./crash_gen.sh - ``` - -Windows platform testing +## Windows platform testing to be updated... -MacOS platform testing +## MacOS platform testing to be updated... From abbc6416328b44ce8d1d2de33e34b513191be5b5 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Wed, 8 Jan 2025 18:49:24 +0800 Subject: [PATCH 12/87] docs: refactor readme --- README.md | 203 +++++++++++++++++++----------------------------- tests/README.md | 43 ++++++++++ 2 files changed, 124 insertions(+), 122 deletions(-) create mode 100644 tests/README.md diff --git a/README.md b/README.md index 8c5ec10df2..5fadbd22d8 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,13 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine 1. [Documentation](#2-documentation) 1. [Prerequisites](#3-prerequisites) 1. [Building](#4-building) -1. [Packaging](#5-packaging) -1. [Installation](#6-installation) -1. [Running](#7-running) -1. [Testing](#8-testing) -1. [Releasing](#9-releasing) -1. [CI/CD](#10-cicd) -1. [Coverage --replace](#11-coverage---replace) -1. [Contributing](#12-contributing) +1. [Installation](#5-installation) +1. [Running](#6-running) +1. [Testing](#7-testing) +1. [Releasing](#8-releasing) +1. [CI/CD](#9-cicd) +1. [Coverage](#10-coverage) +1. [Contributing](#11-contributing) # 1. Introduction @@ -63,42 +62,49 @@ For user manual, system design and architecture, please refer to [TDengine Docum # 3. Prerequisites -## 3.1 Install dependencies tools +## 3.1 On Linux -### The required package for linux +### For Ubuntu 18.04 or Later ```bash -apt-get install -y llvm gcc make cmake libssl-dev pkg-config perl g++ lzma curl locales psmisc sudo tree libgeos-dev libgflags2.2 libgflags-dev libgoogle-glog-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g build-essential valgrind rsync vim libjemalloc-dev openssh-server screen sshpass net-tools dirmngr gnupg apt-transport-https ca-certificates software-properties-common r-base iputils-ping -``` - -### The required package for windows - -to be updated... - -### The required package for macOS ``` + +### For CentOS 7 + +```bash + +``` + +### For Fedora or Rocky Linux + +```bash + +``` + +## 3.2 On macOS + +Please intall the dependencies with [brew](https://brew.sh/). + +```bash brew install argp-standalone gflags pkgconfig ``` -## 3.2 Get the source codes +## 3.3 On Windows -First of all, you may clone the source codes from github: +Work in Progress. + +## 3.4 Clone the repo + +Clone the repository to the target machine: ```bash git clone https://github.com/taosdata/TDengine.git cd TDengine ``` -You can modify the file ~/.gitconfig to use ssh protocol instead of https for better download speed. You will need to upload ssh public key to GitHub first. Please refer to GitHub official documentation for detail. - -``` -[url "git@github.com:"] - insteadOf = https://github.com/ -``` - -## 3.3 Special Note -[JDBC Connector](https://github.com/taosdata/taos-connector-jdbc)， [Go Connector](https://github.com/taosdata/driver-go)，[Python Connector](https://github.com/taosdata/taos-connector-python)，[Node.js Connector](https://github.com/taosdata/taos-connector-node)，[C# Connector](https://github.com/taosdata/taos-connector-dotnet) ，[Rust Connector](https://github.com/taosdata/taos-connector-rust) and [Grafana plugin](https://github.com/taosdata/grafanaplugin) has been moved to standalone repository. +> [!NOTE] +> TDengine Connectors can be found in following repositories: [JDBC Connector](https://github.com/taosdata/taos-connector-jdbc), [Go Connector](https://github.com/taosdata/driver-go), [Python Connector](https://github.com/taosdata/taos-connector-python), [Node.js Connector](https://github.com/taosdata/taos-connector-node), [C# Connector](https://github.com/taosdata/taos-connector-dotnet), [Rust Connector](https://github.com/taosdata/taos-connector-rust). # 4. Building @@ -110,7 +116,7 @@ TDengine provide a few useful tools such as taosBenchmark (was named taosdemo) a To build TDengine, use [CMake](https://cmake.org/) 3.13.0 or higher versions in the project directory. -## 4.1 Linux platform building +## 4.1 Build on Linux You can run the bash script `build.sh` to build both TDengine and taosTools including taosBenchmark and taosdump as below: @@ -129,7 +135,7 @@ make You can use Jemalloc as memory allocator instead of glibc: -``` +```bash apt install autoconf cmake .. -DJEMALLOC_ENABLED=true ``` @@ -143,7 +149,16 @@ aarch64: cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` -## 4.2 Windows platform building +## 4.2 Build on macOS + +Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. + +```shell +mkdir debug && cd debug +cmake .. && cmake --build . +``` + +## 4.3 Build on Windows If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". Please specify "amd64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. @@ -175,22 +190,9 @@ cmake .. -G "NMake Makefiles" nmake ``` -## 4.3 MacOS platform building +# 5. Installation -Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. - -```shell -mkdir debug && cd debug -cmake .. && cmake --build . -``` - -# 5. Packaging - -To be updated... - -# 6. Installation - -## 6.1 Linux platform installation +## 5.1 Install on Linux After building successfully, TDengine can be installed by @@ -202,15 +204,7 @@ Users can find more information about directories installed on the system in the Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/deploy-from-package/) for it. -## 6.2 Windows platform installation - -After building successfully, TDengine can be installed by: - -```cmd -nmake install -``` - -## 6.3 MacOS platform installation +## 5.2 Install on macOS After building successfully, TDengine can be installed by: @@ -218,9 +212,17 @@ After building successfully, TDengine can be installed by: sudo make install ``` -# 7. Running +## 5.3 Install on Windows -## 7.1 Run TDengine on linux +After building successfully, TDengine can be installed by: + +```cmd +nmake install +``` + +# 6. Running + +## 6.1 Run TDengine on Linux To start the service after installation on linux, in a terminal, use: @@ -250,9 +252,9 @@ In another terminal, use the TDengine CLI to connect the server: option "-c test/cfg" specifies the system configuration file directory. -## 7.2 Run TDengine on windows +## 6.2 Run TDengine on Windows -To start the service after installation on windows, in a terminal, use: +To start the service after installation on Windows, in a terminal, use: ```bash to be updated @@ -264,7 +266,7 @@ Then users can use the TDengine CLI to connect the TDengine server. In a termina taos ``` -## 7.3 Run TDengine on MacOS +## 6.3 Run TDengine on macOS To start the service after installation on macOS, double-click the /applications/TDengine to start the program, or in a terminal, use: @@ -280,77 +282,34 @@ taos If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. -# 8. Testing +# 7. Testing -## 8.1 Run the TSIM test script +For how to run different types of tests on TDengine, please see [Testing TDengine](./tests/README.md). -TSIM test framework is developed by c++ in the start-up period of TDengine， the test scripts are basic cases, you can run -the script with below command: +# 8. Releasing + +For the complete list of TDengine Releases, please see [Releases](https://github.com/taosdata/TDengine/releases). + +# 9. CI/CD + +Now, Jenkins is mainly used to build CI/CD pipeline for TDengine. To run the tests in the CI/CD pipeline, please run following commands: ```bash -cd /root/TDengine/tests/script -./test.sh -f tsim/db/basic1.sim +cd tests +./run_all_ci_cases.sh -b main # on main branch ``` -## 8.2 Run the Python test script +TDengine build check workflow can be found in this [Github Action](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml). -Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only -work for TDengine enterprise version, you can run the script with below command: +# 10. Coverage + +Latest TDengine test coverage report can be found on [coveralls.io](https://coveralls.io/github/taosdata/TDengine). To create the test coverage report (in HTML format) locally, please run following commands: ```bash -cd /root/TDengine/tests/system-test -python3 ./test.py -f 2-query/floor.py -``` - -## 8.3 Run unittest - -Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run -the script with below command: - -```bash -cd /root/TDengine/tests/unit-test/ -bash test.sh -``` - -## 8.4 Smoke Testing - -Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine, you can run the -script with below command: - -```bash -cd /root/TDengine/packaging/smokeTest -./test_smoking_selfhost.sh -``` - -## Windows platform testing - - to be updated... - -## MacOS platform testing - - to be updated... - -# 9. Releasing - -You can access the [Release](https://github.com/taosdata/TDengine/releases) for TDengine release version list. - -# 10. CI/CD -You can run ci script locally with below commands: - -```bash -cd /root/TDengine/tests -./run_all_ci_cases.sh -b main -``` - -# 11. Coverage --replace -You can see coverage result in https://coveralls.io/github/taosdata/TDengine -You can also run coverage script locally. - -```bash -cd /root/TDengine/tests +cd tests ./run_local_coverage.sh ``` -# 12. Contributing +# 11. Contributing -Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to the project. +Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to TDengine. diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000..55e6688a8f --- /dev/null +++ b/tests/README.md @@ -0,0 +1,43 @@ +# Testing TDengine + +## Unit Test + +Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run +the script with below command: + +```bash +cd tests/unit-test/ +bash test.sh +``` + +## System Test + +Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only +work for TDengine enterprise version, you can run the script with below command: + +```bash +cd tests/system-test +python3 ./test.py -f 2-query/floor.py +``` + +## Smoke Test + +Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine, you can run the +script with commands below: + +```bash +cd /root/TDengine/packaging/smokeTest +./test_smoking_selfhost.sh +``` + +## Legacy Test + +In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++. To run the legacy test cases, please execute the following commands: + +```bash +cd tests/script +./test.sh -f tsim/db/basic1.sim +``` + + + From a100d96ef3395ab7b890394f3b9eac66f7303ace Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Wed, 8 Jan 2025 19:44:56 +0800 Subject: [PATCH 13/87] docs: collaps different platforms --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 5fadbd22d8..91bcd40f55 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,10 @@ To build TDengine, use [CMake](https://cmake.org/) 3.13.0 or higher versions in ## 4.1 Build on Linux +
+ +Detailed steps to build on Linux + You can run the bash script `build.sh` to build both TDengine and taosTools including taosBenchmark and taosdump as below: ```bash @@ -149,8 +153,14 @@ aarch64: cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` +
+ ## 4.2 Build on macOS +
+ +Detailed steps to build on macOS + Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. ```shell @@ -158,8 +168,14 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` +
+ ## 4.3 Build on Windows +
+ +Detailed steps to build on Windows + If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". Please specify "amd64" for 64 bits Windows or specify "x86" for 32 bits Windows when you execute vcvarsall.bat. @@ -189,6 +205,7 @@ mkdir debug && cd debug cmake .. -G "NMake Makefiles" nmake ``` +
# 5. Installation From 40a89c04bf088ababb8b1f48085c9104acb7111d Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Thu, 9 Jan 2025 14:23:00 +0800 Subject: [PATCH 14/87] add note for sql using in parameter binding insertion --- docs/en/07-develop/05-stmt.md | 10 ++++++++++ docs/zh/07-develop/05-stmt.md | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/docs/en/07-develop/05-stmt.md b/docs/en/07-develop/05-stmt.md index 11b055bcf9..c62918fad7 100644 --- a/docs/en/07-develop/05-stmt.md +++ b/docs/en/07-develop/05-stmt.md @@ -15,6 +15,16 @@ When inserting data using parameter binding, it can avoid the resource consumpti **Tips: It is recommended to use parameter binding for data insertion** + :::note + We only recommend using the following two forms of SQL for parameter binding data insertion: + + ```sql + 1. INSERT INTO meters (tbname, ts, current, voltage, phase, location, group_id) VALUES(?, ?, ?, ?, ?, ?, ?) + 2. INSERT INTO ? USING meters TAGS (?, ?) VALUES (?, ?, ?, ?) + ``` + + ::: + Next, we continue to use smart meters as an example to demonstrate the efficient writing functionality of parameter binding with various language connectors: 1. Prepare a parameterized SQL insert statement for inserting data into the supertable `meters`. This statement allows dynamically specifying subtable names, tags, and column values. diff --git a/docs/zh/07-develop/05-stmt.md b/docs/zh/07-develop/05-stmt.md index 1917a86e74..f30969cb5b 100644 --- a/docs/zh/07-develop/05-stmt.md +++ b/docs/zh/07-develop/05-stmt.md @@ -15,6 +15,16 @@ import TabItem from "@theme/TabItem"; **Tips: 数据写入推荐使用参数绑定方式** + :::note + 我们只推荐使用下面两种形式的 SQL 进行参数绑定写入： + + ```sql + 1. INSERT INTO meters (tbname, ts, current, voltage, phase, location, group_id) VALUES(?, ?, ?, ?, ?, ?, ?) + 2. INSERT INTO ? USING meters TAGS (?, ?) VALUES (?, ?, ?, ?) + ``` + + ::: + 下面我们继续以智能电表为例，展示各语言连接器使用参数绑定高效写入的功能： 1. 准备一个参数化的 SQL 插入语句，用于向超级表 `meters` 中插入数据。这个语句允许动态地指定子表名、标签和列值。 2. 循环生成多个子表及其对应的数据行。对于每个子表： From 7c1363f34c22f8638118ad095764968a09faf0a4 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 9 Jan 2025 17:30:04 +0800 Subject: [PATCH 15/87] update README and test/README by charles --- README.md | 36 +++++++++++++++++++++++++----------- tests/README.md | 21 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 91bcd40f55..953bb7c6b1 100644 --- a/README.md +++ b/README.md @@ -67,19 +67,32 @@ For user manual, system design and architecture, please refer to [TDengine Docum ### For Ubuntu 18.04 or Later ```bash - +sudo apt-get udpate +sudo apt-get install -y gcc cmake build-essential git libjansson-dev libsnappy-dev liblzma-dev zlib1g-dev pkg-config ``` -### For CentOS 7 +### For CentOS 8 ```bash - +sudo yum update +yum install -y epel-release gcc gcc-c++ make cmake git perl dnf-plugins-core +yum config-manager --set-enabled powertools +yum install -y zlib-static xz-devel snappy-devel jansson-devel pkgconfig ``` ### For Fedora or Rocky Linux ```bash +sudo dnf install -y gcc gcc-c++ make cmake git perl +sudo dnf install -y zlib-devel xz-devel snappy-devel jansson-devel pkgconfig +``` +To build the [taosTools](https://github.com/taosdata/taos-tools) on Fedora or Rocky, the following packages need to be installed. + +```bash +sudo dnf install -y dnf-plugins-core +sudo dnf config-manager --set-enabled powertools +sudo dnf install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static ``` ## 3.2 On macOS @@ -110,7 +123,7 @@ cd TDengine At the moment, TDengine server supports running on Linux/Windows/MacOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service. TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. Right now we don't support build with cross-compiling environment. -You can choose to install through source code, [container](https://docs.tdengine.com/get-started/docker/), [installation package](https://docs.tdengine.com/get-started/package/) or [Kubernetes](https://docs.tdengine.com/deployment/k8s/). This quick guide only applies to installing from source. +You can choose to install through source code, [container](https://docs.tdengine.com/get-started/docker/), [installation package](https://docs.tdengine.com/get-started/package/) or [Kubernetes](https://docs.tdengine.com/deployment/k8s/). This quick guide only applies to install from source. TDengine provide a few useful tools such as taosBenchmark (was named taosdemo) and taosdump. They were part of TDengine. By default, TDengine compiling does not include taosTools. You can use `cmake .. -DBUILD_TOOLS=true` to make them be compiled with TDengine. @@ -140,7 +153,6 @@ make You can use Jemalloc as memory allocator instead of glibc: ```bash -apt install autoconf cmake .. -DJEMALLOC_ENABLED=true ``` @@ -271,18 +283,20 @@ option "-c test/cfg" specifies the system configuration file directory. ## 6.2 Run TDengine on Windows -To start the service after installation on Windows, in a terminal, use: +You can start TDengine server on windows platform with below commands: -```bash -to be updated +```cmd +.\build\bin\taosd.exe -c test\cfg ``` -Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: +In another terminal, use the TDengine CLI to connect the server: -```bash -taos +```cmd +.\build\bin\taos.exe -c test\cfg ``` +option "-c test/cfg" specifies the system configuration file directory. + ## 6.3 Run TDengine on macOS To start the service after installation on macOS, double-click the /applications/TDengine to start the program, or in a terminal, use: diff --git a/tests/README.md b/tests/README.md index 55e6688a8f..bd5696fc83 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,5 +1,26 @@ # Testing TDengine +## Install the required tools + +Install Python3. + +```bash +apt install python3 +apt install python3-pip +``` + +Install the dependent Python components. + +```bash +pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog +``` + +Install the Python connector for TDengine. + +```bash +pip3 install taospy taos-ws-py +``` + ## Unit Test Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run From c7b436fb62e93d1ca3af09f0833e5543b9f047b9 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 9 Jan 2025 18:18:03 +0800 Subject: [PATCH 16/87] test:alter coverage describe --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 953bb7c6b1..00306aaf2b 100644 --- a/README.md +++ b/README.md @@ -338,8 +338,11 @@ Latest TDengine test coverage report can be found on [coveralls.io](https://cove ```bash cd tests -./run_local_coverage.sh +bash setup-lcov.sh -v 1.16 && ./run_local_coverage.sh -b main -c task +# on main branch and run cases in longtimeruning_cases.task +# for more infomation about options please refer to ./run_local_coverage.sh -h ``` +Please note that the -b and -i options will recompile TDengine with the -DCOVER=true option, which may take a amount of time. # 11. Contributing From e9945c096940c6a51a445732f282e76dd9d4532e Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Fri, 10 Jan 2025 11:46:29 +0800 Subject: [PATCH 17/87] udpate testing section to add -e parameter for unit test and two packages for centos 8 building section by charles --- README.md | 5 +++-- tests/README.md | 2 +- tests/unit-test/test.sh | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 00306aaf2b..8824365fc6 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ sudo apt-get install -y gcc cmake build-essential git libjansson-dev libsnappy-d sudo yum update yum install -y epel-release gcc gcc-c++ make cmake git perl dnf-plugins-core yum config-manager --set-enabled powertools -yum install -y zlib-static xz-devel snappy-devel jansson-devel pkgconfig +yum install -y zlib-static xz-devel snappy-devel jansson-devel pkgconfig libatomic-static libstdc++-static ``` ### For Fedora or Rocky Linux @@ -342,7 +342,8 @@ bash setup-lcov.sh -v 1.16 && ./run_local_coverage.sh -b main -c task # on main branch and run cases in longtimeruning_cases.task # for more infomation about options please refer to ./run_local_coverage.sh -h ``` -Please note that the -b and -i options will recompile TDengine with the -DCOVER=true option, which may take a amount of time. +> [!NOTE] +> Please note that the -b and -i options will recompile TDengine with the -DCOVER=true option, which may take a amount of time. # 11. Contributing diff --git a/tests/README.md b/tests/README.md index bd5696fc83..9d7f98b564 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,7 +27,7 @@ Unit test script is the smallest testable part and developed for some function, the script with below command: ```bash -cd tests/unit-test/ +cd tests/unit-test/ -e 0 bash test.sh ``` diff --git a/tests/unit-test/test.sh b/tests/unit-test/test.sh index 21461bc6a5..46fc0aedb3 100755 --- a/tests/unit-test/test.sh +++ b/tests/unit-test/test.sh @@ -7,10 +7,10 @@ function usage() { } ent=1 -while getopts "eh" opt; do +while getopts "e:h" opt; do case $opt in e) - ent=1 + ent="$OPTARG" ;; h) usage From 53b3ff8f6cc5fdede941ec5cfb88df43d9816516 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Fri, 10 Jan 2025 13:38:12 +0800 Subject: [PATCH 18/87] update readme --- README.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8824365fc6..33874cf05d 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,8 @@ For user manual, system design and architecture, please refer to [TDengine Docum ```bash sudo apt-get udpate -sudo apt-get install -y gcc cmake build-essential git libjansson-dev libsnappy-dev liblzma-dev zlib1g-dev pkg-config +sudo apt-get install -y gcc cmake build-essential git libjansson-dev \ + libsnappy-dev liblzma-dev zlib1g-dev pkg-config ``` ### For CentOS 8 @@ -92,7 +93,8 @@ To build the [taosTools](https://github.com/taosdata/taos-tools) on Fedora or Ro ```bash sudo dnf install -y dnf-plugins-core sudo dnf config-manager --set-enabled powertools -sudo dnf install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static +sudo dnf install -y zlib-devel zlib-static xz-devel snappy-devel jansson \ + jansson-devel pkgconfig libatomic libatomic-static libstdc++-static ``` ## 3.2 On macOS @@ -144,8 +146,7 @@ You can run the bash script `build.sh` to build both TDengine and taosTools incl It equals to execute following commands: ```bash -mkdir debug -cd debug +mkdir debug && cd debug cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true make ``` @@ -156,10 +157,8 @@ You can use Jemalloc as memory allocator instead of glibc: cmake .. -DJEMALLOC_ENABLED=true ``` -TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 platform. -You can also specify CPUTYPE option like aarch64 too if the detection result is not correct: - -aarch64: +TDengine build script can auto-detect the host machine's architecture on x86, x86-64, arm64 platform. +You can also specify architecture manually by CPUTYPE option: ```bash cmake .. -DCPUTYPE=aarch64 && cmake --build . @@ -223,15 +222,13 @@ nmake ## 5.1 Install on Linux -After building successfully, TDengine can be installed by +After building successfully, TDengine can be installed by: ```bash sudo make install ``` -Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section. - -Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/deploy-from-package/) for it. +Installing from source code will also configure service management for TDengine. Users can also choose to [install from packages](https://docs.tdengine.com/get-started/deploy-from-package/) for it. ## 5.2 Install on macOS @@ -279,11 +276,11 @@ In another terminal, use the TDengine CLI to connect the server: ./build/bin/taos -c test/cfg ``` -option "-c test/cfg" specifies the system configuration file directory. +Option `-c test/cfg` specifies the system configuration file directory. ## 6.2 Run TDengine on Windows -You can start TDengine server on windows platform with below commands: +You can start TDengine server on Windows platform with below commands: ```cmd .\build\bin\taosd.exe -c test\cfg From 8505d9b7a9fe0481c76d55e45406a23221af2cb7 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Sat, 11 Jan 2025 15:49:41 +0800 Subject: [PATCH 19/87] feat: add trivy --- packaging/setup_env.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packaging/setup_env.sh b/packaging/setup_env.sh index e1a7a26579..2cf1e9e952 100644 --- a/packaging/setup_env.sh +++ b/packaging/setup_env.sh @@ -174,6 +174,7 @@ help() { echo " config_qemu_guest_agent - Configure QEMU guest agent" echo " deploy_docker - Deploy Docker" echo " deploy_docker_compose - Deploy Docker Compose" + echo " install_trivy - Install Trivy" echo " clone_enterprise - Clone the enterprise repository" echo " clone_community - Clone the community repository" echo " clone_taosx - Clone TaosX repository" @@ -1731,6 +1732,32 @@ deploy_docker_compose() { fi } +# Instal trivy +install_trivy() { + # Install jq + install_package jq + # Get latest version + LATEST_VERSION=$(curl -s https://api.github.com/repos/aquasecurity/trivy/releases/latest | jq -r .tag_name) + # Download + if [ -f /etc/debian_version ]; then + wget https://github.com/aquasecurity/trivy/releases/download/"${LATEST_VERSION}"/trivy_"${LATEST_VERSION#v}"_Linux-64bit.deb + # Install + dpkg -i trivy_"${LATEST_VERSION#v}"_Linux-64bit.deb + + elif [ -f /etc/redhat-release ]; then + wget https://github.com/aquasecurity/trivy/releases/download/"${LATEST_VERSION}"/trivy_"${LATEST_VERSION#v}"_Linux-64bit.rpm + # Install + rpm -ivh trivy_"${LATEST_VERSION#v}"_Linux-64bit.rpm + else + echo "Unsupported Linux distribution." + exit 1 + fi + # Check + trivy --version + check_status "Failed to install Trivy" "Trivy installed successfully." $? + rm -rf trivy_"${LATEST_VERSION#v}"_Linux-64bit.deb trivy_"${LATEST_VERSION#v}"_Linux-64bit.rpm +} + # Reconfigure cloud-init reconfig_cloud_init() { echo "Reconfiguring cloud-init..." @@ -2004,6 +2031,7 @@ deploy_dev() { install_nginx deploy_docker deploy_docker_compose + install_trivy check_status "Failed to deploy some tools" "Deploy all tools successfully" $? } @@ -2159,6 +2187,9 @@ main() { deploy_docker_compose) deploy_docker_compose ;; + install_trivy) + install_trivy + ;; clone_enterprise) clone_enterprise ;; From 2b10e8a9b78bf71fd439345fc61b248a7fd0f279 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Sat, 11 Jan 2025 16:06:49 +0800 Subject: [PATCH 20/87] feat: add trivy --- packaging/setup_env.sh | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/packaging/setup_env.sh b/packaging/setup_env.sh index 2cf1e9e952..9a624bfa9e 100644 --- a/packaging/setup_env.sh +++ b/packaging/setup_env.sh @@ -317,6 +317,17 @@ add_config_if_not_exist() { grep -qF -- "$config" "$file" || echo "$config" >> "$file" } +# Function to check if a tool is installed +check_installed() { + local command_name="$1" + if command -v "$command_name" >/dev/null 2>&1; then + echo "$command_name is already installed. Skipping installation." + return 0 + else + echo "$command_name is not installed." + return 1 + fi +} # General error handling function check_status() { local message_on_failure="$1" @@ -585,9 +596,12 @@ centos_skip_check() { # Deploy cmake deploy_cmake() { # Check if cmake is installed - if command -v cmake >/dev/null 2>&1; then - echo "Cmake is already installed. Skipping installation." - cmake --version + # if command -v cmake >/dev/null 2>&1; then + # echo "Cmake is already installed. Skipping installation." + # cmake --version + # return + # fi + if check_installed "cmake"; then return fi install_package "cmake3" @@ -1059,11 +1073,13 @@ deploy_go() { GOPATH_DIR="/root/go" # Check if Go is installed - if command -v go >/dev/null 2>&1; then - echo "Go is already installed. Skipping installation." + # if command -v go >/dev/null 2>&1; then + # echo "Go is already installed. Skipping installation." + # return + # fi + if check_installed "gp"; then return fi - # Fetch the latest version number of Go GO_LATEST_DATA=$(curl --retry 10 --retry-delay 5 --retry-max-time 120 -s https://golang.google.cn/VERSION?m=text) GO_LATEST_VERSION=$(echo "$GO_LATEST_DATA" | grep -oP 'go[0-9]+\.[0-9]+\.[0-9]+') @@ -1734,6 +1750,16 @@ deploy_docker_compose() { # Instal trivy install_trivy() { + echo -e "${YELLOW}Installing Trivy...${NO_COLOR}" + # Check if Trivy is already installed + # if command -v trivy >/dev/null 2>&1; then + # echo "Trivy is already installed. Skipping installation." + # trivy --version + # return + # fi + if check_installed "trivy"; then + return + fi # Install jq install_package jq # Get latest version From 1f656c0ef0624d6fdd18c16bf7fa65fd17dc4f81 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Sat, 11 Jan 2025 16:08:35 +0800 Subject: [PATCH 21/87] fix: typo --- packaging/setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/setup_env.sh b/packaging/setup_env.sh index 9a624bfa9e..32451072ab 100644 --- a/packaging/setup_env.sh +++ b/packaging/setup_env.sh @@ -1077,7 +1077,7 @@ deploy_go() { # echo "Go is already installed. Skipping installation." # return # fi - if check_installed "gp"; then + if check_installed "go"; then return fi # Fetch the latest version number of Go From dc4feb8690496fede5c2b5efc4917035b70827ca Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Mon, 13 Jan 2025 15:01:58 +0800 Subject: [PATCH 22/87] improve stmt doc, fix jdbc demo resultset bug --- docs/en/07-develop/05-stmt.md | 7 +++++-- .../src/main/java/com/taosdata/example/JdbcBasicDemo.java | 2 -- .../src/main/java/com/taosdata/example/JdbcDemo.java | 5 +++-- docs/zh/07-develop/05-stmt.md | 7 +++++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/en/07-develop/05-stmt.md b/docs/en/07-develop/05-stmt.md index c62918fad7..16fe156cc3 100644 --- a/docs/en/07-develop/05-stmt.md +++ b/docs/en/07-develop/05-stmt.md @@ -19,8 +19,11 @@ When inserting data using parameter binding, it can avoid the resource consumpti We only recommend using the following two forms of SQL for parameter binding data insertion: ```sql - 1. INSERT INTO meters (tbname, ts, current, voltage, phase, location, group_id) VALUES(?, ?, ?, ?, ?, ?, ?) - 2. INSERT INTO ? USING meters TAGS (?, ?) VALUES (?, ?, ?, ?) + a. Subtables already exists: + 1. INSERT INTO meters (tbname, ts, current, voltage, phase) VALUES(?, ?, ?, ?, ?) + b. Automatic table creation on insert: + 1. INSERT INTO meters (tbname, ts, current, voltage, phase, location, group_id) VALUES(?, ?, ?, ?, ?, ?, ?) + 2. INSERT INTO ? USING meters TAGS (?, ?) VALUES (?, ?, ?, ?) ``` ::: diff --git a/docs/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcBasicDemo.java b/docs/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcBasicDemo.java index 0de386447c..0a63504b91 100644 --- a/docs/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcBasicDemo.java +++ b/docs/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcBasicDemo.java @@ -1,6 +1,4 @@ package com.taosdata.example; - -import com.alibaba.fastjson.JSON; import com.taosdata.jdbc.AbstractStatement; import java.sql.*; diff --git a/docs/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java b/docs/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java index ec4adf8db9..7fba500c49 100644 --- a/docs/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java +++ b/docs/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java @@ -104,8 +104,9 @@ public class JdbcDemo { private void executeQuery(String sql) { long start = System.currentTimeMillis(); - try (Statement statement = connection.createStatement()) { - ResultSet resultSet = statement.executeQuery(sql); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + long end = System.currentTimeMillis(); printSql(sql, true, (end - start)); Util.printResult(resultSet); diff --git a/docs/zh/07-develop/05-stmt.md b/docs/zh/07-develop/05-stmt.md index f30969cb5b..5f218689be 100644 --- a/docs/zh/07-develop/05-stmt.md +++ b/docs/zh/07-develop/05-stmt.md @@ -19,8 +19,11 @@ import TabItem from "@theme/TabItem"; 我们只推荐使用下面两种形式的 SQL 进行参数绑定写入： ```sql - 1. INSERT INTO meters (tbname, ts, current, voltage, phase, location, group_id) VALUES(?, ?, ?, ?, ?, ?, ?) - 2. INSERT INTO ? USING meters TAGS (?, ?) VALUES (?, ?, ?, ?) + 一、确定子表存在： + 1. INSERT INTO meters (tbname, ts, current, voltage, phase) VALUES(?, ?, ?, ?, ?) + 二、自动建表： + 1. INSERT INTO meters (tbname, ts, current, voltage, phase, location, group_id) VALUES(?, ?, ?, ?, ?, ?, ?) + 2. INSERT INTO ? USING meters TAGS (?, ?) VALUES (?, ?, ?, ?) ``` ::: From 847ce16506490acfc8dce326ea05bfbfe2e7c69c Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Mon, 13 Jan 2025 10:31:46 +0800 Subject: [PATCH 23/87] fix stmt/stmt2 unit test --- source/client/test/stmt2Test.cpp | 298 ++++++++++++++++++------------- source/client/test/stmtTest.cpp | 72 +++++--- 2 files changed, 213 insertions(+), 157 deletions(-) diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index 1ce844e98f..3e21721c47 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -112,6 +112,11 @@ void do_query(TAOS* taos, const char* sql) { TAOS_RES* result = taos_query(taos, sql); // printf("sql: %s\n", sql); int code = taos_errno(result); + while (code == TSDB_CODE_MND_DB_IN_CREATING || code == TSDB_CODE_MND_DB_IN_DROPPING) { + taosMsleep(2000); + result = taos_query(taos, sql); + code = taos_errno(result); + } if (code != TSDB_CODE_SUCCESS) { printf("query failen sql : %s\n errstr : %s\n", sql, taos_errstr(result)); ASSERT_EQ(taos_errno(result), TSDB_CODE_SUCCESS); @@ -122,9 +127,9 @@ void do_query(TAOS* taos, const char* sql) { void do_stmt(TAOS* taos, TAOS_STMT2_OPTION* option, const char* sql, int CTB_NUMS, int ROW_NUMS, int CYC_NUMS, bool hastags, bool createTable) { printf("test sql : %s\n", sql); - do_query(taos, "drop database if exists testdb1"); - do_query(taos, "create database IF NOT EXISTS testdb1"); - do_query(taos, "create stable testdb1.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); + do_query(taos, "drop database if exists stmt2_testdb_1"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_1"); + do_query(taos, "create stable stmt2_testdb_1.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); TAOS_STMT2* stmt = taos_stmt2_init(taos, option); ASSERT_NE(stmt, nullptr); @@ -139,7 +144,7 @@ void do_stmt(TAOS* taos, TAOS_STMT2_OPTION* option, const char* sql, int CTB_NUM sprintf(tbs[i], "ctb_%d", i); if (createTable) { char* tmp = (char*)taosMemoryMalloc(sizeof(char) * 100); - sprintf(tmp, "create table testdb1.%s using testdb1.stb tags(0, 'after')", tbs[i]); + sprintf(tmp, "create table stmt2_testdb_1.%s using stmt2_testdb_1.stb tags(0, 'after')", tbs[i]); do_query(taos, tmp); } } @@ -235,14 +240,15 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - do_query(taos, "drop database if exists testdb2"); - do_query(taos, "create database IF NOT EXISTS testdb2 PRECISION 'ns'"); + do_query(taos, "drop database if exists stmt2_testdb_2"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_2 PRECISION 'ns'"); do_query(taos, - "create stable testdb2.stb (ts timestamp, b binary(10)) tags(t1 " + "create stable stmt2_testdb_2.stb (ts timestamp, b binary(10)) tags(t1 " "int, t2 binary(10))"); do_query( taos, - "create stable if not exists testdb2.all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, " + "create stable if not exists stmt2_testdb_2.all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 " + "bigint, " "v6 tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 double, v12 " "binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20))tags(tts timestamp, tv1 bool, tv2 tinyint, tv3 " "smallint, tv4 int, tv5 bigint, tv6 tinyint unsigned, tv7 smallint unsigned, tv8 int unsigned, tv9 bigint " @@ -251,7 +257,7 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { // case 1 : test super table { - const char* sql = "insert into testdb2.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, @@ -263,7 +269,7 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { { // case 2 : no tag - const char* sql = "insert into testdb2.stb(ts,b,tbname) values(?,?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(ts,b,tbname) values(?,?,?)"; TAOS_FIELD_ALL expectedFields[3] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; @@ -273,7 +279,7 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { // case 3 : random order { - const char* sql = "insert into testdb2.stb(tbname,ts,t2,b,t1) values(?,?,?,?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(tbname,ts,t2,b,t1) values(?,?,?,?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, @@ -285,7 +291,7 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { // case 4 : random order 2 { - const char* sql = "insert into testdb2.stb(ts,tbname,b,t2,t1) values(?,?,?,?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(ts,tbname,b,t2,t1) values(?,?,?,?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, {"b", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, @@ -297,7 +303,7 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { // case 5 : 'db'.'stb' { - const char* sql = "insert into 'testdb2'.'stb'(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; + const char* sql = "insert into 'stmt2_testdb_2'.'stb'(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, @@ -309,7 +315,7 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { // case 6 : use db { - do_query(taos, "use testdb2"); + do_query(taos, "use stmt2_testdb_2"); const char* sql = "insert into stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, @@ -322,7 +328,7 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { // case 7 : less param { - const char* sql = "insert into testdb2.stb(ts,tbname) values(?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(ts,tbname) values(?,?)"; TAOS_FIELD_ALL expectedFields[2] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, {"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}}; printf("case 7 : %s\n", sql); @@ -378,67 +384,68 @@ TEST(stmt2Case, insert_stb_get_fields_Test) { // case 1 : add in main TD-33353 { - const char* sql = "insert into testdb2.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)"; + const char* sql = "insert into stmt2_testdb_2.stb(t1,t2,ts,b,tbname) values(1,?,?,'abc',?)"; printf("case 1dif : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); } // case 2 : no pk { - const char* sql = "insert into testdb2.stb(b,tbname) values(?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(b,tbname) values(?,?)"; printf("case 2 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); } // case 3 : no tbname and tag(not support bind) { - const char* sql = "insert into testdb2.stb(ts,b) values(?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(ts,b) values(?,?)"; printf("case 3 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); } // case 4 : no col and tag(not support bind) { - const char* sql = "insert into testdb2.stb(tbname) values(?)"; + const char* sql = "insert into stmt2_testdb_2.stb(tbname) values(?)"; printf("case 4 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); } // case 5 : no field name { - const char* sql = "insert into testdb2.stb(?,?,?,?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(?,?,?,?,?)"; printf("case 5 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); } // case 6 : test super table not exist { - const char* sql = "insert into testdb2.nstb(?,?,?,?,?)"; + const char* sql = "insert into stmt2_testdb_2.nstb(?,?,?,?,?)"; printf("case 6 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); } // case 7 : no col { - const char* sql = "insert into testdb2.stb(t1,t2,tbname) values(?,?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(t1,t2,tbname) values(?,?,?)"; printf("case 7 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_TSC_INVALID_OPERATION); } // case 8 : wrong para nums { - const char* sql = "insert into testdb2.stb(ts,b,tbname) values(?,?,?,?,?)"; + const char* sql = "insert into stmt2_testdb_2.stb(ts,b,tbname) values(?,?,?,?,?)"; printf("case 8 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); } // case 9 : wrong simbol { - const char* sql = "insert into testdb2.stb(t1,t2,ts,b,tbname) values(*,*,*,*,*)"; + const char* sql = "insert into stmt2_testdb_2.stb(t1,t2,ts,b,tbname) values(*,*,*,*,*)"; printf("case 9 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); } + do_query(taos, "drop database if exists stmt2_testdb_2"); taos_close(taos); } @@ -446,24 +453,25 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - do_query(taos, "drop database if exists testdb3"); - do_query(taos, "create database IF NOT EXISTS testdb3 PRECISION 'ns'"); + do_query(taos, "drop database if exists stmt2_testdb_3"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_3 PRECISION 'ns'"); do_query(taos, - "create stable testdb3.stb (ts timestamp, b binary(10)) tags(t1 " + "create stable stmt2_testdb_3.stb (ts timestamp, b binary(10)) tags(t1 " "int, t2 binary(10))"); do_query( taos, - "create stable if not exists testdb3.all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, " + "create stable if not exists stmt2_testdb_3.all_stb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 " + "bigint, " "v6 tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 double, v12 " "binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20))tags(tts timestamp, tv1 bool, tv2 tinyint, tv3 " "smallint, tv4 int, tv5 bigint, tv6 tinyint unsigned, tv7 smallint unsigned, tv8 int unsigned, tv9 bigint " "unsigned, tv10 float, tv11 double, tv12 binary(20), tv13 varbinary(20), tv14 geometry(100), tv15 nchar(20));"); - do_query(taos, "CREATE TABLE testdb3.t0 USING testdb3.stb (t1,t2) TAGS (7,'Cali');"); + do_query(taos, "CREATE TABLE stmt2_testdb_3.t0 USING stmt2_testdb_3.stb (t1,t2) TAGS (7,'Cali');"); printf("support case \n"); // case 1 : test child table already exist { - const char* sql = "INSERT INTO testdb3.t0(ts,b)using testdb3.stb (t1,t2) TAGS(?,?) VALUES (?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_3.t0(ts,b)using stmt2_testdb_3.stb (t1,t2) TAGS(?,?) VALUES (?,?)"; TAOS_FIELD_ALL expectedFields[4] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, @@ -474,7 +482,7 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // case 2 : insert clause { - const char* sql = "INSERT INTO testdb3.? using testdb3.stb (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_3.? using stmt2_testdb_3.stb (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, @@ -486,7 +494,7 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // case 3 : insert child table not exist { - const char* sql = "INSERT INTO testdb3.d1 using testdb3.stb (t1,t2)TAGS(?,?) (ts,b)VALUES(?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_3.d1 using stmt2_testdb_3.stb (t1,t2)TAGS(?,?) (ts,b)VALUES(?,?)"; TAOS_FIELD_ALL expectedFields[4] = {{"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}, @@ -497,7 +505,7 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // case 4 : random order { - const char* sql = "INSERT INTO testdb3.? using testdb3.stb (t2,t1)TAGS(?,?) (b,ts)VALUES(?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_3.? using stmt2_testdb_3.stb (t2,t1)TAGS(?,?) (b,ts)VALUES(?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, @@ -509,7 +517,7 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // case 5 : less para { - const char* sql = "insert into testdb3.? using testdb3.stb (t2)tags(?) (ts)values(?)"; + const char* sql = "insert into stmt2_testdb_3.? using stmt2_testdb_3.stb (t2)tags(?) (ts)values(?)"; TAOS_FIELD_ALL expectedFields[3] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, {"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; @@ -520,7 +528,7 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // case 6 : insert into db.? using db.stb tags(?, ?) values(?,?) // no field name { - const char* sql = "insert into testdb3.? using testdb3.stb tags(?, ?) values(?,?)"; + const char* sql = "insert into stmt2_testdb_3.? using stmt2_testdb_3.stb tags(?, ?) values(?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, @@ -533,7 +541,7 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // case 7 : insert into db.d0 (ts)values(?) // less para { - const char* sql = "insert into testdb3.t0 (ts)values(?)"; + const char* sql = "insert into stmt2_testdb_3.t0 (ts)values(?)"; TAOS_FIELD_ALL expectedFields[1] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 2, 0, 8, TAOS_FIELD_COL}}; printf("case 7 : %s\n", sql); getFieldsSuccess(taos, sql, expectedFields, 1); @@ -541,7 +549,7 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // case 8 : 'db' 'stb' { - const char* sql = "INSERT INTO 'testdb3'.? using 'testdb3'.'stb' (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)"; + const char* sql = "INSERT INTO 'stmt2_testdb_3'.? using 'stmt2_testdb_3'.'stb' (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, {"t2", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_TAG}, @@ -553,7 +561,7 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // case 9 : use db { - do_query(taos, "use testdb3"); + do_query(taos, "use stmt2_testdb_3"); const char* sql = "INSERT INTO ? using stb (t1,t2) TAGS(?,?) (ts,b)VALUES(?,?)"; TAOS_FIELD_ALL expectedFields[5] = {{"tbname", TSDB_DATA_TYPE_BINARY, 0, 0, 271, TAOS_FIELD_TBNAME}, {"t1", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_TAG}, @@ -608,38 +616,40 @@ TEST(stmt2Case, insert_ctb_using_get_fields_Test) { // case 1 : test super table not exist { - const char* sql = "INSERT INTO testdb3.?(ts,b)using testdb3.nstb (t1,t2) TAGS(?,?) VALUES (?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_3.?(ts,b)using stmt2_testdb_3.nstb (t1,t2) TAGS(?,?) VALUES (?,?)"; printf("case 1 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); } // case 2 : no pk { - const char* sql = "INSERT INTO testdb3.?(ts,b)using testdb3.nstb (t1,t2) TAGS(?,?) (n)VALUES (?)"; + const char* sql = "INSERT INTO stmt2_testdb_3.?(ts,b)using stmt2_testdb_3.nstb (t1,t2) TAGS(?,?) (n)VALUES (?)"; printf("case 2 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); } // case 3 : less param and no filed name { - const char* sql = "INSERT INTO testdb3.?(ts,b)using testdb3.stb TAGS(?)VALUES (?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_3.?(ts,b)using stmt2_testdb_3.stb TAGS(?)VALUES (?,?)"; printf("case 3 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); } // case 4 : none para for ctbname { - const char* sql = "INSERT INTO testdb3.d0 using testdb3.stb values(?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_3.d0 using stmt2_testdb_3.stb values(?,?)"; printf("case 4 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_TSC_SQL_SYNTAX_ERROR); } // case 5 : none para for ctbname { - const char* sql = "insert into ! using testdb3.stb tags(?, ?) values(?,?)"; + const char* sql = "insert into ! using stmt2_testdb_3.stb tags(?, ?) values(?,?)"; printf("case 5 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_TSC_SQL_SYNTAX_ERROR); } + + do_query(taos, "drop database if exists stmt2_testdb_3"); taos_close(taos); } @@ -647,19 +657,20 @@ TEST(stmt2Case, insert_ntb_get_fields_Test) { TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - do_query(taos, "drop database if exists testdb4"); - do_query(taos, "create database IF NOT EXISTS testdb4 PRECISION 'ms'"); - do_query(taos, "CREATE TABLE testdb4.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);"); - do_query(taos, - "create table if not exists testdb4.all_ntb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 " - "bigint, v6 tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 " - "double, v12 binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20));"); + do_query(taos, "drop database if exists stmt2_testdb_4"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_4 PRECISION 'ms'"); + do_query(taos, "CREATE TABLE stmt2_testdb_4.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);"); + do_query( + taos, + "create table if not exists stmt2_testdb_4.all_ntb(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 " + "bigint, v6 tinyint unsigned, v7 smallint unsigned, v8 int unsigned, v9 bigint unsigned, v10 float, v11 " + "double, v12 binary(20), v13 varbinary(20), v14 geometry(100), v15 nchar(20));"); printf("support case \n"); // case 1 : test normal table no field name { - const char* sql = "INSERT INTO testdb4.ntb VALUES(?,?,?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_4.ntb VALUES(?,?,?,?)"; TAOS_FIELD_ALL expectedFields[4] = {{"nts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL}, {"nb", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, {"nvc", TSDB_DATA_TYPE_BINARY, 0, 0, 18, TAOS_FIELD_COL}, @@ -670,7 +681,7 @@ TEST(stmt2Case, insert_ntb_get_fields_Test) { // case 2 : test random order { - const char* sql = "INSERT INTO testdb4.ntb (ni,nb,nvc,nts)VALUES(?,?,?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_4.ntb (ni,nb,nvc,nts)VALUES(?,?,?,?)"; TAOS_FIELD_ALL expectedFields[4] = {{"ni", TSDB_DATA_TYPE_INT, 0, 0, 4, TAOS_FIELD_COL}, {"nb", TSDB_DATA_TYPE_BINARY, 0, 0, 12, TAOS_FIELD_COL}, {"nvc", TSDB_DATA_TYPE_BINARY, 0, 0, 18, TAOS_FIELD_COL}, @@ -681,7 +692,7 @@ TEST(stmt2Case, insert_ntb_get_fields_Test) { // case 3 : less param { - const char* sql = "INSERT INTO testdb4.ntb (nts)VALUES(?)"; + const char* sql = "INSERT INTO stmt2_testdb_4.ntb (nts)VALUES(?)"; TAOS_FIELD_ALL expectedFields[1] = {{"nts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL}}; printf("case 3 : %s\n", sql); getFieldsSuccess(taos, sql, expectedFields, 1); @@ -689,7 +700,7 @@ TEST(stmt2Case, insert_ntb_get_fields_Test) { // case 4 : test all types { - const char* sql = "insert into testdb4.all_ntb values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + const char* sql = "insert into stmt2_testdb_4.all_ntb values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; TAOS_FIELD_ALL expectedFields[16] = {{"ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0, 8, TAOS_FIELD_COL}, {"v1", TSDB_DATA_TYPE_BOOL, 0, 0, 1, TAOS_FIELD_COL}, {"v2", TSDB_DATA_TYPE_TINYINT, 0, 0, 1, TAOS_FIELD_COL}, @@ -721,26 +732,29 @@ TEST(stmt2Case, insert_ntb_get_fields_Test) { // case 2 : normal table must have tbnam { - const char* sql = "insert into testdb4.? values(?,?)"; + const char* sql = "insert into stmt2_testdb_4.? values(?,?)"; printf("case 2 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_TABLE_NOT_EXIST); } // case 3 : wrong para nums { - const char* sql = "insert into testdb4.ntb(nts,ni) values(?,?,?,?,?)"; + const char* sql = "insert into stmt2_testdb_4.ntb(nts,ni) values(?,?,?,?,?)"; printf("case 3 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); } + + do_query(taos, "drop database if exists stmt2_testdb_4"); + taos_close(taos); } TEST(stmt2Case, select_get_fields_Test) { TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - do_query(taos, "drop database if exists testdb5"); - do_query(taos, "create database IF NOT EXISTS testdb5 PRECISION 'ns'"); - do_query(taos, "use testdb5"); - do_query(taos, "CREATE TABLE testdb5.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);"); + do_query(taos, "drop database if exists stmt2_testdb_5"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_5 PRECISION 'ns'"); + do_query(taos, "use stmt2_testdb_5"); + do_query(taos, "CREATE TABLE stmt2_testdb_5.ntb(nts timestamp, nb binary(10),nvc varchar(16),ni int);"); { // case 1 : const char* sql = "select * from ntb where ts = ?"; @@ -761,6 +775,8 @@ TEST(stmt2Case, select_get_fields_Test) { printf("case 3 : %s\n", sql); getFieldsError(taos, sql, TSDB_CODE_PAR_SYNTAX_ERROR); } + + do_query(taos, "drop database if exists stmt2_testdb_5"); taos_close(taos); } @@ -797,9 +813,9 @@ TEST(stmt2Case, stmt2_init_prepare_Test) { ASSERT_NE(stmt, nullptr); ASSERT_EQ(((STscStmt2*)stmt)->db, nullptr); - code = taos_stmt2_prepare(stmt, "insert into 'testdb5'.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)", 0); + code = taos_stmt2_prepare(stmt, "insert into 'stmt2_testdb_5'.stb(t1,t2,ts,b,tbname) values(?,?,?,?,?)", 0); ASSERT_NE(stmt, nullptr); - ASSERT_STREQ(((STscStmt2*)stmt)->db, "testdb5"); // add in main TD-33332 + ASSERT_STREQ(((STscStmt2*)stmt)->db, "stmt2_testdb_5"); // add in main TD-33332 taos_stmt2_close(stmt); } @@ -824,22 +840,28 @@ TEST(stmt2Case, stmt2_stb_insert) { ASSERT_NE(taos, nullptr); // normal TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; - { do_stmt(taos, &option, "insert into `testdb1`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, true); } { - do_stmt(taos, &option, "insert into `testdb1`.? using `testdb1`.`stb` tags(?,?) values(?,?)", 3, 3, 3, true, true); + do_stmt(taos, &option, "insert into `stmt2_testdb_1`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, + true); + } + { + do_stmt(taos, &option, "insert into `stmt2_testdb_1`.? using `stmt2_testdb_1`.`stb` tags(?,?) values(?,?)", 3, 3, 3, + true, true); } // async option = {0, true, true, stmtAsyncQueryCb, NULL}; - { do_stmt(taos, &option, "insert into testdb1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, true); } { - do_stmt(taos, &option, "insert into testdb1.? using testdb1.stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3, 3, true, - true); + do_stmt(taos, &option, "insert into stmt2_testdb_1.stb (ts,b,tbname,t1,t2) values(?,?,?,?,?)", 3, 3, 3, true, true); + } + { + do_stmt(taos, &option, "insert into stmt2_testdb_1.? using stmt2_testdb_1.stb (t1,t2)tags(?,?) (ts,b)values(?,?)", + 3, 3, 3, true, true); } // { do_stmt(taos, &option, "insert into db.? values(?,?)", 3, 3, 3, false, true); } // interlace = 0 & use db] - do_query(taos, "use testdb1"); + do_query(taos, "use stmt2_testdb_1"); option = {0, false, false, NULL, NULL}; { do_stmt(taos, &option, "insert into stb (tbname,ts,b) values(?,?,?)", 3, 3, 3, false, true); } { do_stmt(taos, &option, "insert into ? using stb (t1,t2)tags(?,?) (ts,b)values(?,?)", 3, 3, 3, true, true); } @@ -851,6 +873,7 @@ TEST(stmt2Case, stmt2_stb_insert) { option = {0, true, true, NULL, NULL}; { do_stmt(taos, &option, "insert into ? values(?,?)", 3, 3, 3, false, true); } + do_query(taos, "drop database if exists stmt2_testdb_1"); taos_close(taos); } @@ -858,10 +881,10 @@ TEST(stmt2Case, stmt2_stb_insert) { TEST(stmt2Case, stmt2_insert_non_statndard) { TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); ASSERT_NE(taos, nullptr); - do_query(taos, "drop database if exists example_all_type_stmt1"); - do_query(taos, "create database IF NOT EXISTS example_all_type_stmt1"); + do_query(taos, "drop database if exists stmt2_testdb_6"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_6"); do_query(taos, - "create stable example_all_type_stmt1.stb1 (ts timestamp, int_col int,long_col bigint,double_col " + "create stable stmt2_testdb_6.stb1 (ts timestamp, int_col int,long_col bigint,double_col " "double,bool_col bool,binary_col binary(20),nchar_col nchar(20),varbinary_col varbinary(20),geometry_col " "geometry(200)) tags(int_tag int,long_tag bigint,double_tag double,bool_tag bool,binary_tag " "binary(20),nchar_tag nchar(20),varbinary_tag varbinary(20),geometry_tag geometry(200));"); @@ -872,7 +895,7 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { { TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); ASSERT_NE(stmt, nullptr); - const char* sql = "INSERT INTO example_all_type_stmt1.stb1 (ts,int_tag,tbname) VALUES (?,?,?)"; + const char* sql = "INSERT INTO stmt2_testdb_6.stb1 (ts,int_tag,tbname) VALUES (?,?,?)"; int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); int total_affect_rows = 0; @@ -912,9 +935,8 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { { TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); ASSERT_NE(stmt, nullptr); - const char* sql = - "INSERT INTO example_all_type_stmt1.stb1 (binary_tag,int_col,tbname,ts,int_tag) VALUES (?,?,?,?,?)"; - int code = taos_stmt2_prepare(stmt, sql, 0); + const char* sql = "INSERT INTO stmt2_testdb_6.stb1 (binary_tag,int_col,tbname,ts,int_tag) VALUES (?,?,?,?,?)"; + int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); int tag_i = 0; @@ -954,6 +976,7 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { taos_stmt2_close(stmt); } + do_query(taos, "drop database if exists stmt2_testdb_6"); taos_close(taos); } @@ -961,10 +984,10 @@ TEST(stmt2Case, stmt2_insert_non_statndard) { TEST(stmt2Case, stmt2_insert_db) { TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); ASSERT_NE(taos, nullptr); - do_query(taos, "drop database if exists example_all_type_stmt1"); - do_query(taos, "create database IF NOT EXISTS example_all_type_stmt1"); + do_query(taos, "drop database if exists stmt2_testdb_12"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_12"); do_query(taos, - "create stable `example_all_type_stmt1`.`stb1` (ts timestamp, int_col int,long_col bigint,double_col " + "create stable `stmt2_testdb_12`.`stb1` (ts timestamp, int_col int,long_col bigint,double_col " "double,bool_col bool,binary_col binary(20),nchar_col nchar(20),varbinary_col varbinary(20),geometry_col " "geometry(200)) tags(int_tag int,long_tag bigint,double_tag double,bool_tag bool,binary_tag " "binary(20),nchar_tag nchar(20),varbinary_tag varbinary(20),geometry_tag geometry(200));"); @@ -973,7 +996,7 @@ TEST(stmt2Case, stmt2_insert_db) { TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); ASSERT_NE(stmt, nullptr); - const char* sql = "INSERT INTO `example_all_type_stmt1`.`stb1` (ts,int_tag,tbname) VALUES (?,?,?)"; + const char* sql = "INSERT INTO `stmt2_testdb_12`.`stb1` (ts,int_tag,tbname) VALUES (?,?,?)"; int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); @@ -1006,38 +1029,38 @@ TEST(stmt2Case, stmt2_insert_db) { ASSERT_EQ(total_affect_rows, 12); taos_stmt2_close(stmt); + do_query(taos, "drop database if exists stmt2_testdb_12"); taos_close(taos); } TEST(stmt2Case, stmt2_query) { TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); ASSERT_NE(taos, nullptr); - do_query(taos, "drop database if exists testdb7"); - do_query(taos, "create database IF NOT EXISTS testdb7"); - do_query(taos, "create stable testdb7.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); + do_query(taos, "drop database if exists stmt2_testdb_7"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_7"); + do_query(taos, "create stable stmt2_testdb_7.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); do_query(taos, - "insert into testdb7.tb1 using testdb7.stb tags(1,'abc') values(1591060628000, " + "insert into stmt2_testdb_7.tb1 using stmt2_testdb_7.stb tags(1,'abc') values(1591060628000, " "'abc'),(1591060628001,'def'),(1591060628002, 'hij')"); do_query(taos, - "insert into testdb7.tb2 using testdb7.stb tags(2,'xyz') values(1591060628000, " - "'abc'),(1591060628001,'def'),(1591060628002, 'hij')"); - do_query(taos, "use testdb7"); + "insert into stmt2_testdb_7.tb2 using stmt2_testdb_7.stb tags(2,'xyz') values(1591060628000, " + "'abc'),(1591060628001,'def'),(1591060628004, 'hij')"); + do_query(taos, "use stmt2_testdb_7"); TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); ASSERT_NE(stmt, nullptr); - const char* sql = "select * from testdb7.stb where ts = ? and tbname = ?"; + const char* sql = "select * from stmt2_testdb_7.stb where ts = ?"; int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); int t64_len[1] = {sizeof(int64_t)}; int b_len[1] = {3}; int64_t ts = 1591060628000; - TAOS_STMT2_BIND params[2] = {{TSDB_DATA_TYPE_TIMESTAMP, &ts, t64_len, NULL, 1}, - {TSDB_DATA_TYPE_BINARY, (void*)"tb1", b_len, NULL, 1}}; - TAOS_STMT2_BIND* paramv = ¶ms[0]; + TAOS_STMT2_BIND params = {TSDB_DATA_TYPE_TIMESTAMP, &ts, t64_len, NULL, 1}; + TAOS_STMT2_BIND* paramv = ¶ms; TAOS_STMT2_BINDV bindv = {1, NULL, NULL, ¶mv}; code = taos_stmt2_bind_param(stmt, &bindv, -1); checkError(stmt, code); @@ -1048,15 +1071,31 @@ TEST(stmt2Case, stmt2_query) { TAOS_RES* pRes = taos_stmt2_result(stmt); ASSERT_NE(pRes, nullptr); - int getRecordCounts = 0; - TAOS_ROW row; - while ((row = taos_fetch_row(pRes))) { + int getRecordCounts = 0; + while ((taos_fetch_row(pRes))) { + getRecordCounts++; + } + ASSERT_EQ(getRecordCounts, 2); + // test 1 result + ts = 1591060628004; + params = {TSDB_DATA_TYPE_TIMESTAMP, &ts, t64_len, NULL, 1}; + code = taos_stmt2_bind_param(stmt, &bindv, -1); + checkError(stmt, code); + + taos_stmt2_exec(stmt, NULL); + checkError(stmt, code); + + pRes = taos_stmt2_result(stmt); + ASSERT_NE(pRes, nullptr); + + getRecordCounts = 0; + while ((taos_fetch_row(pRes))) { getRecordCounts++; } ASSERT_EQ(getRecordCounts, 1); // taos_free_result(pRes); - taos_stmt2_close(stmt); + do_query(taos, "drop database if exists stmt2_testdb_7"); taos_close(taos); } @@ -1064,16 +1103,16 @@ TEST(stmt2Case, stmt2_ntb_insert) { TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); ASSERT_NE(taos, nullptr); TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; - do_query(taos, "drop database if exists testdb8"); - do_query(taos, "create database IF NOT EXISTS testdb8"); - do_query(taos, "create table testdb8.ntb(ts timestamp, b binary(10))"); - do_query(taos, "use testdb8"); + do_query(taos, "drop database if exists stmt2_testdb_8"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_8"); + do_query(taos, "create table stmt2_testdb_8.ntb(ts timestamp, b binary(10))"); + do_query(taos, "use stmt2_testdb_8"); TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); ASSERT_NE(stmt, nullptr); int total_affected_rows = 0; - const char* sql = "insert into testdb8.ntb values(?,?)"; + const char* sql = "insert into stmt2_testdb_8.ntb values(?,?)"; int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); for (int i = 0; i < 3; i++) { @@ -1101,6 +1140,7 @@ TEST(stmt2Case, stmt2_ntb_insert) { ASSERT_EQ(total_affected_rows, 9); taos_stmt2_close(stmt); + do_query(taos, "drop database if exists stmt2_testdb_8"); taos_close(taos); } @@ -1125,7 +1165,7 @@ TEST(stmt2Case, stmt2_status_Test) { ASSERT_EQ(code, TSDB_CODE_TSC_STMT_API_ERROR); ASSERT_STREQ(taos_stmt2_error(stmt), "Stmt API usage error"); - const char* sql = "insert into testdb9.ntb values(?,?)"; + const char* sql = "insert into stmt2_testdb_9.ntb values(?,?)"; code = taos_stmt2_prepare(stmt, sql, 0); ASSERT_EQ(code, TSDB_CODE_TSC_STMT_API_ERROR); ASSERT_STREQ(taos_stmt2_error(stmt), "Stmt API usage error"); @@ -1136,9 +1176,9 @@ TEST(stmt2Case, stmt2_status_Test) { TEST(stmt2Case, stmt2_nchar) { TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); - do_query(taos, "drop database if exists testdb10;"); - do_query(taos, "create database IF NOT EXISTS testdb10;"); - do_query(taos, "use testdb10;"); + do_query(taos, "drop database if exists stmt2_testdb_10;"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_10;"); + do_query(taos, "use stmt2_testdb_10;"); do_query(taos, "create table m1 (ts timestamp, blob2 nchar(10), blob nchar(10),blob3 nchar(10),blob4 nchar(10),blob5 " "nchar(10))"); @@ -1244,6 +1284,7 @@ TEST(stmt2Case, stmt2_nchar) { ASSERT_EQ(affected_rows, 10); taos_stmt2_close(stmt); + do_query(taos, "drop database if exists stmt2_testdb_10;"); taos_close(taos); taosMemoryFree(blob_len); taosMemoryFree(blob_len2); @@ -1256,11 +1297,12 @@ TEST(stmt2Case, all_type) { TAOS* taos = taos_connect("localhost", "root", "taosdata", "", 0); ASSERT_NE(taos, nullptr); - do_query(taos, "drop database if exists testdb11"); - do_query(taos, "create database IF NOT EXISTS testdb11"); + do_query(taos, "drop database if exists stmt2_testdb_11"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_11"); do_query( taos, - "create stable testdb11.stb(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 smallint, c7 " + "create stable stmt2_testdb_11.stb(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 " + "smallint, c7 " "tinyint, c8 bool, c9 nchar(8), c10 geometry(256))TAGS(tts timestamp, t1 int, t2 bigint, t3 float, t4 double, t5 " "binary(8), t6 smallint, t7 tinyint, t8 bool, t9 nchar(8), t10 geometry(256))"); @@ -1370,7 +1412,7 @@ TEST(stmt2Case, all_type) { params[10].is_null = NULL; params[10].num = 1; - char* stmt_sql = "insert into testdb11.? using stb tags(?,?,?,?,?,?,?,?,?,?,?)values (?,?,?,?,?,?,?,?,?,?,?)"; + char* stmt_sql = "insert into stmt2_testdb_11.? using stb tags(?,?,?,?,?,?,?,?,?,?,?)values (?,?,?,?,?,?,?,?,?,?,?)"; code = taos_stmt2_prepare(stmt, stmt_sql, 0); checkError(stmt, code); @@ -1388,6 +1430,7 @@ TEST(stmt2Case, all_type) { geosFreeBuffer(outputGeom1); taos_stmt2_close(stmt); + do_query(taos, "drop database if exists stmt2_testdb_11"); taos_close(taos); } @@ -1395,31 +1438,29 @@ TEST(stmt2Case, geometry) { TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - do_query(taos, "DROP DATABASE IF EXISTS testdb15"); - do_query(taos, "CREATE DATABASE IF NOT EXISTS testdb15"); - do_query(taos, "CREATE TABLE testdb15.tb1(ts timestamp,c1 geometry(256))"); + do_query(taos, "DROP DATABASE IF EXISTS stmt2_testdb_13"); + do_query(taos, "CREATE DATABASE IF NOT EXISTS stmt2_testdb_13"); + do_query(taos, "CREATE TABLE stmt2_testdb_13.tb1(ts timestamp,c1 geometry(256))"); TAOS_STMT2_OPTION option = {0}; TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); ASSERT_NE(stmt, nullptr); - unsigned char wkb1[] = { // 1 - 0x01, // 字节顺序：小端字节序 - 0x01, 0x00, 0x00, 0x00, // 几何类型：Point (1) - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, // p1 - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, // p2 - // 2 - 0x01, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, - 0x3f, - // 3 - 0x01, - 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, - 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x40}; + unsigned char wkb1[] = { + // 1 + 0x01, // 字节顺序：小端字节序 + 0x01, 0x00, 0x00, 0x00, // 几何类型：Point (1) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, // p1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, // p2 + // 2 + 0x01, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xf0, 0x3f, + // 3 + 0x01, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40}; // unsigned char* wkb_all[3]{&wkb1[0], &wkb2[0], &wkb3[0]}; int32_t wkb_len[3] = {21, 61, 41}; @@ -1440,7 +1481,7 @@ TEST(stmt2Case, geometry) { params[1].is_null = NULL; params[1].num = 3; - char* stmt_sql = "insert into testdb15.tb1 (ts,c1)values(?,?)"; + char* stmt_sql = "insert into stmt2_testdb_13.tb1 (ts,c1)values(?,?)"; int code = taos_stmt2_prepare(stmt, stmt_sql, 0); checkError(stmt, code); @@ -1455,6 +1496,7 @@ TEST(stmt2Case, geometry) { ASSERT_EQ(affected_rows, 3); taos_stmt2_close(stmt); + do_query(taos, "DROP DATABASE IF EXISTS stmt2_testdb_13"); taos_close(taos); } #pragma GCC diagnostic pop diff --git a/source/client/test/stmtTest.cpp b/source/client/test/stmtTest.cpp index 510322ca87..77130e41db 100644 --- a/source/client/test/stmtTest.cpp +++ b/source/client/test/stmtTest.cpp @@ -52,6 +52,11 @@ void do_query(TAOS *taos, const char *sql) { TAOS_RES *result = taos_query(taos, sql); // printf("sql: %s\n", sql); int code = taos_errno(result); + while (code == TSDB_CODE_MND_DB_IN_CREATING || code == TSDB_CODE_MND_DB_IN_DROPPING) { + taosMsleep(2000); + result = taos_query(taos, sql); + code = taos_errno(result); + } if (code != TSDB_CODE_SUCCESS) { printf("query failen sql : %s\n errstr : %s\n", sql, taos_errstr(result)); ASSERT_EQ(taos_errno(result), TSDB_CODE_SUCCESS); @@ -69,12 +74,13 @@ typedef struct { void insertData(TAOS *taos, TAOS_STMT_OPTIONS *option, const char *sql, int CTB_NUMS, int ROW_NUMS, int CYC_NUMS, bool isCreateTable) { // create database and table - do_query(taos, "DROP DATABASE IF EXISTS testdb2"); - do_query(taos, "CREATE DATABASE IF NOT EXISTS testdb2"); - do_query(taos, - "CREATE STABLE IF NOT EXISTS testdb2.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS " - "(groupId INT, location BINARY(24))"); - do_query(taos, "USE testdb2"); + do_query(taos, "DROP DATABASE IF EXISTS stmt_testdb_2"); + do_query(taos, "CREATE DATABASE IF NOT EXISTS stmt_testdb_2"); + do_query( + taos, + "CREATE STABLE IF NOT EXISTS stmt_testdb_2.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS " + "(groupId INT, location BINARY(24))"); + do_query(taos, "USE stmt_testdb_2"); // init TAOS_STMT *stmt; @@ -173,7 +179,7 @@ void insertData(TAOS *taos, TAOS_STMT_OPTIONS *option, const char *sql, int CTB_ for (int j = 0; j < ROW_NUMS; j++) { struct timeval tv; (&tv, NULL); - int64_t ts = 1591060628000 + j + k * 100; + int64_t ts = 1591060628000 + j + k * 100000; float current = (float)0.0001f * j; int voltage = j; float phase = (float)0.0001f * j; @@ -207,12 +213,13 @@ void insertData(TAOS *taos, TAOS_STMT_OPTIONS *option, const char *sql, int CTB_ void getFields(TAOS *taos, const char *sql, int expectedALLFieldNum, TAOS_FIELD_E *expectedTagFields, int expectedTagFieldNum, TAOS_FIELD_E *expectedColFields, int expectedColFieldNum) { // create database and table - do_query(taos, "DROP DATABASE IF EXISTS testdb3"); - do_query(taos, "CREATE DATABASE IF NOT EXISTS testdb3"); - do_query(taos, "USE testdb3"); - do_query(taos, - "CREATE STABLE IF NOT EXISTS testdb3.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS " - "(groupId INT, location BINARY(24))"); + do_query(taos, "DROP DATABASE IF EXISTS stmt_testdb_3"); + do_query(taos, "CREATE DATABASE IF NOT EXISTS stmt_testdb_3"); + do_query(taos, "USE stmt_testdb_3"); + do_query( + taos, + "CREATE STABLE IF NOT EXISTS stmt_testdb_3.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS " + "(groupId INT, location BINARY(24))"); TAOS_STMT *stmt = taos_stmt_init(taos); ASSERT_NE(stmt, nullptr); @@ -271,7 +278,7 @@ TEST(stmtCase, stb_insert) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); // interlace = 0 - { insertData(taos, nullptr, "INSERT INTO testdb2.? USING meters TAGS(?,?) VALUES (?,?,?,?)", 1, 1, 1, false); } + { insertData(taos, nullptr, "INSERT INTO stmt_testdb_2.? USING meters TAGS(?,?) VALUES (?,?,?,?)", 1, 1, 1, false); } { insertData(taos, nullptr, "INSERT INTO ? USING meters TAGS(?,?) VALUES (?,?,?,?)", 3, 3, 3, false); } @@ -283,6 +290,7 @@ TEST(stmtCase, stb_insert) { insertData(taos, &options, "INSERT INTO ? VALUES (?,?,?,?)", 3, 3, 3, true); } + do_query(taos, "DROP DATABASE IF EXISTS stmt_testdb_2"); taos_close(taos); } @@ -299,18 +307,20 @@ TEST(stmtCase, get_fields) { {"phase", TSDB_DATA_TYPE_FLOAT, 0, 0, sizeof(float)}}; getFields(taos, "INSERT INTO ? USING meters TAGS(?,?) VALUES (?,?,?,?)", 7, &tagFields[0], 2, &colFields[0], 4); } + do_query(taos, "DROP DATABASE IF EXISTS stmt_testdb_3"); taos_close(taos); } -/* + TEST(stmtCase, all_type) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - do_query(taos, "DROP DATABASE IF EXISTS testdb1"); - do_query(taos, "CREATE DATABASE IF NOT EXISTS testdb1"); + do_query(taos, "DROP DATABASE IF EXISTS stmt_testdb_1"); + do_query(taos, "CREATE DATABASE IF NOT EXISTS stmt_testdb_1"); do_query( taos, - "CREATE STABLE testdb1.stb(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 smallint, c7 " + "CREATE STABLE stmt_testdb_1.stb1(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 " + "smallint, c7 " "tinyint, c8 bool, c9 nchar(8), c10 geometry(100))TAGS(tts timestamp, t1 int, t2 bigint, t3 float, t4 double, t5 " "binary(8), t6 smallint, t7 tinyint, t8 bool, t9 nchar(8), t10 geometry(100))"); @@ -418,18 +428,22 @@ TEST(stmtCase, all_type) { params[9].is_null = NULL; params[9].num = 1; + size_t size; + int code = initCtxGeomFromText(); + checkError(stmt, code); + unsigned char *outputGeom1; - size_t size1; - initCtxMakePoint(); - int code = doMakePoint(1.000, 2.000, &outputGeom1, &size1); + const char *wkt = "LINESTRING(1.0 1.0, 2.0 2.0)"; + code = doGeomFromText(wkt, &outputGeom1, &size); checkError(stmt, code); params[10].buffer_type = TSDB_DATA_TYPE_GEOMETRY; params[10].buffer = outputGeom1; - params[10].length = (int32_t *)&size1; + params[9].buffer_length = size; + params[10].length = (int32_t *)&size; params[10].is_null = NULL; params[10].num = 1; - char *stmt_sql = "insert into testdb1.? using stb tags(?,?,?,?,?,?,?,?,?,?,?)values (?,?,?,?,?,?,?,?,?,?,?)"; + char *stmt_sql = "insert into stmt_testdb_1.? using stb1 tags(?,?,?,?,?,?,?,?,?,?,?)values (?,?,?,?,?,?,?,?,?,?,?)"; code = taos_stmt_prepare(stmt, stmt_sql, 0); checkError(stmt, code); @@ -449,17 +463,17 @@ TEST(stmtCase, all_type) { checkError(stmt, code); taos_stmt_close(stmt); + do_query(taos, "DROP DATABASE IF EXISTS stmt_testdb_1"); taos_close(taos); } -*/ TEST(stmtCase, geometry) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); - do_query(taos, "DROP DATABASE IF EXISTS testdb5"); - do_query(taos, "CREATE DATABASE IF NOT EXISTS testdb5"); - do_query(taos, "CREATE TABLE testdb5.tb1(ts timestamp,c1 geometry(256))"); + do_query(taos, "DROP DATABASE IF EXISTS stmt_testdb_5"); + do_query(taos, "CREATE DATABASE IF NOT EXISTS stmt_testdb_5"); + do_query(taos, "CREATE TABLE stmt_testdb_5.tb1(ts timestamp,c1 geometry(256))"); TAOS_STMT *stmt = taos_stmt_init(taos); ASSERT_NE(stmt, nullptr); @@ -468,7 +482,6 @@ TEST(stmtCase, geometry) { 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, }, - // {0x01, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, @@ -503,7 +516,7 @@ TEST(stmtCase, geometry) { params[1].is_null = NULL; params[1].num = 3; - char *stmt_sql = "insert into testdb5.tb1 (ts,c1)values(?,?)"; + char *stmt_sql = "insert into stmt_testdb_5.tb1 (ts,c1)values(?,?)"; int code = taos_stmt_prepare(stmt, stmt_sql, 0); checkError(stmt, code); @@ -522,6 +535,7 @@ TEST(stmtCase, geometry) { taosMemoryFree(t64_len); taosMemoryFree(wkb_len); taos_stmt_close(stmt); + do_query(taos, "DROP DATABASE IF EXISTS stmt_testdb_5"); taos_close(taos); } From 9729b8b9dce16867cfaa1c6c803a6739a25e6e69 Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Mon, 13 Jan 2025 15:51:15 +0800 Subject: [PATCH 24/87] fix other async create database case --- source/client/test/clientTests.cpp | 12 ++++++++++++ source/client/test/connectOptionsTest.cpp | 8 +++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 60f0a72e39..54c0e59817 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -532,6 +532,10 @@ TEST(clientCase, create_stable_Test) { taos_free_result(pRes); pRes = taos_query(pConn, "use abc1"); + while (taos_errno(pRes) == TSDB_CODE_MND_DB_IN_CREATING || taos_errno(pRes) == TSDB_CODE_MND_DB_IN_DROPPING) { + taosMsleep(2000); + pRes = taos_query(pConn, "use abc1"); + } taos_free_result(pRes); pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); @@ -664,6 +668,10 @@ TEST(clientCase, create_multiple_tables) { taos_free_result(pRes); pRes = taos_query(pConn, "use abc1"); + while (taos_errno(pRes) == TSDB_CODE_MND_DB_IN_CREATING || taos_errno(pRes) == TSDB_CODE_MND_DB_IN_DROPPING) { + taosMsleep(2000); + pRes = taos_query(pConn, "use abc1"); + } if (taos_errno(pRes) != 0) { (void)printf("failed to use db, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -1524,6 +1532,10 @@ TEST(clientCase, timezone_Test) { taos_free_result(pRes); pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)"); + while (taos_errno(pRes) == TSDB_CODE_MND_DB_IN_CREATING || taos_errno(pRes) == TSDB_CODE_MND_DB_IN_DROPPING) { + taosMsleep(2000); + pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)"); + } ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); taos_free_result(pRes); diff --git a/source/client/test/connectOptionsTest.cpp b/source/client/test/connectOptionsTest.cpp index 95596e9ed3..4f0dbb579b 100644 --- a/source/client/test/connectOptionsTest.cpp +++ b/source/client/test/connectOptionsTest.cpp @@ -55,7 +55,13 @@ TAOS* getConnWithOption(const char *tz){ void execQuery(TAOS* pConn, const char *sql){ TAOS_RES* pRes = taos_query(pConn, sql); - ASSERT(taos_errno(pRes) == TSDB_CODE_SUCCESS); + int code = taos_errno(pRes); + while (code == TSDB_CODE_MND_DB_IN_CREATING || code == TSDB_CODE_MND_DB_IN_DROPPING) { + taosMsleep(2000); + TAOS_RES* pRes = taos_query(pConn, sql); + code = taos_errno(pRes); + } + ASSERT(code == TSDB_CODE_SUCCESS); taos_free_result(pRes); } From f0fb2fb55dd4052ef7130a1ae04d1716f6138bf7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Jan 2025 18:27:59 +0800 Subject: [PATCH 25/87] fix:[TD-33504]add test case --- tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/tmq_td33504.py | 84 ++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 tests/system-test/7-tmq/tmq_td33504.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ea92f0bef7..15e073e8e6 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -328,6 +328,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_td33504.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py diff --git a/tests/system-test/7-tmq/tmq_td33504.py b/tests/system-test/7-tmq/tmq_td33504.py new file mode 100644 index 0000000000..085b245dd5 --- /dev/null +++ b/tests/system-test/7-tmq/tmq_td33504.py @@ -0,0 +1,84 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +from taos import * + +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def test(self): + tdSql.execute(f'create database if not exists db') + tdSql.execute(f'use db') + tdSql.execute(f'CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)') + tdSql.execute("INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-10-05 14:38:05.000',10.30000,219,0.31000)") + tdSql.execute("INSERT INTO d1002 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-10-05 14:38:05.000',10.30000,219,0.31000)") + tdSql.execute("INSERT INTO d1003 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-10-05 14:38:05.000',10.30000,219,0.31000)") + tdSql.execute("INSERT INTO d1004 USING meters TAGS('California.SanFrancisco', 2) VALUES('2018-10-05 14:38:05.000',10.30000,219,0.31000)") + + tdSql.execute(f'create topic t0 as select * from meters') + tdSql.execute(f'create topic t1 as select * from meters') + + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["t0"]) + except TmqError: + tdLog.exit(f"subscribe error") + + try: + res = consumer.poll(1) + print(res) + + consumer.unsubscribe() + + try: + consumer.subscribe(["t1"]) + except TmqError: + tdLog.exit(f"subscribe error") + + + res = consumer.poll(1) + print(res) + if res == None and taos_errno(None) != 0: + tdLog.exit(f"poll error %d" % taos_errno(None)) + + except TmqError: + tdLog.exit(f"poll error") + finally: + consumer.close() + + + def run(self): + self.test() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 982790b93a5e0ae1949e2e75075926ac36fd29d0 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Tue, 14 Jan 2025 11:32:25 +0800 Subject: [PATCH 26/87] Update build step notice and commands by charles --- tests/README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 9d7f98b564..6ac47ff7e0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -21,14 +21,23 @@ Install the Python connector for TDengine. pip3 install taospy taos-ws-py ``` +>[!NTOE] +>Please make sure building operation with option '-DBUILD_TEST=true' has been finished, execute the below commands if not: + +```bash +cd debug +cmake .. -DBUILD_TEST=true -DBUILD_CONTRIB=true +make && make install +``` + ## Unit Test Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run the script with below command: ```bash -cd tests/unit-test/ -e 0 -bash test.sh +cd tests/unit-test/ +bash test.sh -e 0 ``` ## System Test From b3b4623fab3b52e06009db1573dc1bd4d8f7e1d1 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Tue, 14 Jan 2025 11:36:11 +0800 Subject: [PATCH 27/87] fix spelling issue --- tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 6ac47ff7e0..35acb3f019 100644 --- a/tests/README.md +++ b/tests/README.md @@ -21,8 +21,8 @@ Install the Python connector for TDengine. pip3 install taospy taos-ws-py ``` ->[!NTOE] ->Please make sure building operation with option '-DBUILD_TEST=true' has been finished, execute the below commands if not: +> [!NOTE] +> Please make sure building operation with option '-DBUILD_TEST=true' has been finished, execute the below commands if not: ```bash cd debug From ae28c958d573a78dba02932ddd8025fc97f24510 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 14 Jan 2025 11:44:06 +0800 Subject: [PATCH 28/87] fix: use ps -C to find process pid --- packaging/tools/remove.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 5bbfd2a0de..77d46627a7 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -90,7 +90,7 @@ fi kill_service_of() { _service=$1 - pid=$(ps -C $_service | grep -v $uninstallScript | awk '{print $2}') + pid=$(ps -C $_service |grep -w $_service | grep -v $uninstallScript | awk '{print $1}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi @@ -140,9 +140,8 @@ clean_service_of() { clean_service_on_systemd_of $_service elif ((${service_mod} == 1)); then clean_service_on_sysvinit_of $_service - else - kill_service_of $_service fi + kill_service_of $_service } remove_service_of() { From 6e71f41e8c2c0e86f0acac1a989e45a67e04b142 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 14 Jan 2025 12:23:59 +0800 Subject: [PATCH 29/87] fix: use ps -C to find process pid --- packaging/tools/remove_client.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 7798bbf16a..c883fa0af2 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -40,7 +40,7 @@ if command -v sudo > /dev/null; then fi function kill_client() { - pid=$(ps -C ${clientName2} | grep -v $uninstallScript2 | awk '{print $2}') + pid=$(ps -C ${clientName2} | gerep -w ${clientName2} | grep -v $uninstallScript2 | awk '{print $1}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi From 184162545ba5d7a0ea1186f78315f0b79ccf8fca Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 14 Jan 2025 13:49:27 +0800 Subject: [PATCH 30/87] Update packaging/tools/remove.sh Co-authored-by: WANG Xu --- packaging/tools/remove.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 77d46627a7..6e309f1f67 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -90,7 +90,7 @@ fi kill_service_of() { _service=$1 - pid=$(ps -C $_service |grep -w $_service | grep -v $uninstallScript | awk '{print $1}') + pid=$(ps -C $_service | grep -w $_service | grep -v $uninstallScript | awk '{print $1}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi From 379be9da02bc8604a2175ad53dd80414db6e4aca Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 14 Jan 2025 13:49:33 +0800 Subject: [PATCH 31/87] Update packaging/tools/remove_client.sh Co-authored-by: WANG Xu --- packaging/tools/remove_client.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index c883fa0af2..3414992ad9 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -40,7 +40,7 @@ if command -v sudo > /dev/null; then fi function kill_client() { - pid=$(ps -C ${clientName2} | gerep -w ${clientName2} | grep -v $uninstallScript2 | awk '{print $1}') + pid=$(ps -C ${clientName2} | grep -w ${clientName2} | grep -v $uninstallScript2 | awk '{print $1}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi From 318a82f30f7a3f1bf88840255319c2379a3eadec Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Tue, 14 Jan 2025 13:55:23 +0800 Subject: [PATCH 32/87] remove space --- packaging/tools/remove.sh | 2 +- packaging/tools/remove_client.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 6e309f1f67..43c2de4ba4 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -90,7 +90,7 @@ fi kill_service_of() { _service=$1 - pid=$(ps -C $_service | grep -w $_service | grep -v $uninstallScript | awk '{print $1}') + pid=$(ps -C $_service | grep -w $_service | grep -v $uninstallScript | awk '{print $1}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 3414992ad9..1d2965f66b 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -40,7 +40,7 @@ if command -v sudo > /dev/null; then fi function kill_client() { - pid=$(ps -C ${clientName2} | grep -w ${clientName2} | grep -v $uninstallScript2 | awk '{print $1}') + pid=$(ps -C ${clientName2} | grep -w ${clientName2} | grep -v $uninstallScript2 | awk '{print $1}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi From 5b64c3429c72134b58f4c1a8dc903ec670a875b6 Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Tue, 14 Jan 2025 14:56:17 +0800 Subject: [PATCH 33/87] fix async error in unit test --- source/client/test/stmt2Test.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index 3e21721c47..52c89e97ab 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -197,7 +197,7 @@ void do_stmt(TAOS* taos, TAOS_STMT2_OPTION* option, const char* sql, int CTB_NUM checkError(stmt, code); // exec - int affected; + int affected = 0; code = taos_stmt2_exec(stmt, &affected); total_affected += affected; checkError(stmt, code); @@ -219,8 +219,9 @@ void do_stmt(TAOS* taos, TAOS_STMT2_OPTION* option, const char* sql, int CTB_NUM taosMemoryFree(tags); } } - - ASSERT_EQ(total_affected, CYC_NUMS * ROW_NUMS * CTB_NUMS); + if (option->asyncExecFn == NULL) { + ASSERT_EQ(total_affected, CYC_NUMS * ROW_NUMS * CTB_NUMS); + } for (int i = 0; i < CTB_NUMS; i++) { taosMemoryFree(tbs[i]); } From cae21da2d40593dd43f69e7e56691cd13a35c7fc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 14 Jan 2025 15:35:43 +0800 Subject: [PATCH 34/87] fix:[TS-5906]clear meta cache for subscription if tag is changed --- source/libs/executor/src/executor.c | 7 +++ tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/tmq_ts5906.py | 90 +++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 tests/system-test/7-tmq/tmq_ts5906.py diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index dffab1b163..1386b0b82f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -469,6 +469,13 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI } SStreamScanInfo* pScanInfo = pInfo->info; + if (pInfo->pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { // clear meta cache for subscription if tag is changed + for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { + int64_t* uid = (int64_t*)taosArrayGet(tableIdList, i); + STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; + taosLRUCacheErase(pTableScanInfo->base.metaCache.pTableMetaEntryCache, uid, LONG_BYTES); + } + } if (isAdd) { // add new table id SArray* qa = NULL; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ea92f0bef7..97509e453a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -329,6 +329,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5906.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts4563.py diff --git a/tests/system-test/7-tmq/tmq_ts5906.py b/tests/system-test/7-tmq/tmq_ts5906.py new file mode 100644 index 0000000000..13e756baa3 --- /dev/null +++ b/tests/system-test/7-tmq/tmq_ts5906.py @@ -0,0 +1,90 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +from taos import * + +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, 'asynclog': 0} + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def test(self): + tdSql.execute(f'create database if not exists db vgroups 1') + tdSql.execute(f'use db') + tdSql.execute(f'CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)') + tdSql.execute("INSERT INTO d1001 USING meters TAGS('California.SanFrancisco1', 2) VALUES('2018-10-05 14:38:05.000',10.30000,219,0.31000)") + + + tdSql.execute(f'create topic t0 as select * from meters') + + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["t0"]) + except TmqError: + tdLog.exit(f"subscribe error") + + index = 0; + try: + while True: + if index == 2: + break + res = consumer.poll(5) + print(res) + if not res: + print("res null") + break + val = res.value() + if val is None: + continue + for block in val: + data = block.fetchall() + for element in data: + print(f"data len: {len(data)}") + print(element) + if index == 0 and data[0][-1] != 2: + tdLog.exit(f"error: {data[0][-1]}") + if index == 1 and data[0][-1] != 100: + tdLog.exit(f"error: {data[0][-1]}") + + tdSql.execute("alter table d1001 set tag groupId = 100") + tdSql.execute("INSERT INTO d1001 VALUES('2018-10-05 14:38:06.000',10.30000,219,0.31000)") + index += 1 + finally: + consumer.close() + + + def run(self): + self.test() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 4ceef6ddf1cf6384d3ad366dbd881718d3a4228c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jan 2025 15:36:33 +0800 Subject: [PATCH 35/87] docs: fix configuration error in doc --- docs/en/14-reference/01-components/01-taosd.md | 5 ----- docs/en/14-reference/03-taos-sql/10-function.md | 2 +- docs/zh/14-reference/01-components/01-taosd.md | 5 ----- docs/zh/14-reference/03-taos-sql/10-function.md | 2 +- 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/en/14-reference/01-components/01-taosd.md b/docs/en/14-reference/01-components/01-taosd.md index 55db20bef0..7456593ddb 100644 --- a/docs/en/14-reference/01-components/01-taosd.md +++ b/docs/en/14-reference/01-components/01-taosd.md @@ -77,12 +77,7 @@ After modifying configuration file parameters, you need to restart the *taosd* s |minReservedMemorySize | |Not supported |The minimum reserved system available memory size, all memory except reserved can be used for queries, unit: MB, default reserved size is 20% of system physical memory, value range 1024-1000000000| |singleQueryMaxMemorySize| |Not supported |The memory limit that a single query can use on a single node (dnode), exceeding this limit will return an error, unit: MB, default value: 0 (no limit), value range 0-1000000000| |filterScalarMode | |Not supported |Force scalar filter mode, 0: off; 1: on, default value 0| -|queryPlannerTrace | |Supported, effective immediately |Internal parameter, whether the query plan outputs detailed logs| -|queryNodeChunkSize | |Supported, effective immediately |Internal parameter, chunk size of the query plan| -|queryUseNodeAllocator | |Supported, effective immediately |Internal parameter, allocation method of the query plan| -|queryMaxConcurrentTables| |Not supported |Internal parameter, concurrency number of the query plan| |queryRsmaTolerance | |Not supported |Internal parameter, tolerance time for determining which level of rsma data to query, in milliseconds| -|enableQueryHb | |Supported, effective immediately |Internal parameter, whether to send query heartbeat messages| |pqSortMemThreshold | |Not supported |Internal parameter, memory threshold for sorting| ### Region Related diff --git a/docs/en/14-reference/03-taos-sql/10-function.md b/docs/en/14-reference/03-taos-sql/10-function.md index ab5c48bce2..8397c59177 100644 --- a/docs/en/14-reference/03-taos-sql/10-function.md +++ b/docs/en/14-reference/03-taos-sql/10-function.md @@ -2171,7 +2171,7 @@ ignore_negative: { **Usage Instructions**: -- Can be used with the columns associated with the selection. For example: select _rowts, DERIVATIVE() from. +- Can be used with the columns associated with the selection. For example: select _rowts, DERIVATIVE(col1, 1s, 1) from tb1. ### DIFF diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index ff1d7617d8..4c3350df7c 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -73,12 +73,7 @@ taosd 命令行参数如下 |minReservedMemorySize | |不支持动态修改 |最小预留的系统可用内存数量，除预留外的内存都可以被用于查询，单位：MB，默认预留大小为系统物理内存的 20%，取值范围 1024 - 1000000000| |singleQueryMaxMemorySize| |不支持动态修改 |单个查询在单个节点(dnode)上可以使用的内存上限，超过该上限将返回错误，单位：MB，默认值：0（无上限），取值范围 0 - 1000000000| |filterScalarMode | |不支持动态修改 |强制使用标量过滤模式，0：关闭；1：开启，默认值 0| -|queryPlannerTrace | |支持动态修改 立即生效 |内部参数，查询计划是否输出详细日志| -|queryNodeChunkSize | |支持动态修改 立即生效 |内部参数，查询计划的块大小| -|queryUseNodeAllocator | |支持动态修改 立即生效 |内部参数，查询计划的分配方法| -|queryMaxConcurrentTables| |不支持动态修改 |内部参数，查询计划的并发数目| |queryRsmaTolerance | |不支持动态修改 |内部参数，用于判定查询哪一级 rsma 数据时的容忍时间，单位为毫秒| -|enableQueryHb | |支持动态修改 立即生效 |内部参数，是否发送查询心跳消息| |pqSortMemThreshold | |不支持动态修改 |内部参数，排序使用的内存阈值| ### 区域相关 diff --git a/docs/zh/14-reference/03-taos-sql/10-function.md b/docs/zh/14-reference/03-taos-sql/10-function.md index eb3a4bb0ed..c0e80e80df 100644 --- a/docs/zh/14-reference/03-taos-sql/10-function.md +++ b/docs/zh/14-reference/03-taos-sql/10-function.md @@ -2099,7 +2099,7 @@ ignore_negative: { **使用说明**: -- 可以与选择相关联的列一起使用。 例如: select \_rowts, DERIVATIVE() from。 +- 可以与选择相关联的列一起使用。 例如: select \_rowts, DERIVATIVE(col1, 1s, 1) from tb1。 ### DIFF From a1a6312db71f17b63300661cda0e5cad40bbc59d Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Tue, 14 Jan 2025 16:01:26 +0800 Subject: [PATCH 36/87] udpate test README to add build options by charles --- tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 35acb3f019..2a25740dc3 100644 --- a/tests/README.md +++ b/tests/README.md @@ -22,11 +22,11 @@ pip3 install taospy taos-ws-py ``` > [!NOTE] -> Please make sure building operation with option '-DBUILD_TEST=true' has been finished, execute the below commands if not: +> Please make sure building operation with option '-DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true' has been finished, execute the below commands if not: ```bash cd debug -cmake .. -DBUILD_TEST=true -DBUILD_CONTRIB=true +cmake .. -DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true make && make install ``` From bb8ef7272573644681df3bd7e5aa32ec889dae1a Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 14 Jan 2025 16:46:39 +0800 Subject: [PATCH 37/87] Update test_smoking_selfhost.sh --- packaging/smokeTest/test_smoking_selfhost.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packaging/smokeTest/test_smoking_selfhost.sh b/packaging/smokeTest/test_smoking_selfhost.sh index a25c5a6d90..6ed0b9c715 100755 --- a/packaging/smokeTest/test_smoking_selfhost.sh +++ b/packaging/smokeTest/test_smoking_selfhost.sh @@ -6,12 +6,6 @@ SUCCESS_FILE="success.txt" FAILED_FILE="failed.txt" REPORT_FILE="report.txt" -# Initialize/clear result files -> "$SUCCESS_FILE" -> "$FAILED_FILE" -> "$LOG_FILE" -> "$REPORT_FILE" - # Switch to the target directory TARGET_DIR="../../tests/system-test/" @@ -24,6 +18,12 @@ else exit 1 fi +# Initialize/clear result files +> "$SUCCESS_FILE" +> "$FAILED_FILE" +> "$LOG_FILE" +> "$REPORT_FILE" + # Define the Python commands to execute commands=( "python3 ./test.py -f 2-query/join.py" @@ -102,4 +102,4 @@ fi echo "Detailed logs can be found in: $(realpath "$LOG_FILE")" echo "Successful commands can be found in: $(realpath "$SUCCESS_FILE")" echo "Failed commands can be found in: $(realpath "$FAILED_FILE")" -echo "Test report can be found in: $(realpath "$REPORT_FILE")" \ No newline at end of file +echo "Test report can be found in: $(realpath "$REPORT_FILE")" From 11d34f728a8c9defe8c8aae42ff0a7ca998bd069 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 14 Jan 2025 17:04:07 +0800 Subject: [PATCH 38/87] fix: data load required --- source/libs/function/src/builtinsimpl.c | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 707018ac65..0f6db39cc8 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -771,7 +771,34 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } +static bool funcNotSupportStringSma(SFunctionNode* pFunc) { + SNode* pParam; + switch (pFunc->funcType) { + case FUNCTION_TYPE_MAX: + case FUNCTION_TYPE_MIN: + case FUNCTION_TYPE_SUM: + case FUNCTION_TYPE_AVG: + case FUNCTION_TYPE_AVG_PARTIAL: + case FUNCTION_TYPE_PERCENTILE: + case FUNCTION_TYPE_SPREAD: + case FUNCTION_TYPE_SPREAD_PARTIAL: + case FUNCTION_TYPE_SPREAD_MERGE: + case FUNCTION_TYPE_TWA: + pParam = nodesListGetNode(pFunc->pParameterList, 0); + if (pParam && nodesIsExprNode(pParam) && (IS_VAR_DATA_TYPE(((SExprNode*)pParam)->resType.type))) { + return true; + } + break; + default: + break; + } + return false; +} + EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { + if(funcNotSupportStringSma(pFunc)) { + return FUNC_DATA_REQUIRED_DATA_LOAD; + } return FUNC_DATA_REQUIRED_SMA_LOAD; } From ad50cde25e637342d1a07b6e88a6e6f8859d7bea Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Tue, 14 Jan 2025 17:31:42 +0800 Subject: [PATCH 39/87] add TSBS and TestNG test description and fold up for installation, running section by charles --- README.md | 81 +++++++++++++++++++++++++++++++++++++------------ tests/README.md | 12 +++++++- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 33874cf05d..a39ddc0f78 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,14 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine 1. [Documentation](#2-documentation) 1. [Prerequisites](#3-prerequisites) 1. [Building](#4-building) -1. [Installation](#5-installation) -1. [Running](#6-running) -1. [Testing](#7-testing) -1. [Releasing](#8-releasing) -1. [CI/CD](#9-cicd) -1. [Coverage](#10-coverage) -1. [Contributing](#11-contributing) +1. [Packaging](#5-packaging) +1. [Installation](#6-installation) +1. [Running](#7-running) +1. [Testing](#8-testing) +1. [Releasing](#9-releasing) +1. [CI/CD](#10-cicd) +1. [Coverage](#11-coverage) +1. [Contributing](#12-contributing) # 1. Introduction @@ -218,9 +219,17 @@ nmake ``` -# 5. Installation +# 5. Packaging -## 5.1 Install on Linux +TDengine packaging scripts depends on some private repositries currently, you can refer the link for detail steps. [Packaging](https://github.com/taosdata/TDinternal/tree/main?tab=readme-ov-file#5-packaging) + +# 6. Installation + +## 6.1 Install on Linux + +
+ +Detailed steps to install on Linux After building successfully, TDengine can be installed by: @@ -230,7 +239,13 @@ sudo make install Installing from source code will also configure service management for TDengine. Users can also choose to [install from packages](https://docs.tdengine.com/get-started/deploy-from-package/) for it. -## 5.2 Install on macOS +
+ +## 6.2 Install on macOS + +
+ +Detailed steps to install on macOS After building successfully, TDengine can be installed by: @@ -238,7 +253,13 @@ After building successfully, TDengine can be installed by: sudo make install ``` -## 5.3 Install on Windows +
+ +## 6.3 Install on Windows + +
+ +Detailed steps to install on windows After building successfully, TDengine can be installed by: @@ -246,9 +267,15 @@ After building successfully, TDengine can be installed by: nmake install ``` -# 6. Running +
-## 6.1 Run TDengine on Linux +# 7. Running + +## 7.1 Run TDengine on Linux + +
+ +Detailed steps to run on Linux To start the service after installation on linux, in a terminal, use: @@ -278,7 +305,13 @@ In another terminal, use the TDengine CLI to connect the server: Option `-c test/cfg` specifies the system configuration file directory. -## 6.2 Run TDengine on Windows +
+ +## 7.2 Run TDengine on Windows + +
+ +Detailed steps to run on windows You can start TDengine server on Windows platform with below commands: @@ -294,7 +327,13 @@ In another terminal, use the TDengine CLI to connect the server: option "-c test/cfg" specifies the system configuration file directory. -## 6.3 Run TDengine on macOS +
+ +## 7.3 Run TDengine on macOS + +
+ +Detailed steps to run on macOS To start the service after installation on macOS, double-click the /applications/TDengine to start the program, or in a terminal, use: @@ -310,15 +349,17 @@ taos If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. -# 7. Testing +
+ +# 8. Testing For how to run different types of tests on TDengine, please see [Testing TDengine](./tests/README.md). -# 8. Releasing +# 9. Releasing For the complete list of TDengine Releases, please see [Releases](https://github.com/taosdata/TDengine/releases). -# 9. CI/CD +# 10. CI/CD Now, Jenkins is mainly used to build CI/CD pipeline for TDengine. To run the tests in the CI/CD pipeline, please run following commands: @@ -329,7 +370,7 @@ cd tests TDengine build check workflow can be found in this [Github Action](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml). -# 10. Coverage +# 11. Coverage Latest TDengine test coverage report can be found on [coveralls.io](https://coveralls.io/github/taosdata/TDengine). To create the test coverage report (in HTML format) locally, please run following commands: @@ -342,6 +383,6 @@ bash setup-lcov.sh -v 1.16 && ./run_local_coverage.sh -b main -c task > [!NOTE] > Please note that the -b and -i options will recompile TDengine with the -DCOVER=true option, which may take a amount of time. -# 11. Contributing +# 12. Contributing Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to TDengine. diff --git a/tests/README.md b/tests/README.md index 2a25740dc3..843b9379da 100644 --- a/tests/README.md +++ b/tests/README.md @@ -43,7 +43,7 @@ bash test.sh -e 0 ## System Test Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only -work for TDengine enterprise version, you can run the script with below command: +work for TDengine Enterprise Edition, you can run the script with below command: ```bash cd tests/system-test @@ -69,5 +69,15 @@ cd tests/script ./test.sh -f tsim/db/basic1.sim ``` +## TSBS Test +Time Series Benchmark Suite (TSBS) is an open-source performance benchmarking platform specifically designed for time-series data processing systems, such as databases. It provides a standardized approach to evaluating the performance of various databases by simulating typical use cases such as IoT and DevOps. +TSBS Test is based on the TDengine Enterprise Edition and need private repositry privilege, you can refer the link for detail steps. [TSBS Test](https://github.com/taosdata/TDinternal/tree/main?tab=readme-ov-file#85-tsbs-test) + +## TestNG Test + +TestNG Test is another test framwork which developed by python, functionally speaking, it's a supplement for system test, and +also run longer time than system test for stability testing purposes. + +TestNG Test is based on the TDengine Enterprise Edition and need private repositry privilege, you can refer the link for detail steps. [TestNG Test](https://github.com/taosdata/TDinternal/tree/main?tab=readme-ov-file#87-testng-test) From c947101b8f09263868450a5fbd58b200eaeb042a Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Tue, 14 Jan 2025 17:55:17 +0800 Subject: [PATCH 40/87] test:add how to add test cases --- tests/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/README.md b/tests/README.md index 843b9379da..83e77ac20b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -69,6 +69,10 @@ cd tests/script ./test.sh -f tsim/db/basic1.sim ``` +## How TO Add Test Cases + +You can add sim test case under tests/script, python test case under tests/system-test or tests/army. When the case passes in the test branch, add the case to the cases.task file under tests/parallel_test, and then merge the pr into main branch to run in the future CI. + ## TSBS Test Time Series Benchmark Suite (TSBS) is an open-source performance benchmarking platform specifically designed for time-series data processing systems, such as databases. It provides a standardized approach to evaluating the performance of various databases by simulating typical use cases such as IoT and DevOps. From d9cbe335a7d96f525ede6caef736500004f2885a Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Tue, 14 Jan 2025 17:56:58 +0800 Subject: [PATCH 41/87] test:add how to add test cases --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 83e77ac20b..d269a43f90 100644 --- a/tests/README.md +++ b/tests/README.md @@ -47,7 +47,7 @@ work for TDengine Enterprise Edition, you can run the script with below command: ```bash cd tests/system-test -python3 ./test.py -f 2-query/floor.py +python3 ./test.py -f 2-query/avg.py ``` ## Smoke Test From f9a7dc0ea353614fd4ff4345d4943b877ba68999 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 15 Jan 2025 14:35:33 +0800 Subject: [PATCH 42/87] test case --- tests/system-test/2-query/smaTest.py | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py index 355ac04707..c0a81720ae 100644 --- a/tests/system-test/2-query/smaTest.py +++ b/tests/system-test/2-query/smaTest.py @@ -75,6 +75,7 @@ class TDTestCase: tdLog.debug(" LIMIT test_case2 ............ [OK]") self.test_TD_33336() + self.ts5900() # stop def stop(self): @@ -137,6 +138,47 @@ class TDTestCase: tdLog.debug("INSERT TABLE DATA ............ [OK]") return + + def ts5900query(self): + sql = "select max(c0) from ts5900.tt1" + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, '99.0') + sql = "select min(c0) from ts5900.tt1" + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, '1.0') + + def ts5900(self): + tdSql.execute("drop database if exists ts5900;") + tdSql.execute("create database ts5900;") + + tdSql.execute("create table ts5900.meters (ts timestamp, c0 varchar(64)) tags(t0 varchar(64));") + + sql = "CREATE TABLE ts5900.`tt1` USING ts5900.`meters` TAGS ('t11')" + tdSql.execute(sql) + for i in range(155): + tdSql.query(f"insert into ts5900.tt1 values(now+{i*10}s, '{i+1}.0')") + tdSql.query("insert into ts5900.tt1 values(now, '1.2')") + tdSql.query("insert into ts5900.tt1 values(now+1s, '2.0')") + tdSql.query("insert into ts5900.tt1 values(now+2s, '3.0')") + tdSql.query("insert into ts5900.tt1 values(now+3s, '105.0')") + tdSql.query("insert into ts5900.tt1 values(now+4s, '4.0')") + + sql = "select count(*) from ts5900.tt1" + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, '160') + + for i in range(10): + tdSql.execute("flush database ts5900") + time.sleep(1) + self.ts5900query() + tdSql.query(f"insert into ts5900.tt1 values(now, '23.0')") + self.ts5900query() + tdLog.info(f"ts5900 test {i} ............ [OK]") + time.sleep(1) + # test case1 base # def test_case1(self): From 904c7c79d3e5f74201bf5f48abf7bf4e70eb0e22 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:18:28 +0800 Subject: [PATCH 43/87] optimize the description in Update 02-database.md --- docs/zh/14-reference/03-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 32df6c60c1..33d327b620 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -209,7 +209,7 @@ REDISTRIBUTE VGROUP vgroup_no DNODE dnode_id1 [DNODE dnode_id2] [DNODE dnode_id3 BALANCE VGROUP LEADER ``` -触发集群所有 vgroup 中的 leader 重新选主，对集群各节点进行负载再均衡操作。 +触发集群所有 vgroup 中的 leader 重新选主，对集群各节点进行负载再均衡操作。（企业版功能） ## 查看数据库工作状态 From d15189291b1bbf927f4f2abbfdca1f4405c2caa6 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 16 Jan 2025 09:57:29 +0800 Subject: [PATCH 44/87] update testing README --- tests/README.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/README.md b/tests/README.md index d269a43f90..b47129e872 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,6 +1,13 @@ -# Testing TDengine +# Table of Contents +1. [Introduction](#1-introduction) +1. [Prerequisites](#2-prerequisites) +2. [Testing Guide](#3-testing-guide) -## Install the required tools +# 1. Introduction + +This manual is intended to provide users with comprehensive guidance to help them verify the TDengine function efficiently. The document is divided into three main sections: introduction, prerequisites and testing guide. + +# 2. Prerequisites Install Python3. @@ -30,6 +37,8 @@ cmake .. -DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true make && make install ``` +# 3. Testing Guide + ## Unit Test Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run @@ -47,7 +56,7 @@ work for TDengine Enterprise Edition, you can run the script with below command: ```bash cd tests/system-test -python3 ./test.py -f 2-query/avg.py +python3 ./test.py -f 2-query/floor.py ``` ## Smoke Test @@ -68,7 +77,6 @@ In the early stage of TDengine development, test cases are run by an internal te cd tests/script ./test.sh -f tsim/db/basic1.sim ``` - ## How TO Add Test Cases You can add sim test case under tests/script, python test case under tests/system-test or tests/army. When the case passes in the test branch, add the case to the cases.task file under tests/parallel_test, and then merge the pr into main branch to run in the future CI. From 5c052c1325e908c7e7bd4d2981206d59cb33f42f Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 16 Jan 2025 11:55:57 +0800 Subject: [PATCH 45/87] docs: update the structure for testing readme --- tests/README.md | 102 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 72 insertions(+), 30 deletions(-) diff --git a/tests/README.md b/tests/README.md index b47129e872..07d576ab4c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,35 +1,44 @@ # Table of Contents + 1. [Introduction](#1-introduction) 1. [Prerequisites](#2-prerequisites) -2. [Testing Guide](#3-testing-guide) +1. [Testing Guide](#3-testing-guide) + 1. [Unit Test](#31-unit-test) + 1. [System Test](#32-system-test) + 1. [Smoke Test](#33-smoke-test) + 1. [Legacy Test](#34-legacy-test) + 1. [Chaos Test](#35-chaos-test) + 1. [TSBS Test](#36-tsbs-test) # 1. Introduction -This manual is intended to provide users with comprehensive guidance to help them verify the TDengine function efficiently. The document is divided into three main sections: introduction, prerequisites and testing guide. +This manual is intended to provide users with comprehensive guidance to help them verify the TDengine function efficiently. The document is divided into three main sections: introduction, prerequisites and testing guide. -# 2. Prerequisites +# 2. Prerequisites -Install Python3. +- Install Python3 ```bash apt install python3 apt install python3-pip ``` -Install the dependent Python components. +- Install Python dependencies ```bash -pip3 install pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog +pip3 install pandas psutil fabric2 requests faker simplejson \ + toml pexpect tzlocal distro decorator loguru hyperloglog ``` -Install the Python connector for TDengine. +- Install Python connector for TDengine ```bash pip3 install taospy taos-ws-py ``` -> [!NOTE] -> Please make sure building operation with option '-DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true' has been finished, execute the below commands if not: +- Building + +Please make sure building operation with option `-DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true` has been finished, otherwise execute commands below: ```bash cd debug @@ -39,57 +48,90 @@ make && make install # 3. Testing Guide -## Unit Test +In `tests` directory, there are different types of tests for TDengine. Below is a brief introduction about how to run them and how to add new cases. -Unit test script is the smallest testable part and developed for some function, method or class of TDengine, you can run -the script with below command: +## 3.1. Unit Test + +Unit test script is the smallest testable part and developed for some function, method or class of TDengine. + +### How to run tests? ```bash cd tests/unit-test/ bash test.sh -e 0 ``` -## System Test +### How to add new cases? + +Copy from the old version, need updates: +You can add sim test case under tests/script, python test case under tests/system-test or tests/army. When the case passes in the test branch, add the case to the cases.task file under tests/parallel_test, and then merge the pr into main branch to run in the future CI. + +## 3.2. System Test Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only -work for TDengine Enterprise Edition, you can run the script with below command: +work for TDengine Enterprise Edition. + +### How to run tests? ```bash cd tests/system-test python3 ./test.py -f 2-query/floor.py ``` -## Smoke Test +### How to add new cases? -Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine, you can run the -script with commands below: +[Placeholder] + +## 3.3. Smoke Test + +Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine. + +### How to run tests? ```bash cd /root/TDengine/packaging/smokeTest ./test_smoking_selfhost.sh ``` -## Legacy Test +### How to add new cases? -In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++. To run the legacy test cases, please execute the following commands: +[Placeholder] + +## 3.4. Legacy Test + +In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++. + +### How to run tests? + +To run the legacy test cases, please execute the following commands: ```bash cd tests/script ./test.sh -f tsim/db/basic1.sim ``` -## How TO Add Test Cases -You can add sim test case under tests/script, python test case under tests/system-test or tests/army. When the case passes in the test branch, add the case to the cases.task file under tests/parallel_test, and then merge the pr into main branch to run in the future CI. +### How to add new cases? -## TSBS Test +[Placeholder] -Time Series Benchmark Suite (TSBS) is an open-source performance benchmarking platform specifically designed for time-series data processing systems, such as databases. It provides a standardized approach to evaluating the performance of various databases by simulating typical use cases such as IoT and DevOps. +## 3.5. Chaos Test + +[Desciprtion] + +### How to run tests? + +[Placeholder] + +### How to add new cases? + +[Placeholder] + +## 3.6. TSBS Test + +[Time Series Benchmark Suite (TSBS)](https://github.com/timescale/tsbs) is an open-source performance benchmarking platform specifically designed for time-series data processing systems, such as databases. It provides a standardized approach to evaluating the performance of various databases by simulating typical use cases such as IoT and DevOps. + +### How to run tests? + +Need updates: must be run from public repo! TSBS Test is based on the TDengine Enterprise Edition and need private repositry privilege, you can refer the link for detail steps. [TSBS Test](https://github.com/taosdata/TDinternal/tree/main?tab=readme-ov-file#85-tsbs-test) - -## TestNG Test - -TestNG Test is another test framwork which developed by python, functionally speaking, it's a supplement for system test, and -also run longer time than system test for stability testing purposes. - -TestNG Test is based on the TDengine Enterprise Edition and need private repositry privilege, you can refer the link for detail steps. [TestNG Test](https://github.com/taosdata/TDinternal/tree/main?tab=readme-ov-file#87-testng-test) From 674a3818be425829a57cdaef6f03a11a7e59ee5d Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 16 Jan 2025 12:07:43 +0800 Subject: [PATCH 46/87] Update README.md --- tests/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/README.md b/tests/README.md index 07d576ab4c..318024e8ff 100644 --- a/tests/README.md +++ b/tests/README.md @@ -130,6 +130,31 @@ cd tests/script [Time Series Benchmark Suite (TSBS)](https://github.com/timescale/tsbs) is an open-source performance benchmarking platform specifically designed for time-series data processing systems, such as databases. It provides a standardized approach to evaluating the performance of various databases by simulating typical use cases such as IoT and DevOps. +1. Clone the code and run the tests locally on your machine. Ensure that your virtual machine supports the AVX instruction set: +```bash + cd /usr/local/src && git clone https://github.com/taosdata/tsbs-internal.git tsbs && \ + cd tsbs && git checkout enh/chr-td-33357 && \ + cd scripts/tsdbComp && ./testTsbs.sh +``` +2. When testing the client and server on separate machines, you should set up your environment as outlined in the steps below: + + 2.1. Modify IP and host of client and server in `test.ini` + ```ini + clientIP="192.168.0.203" # client ip + clientHost="trd03" # client hostname + serverIP="192.168.0.204" # server ip + serverHost="trd04" # server hostname + ``` + 2.2. Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: + ```ini + serverPass="taosdata123" # server root password + ``` + 2.3. Run the following command to start the test: + ```bash + ./testTsbs.sh + ``` +3. When the test is done, the result can be found in `/data2/` directory, which can also be configured in `test.ini`. + ### How to run tests? Need updates: must be run from public repo! From 78479871339b39e3e3f89c63c9bd1a222c4db1e1 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 16 Jan 2025 12:09:18 +0800 Subject: [PATCH 47/87] add tsbs for README.md --- tests/README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/README.md b/tests/README.md index 318024e8ff..0e870b1fea 100644 --- a/tests/README.md +++ b/tests/README.md @@ -130,6 +130,8 @@ cd tests/script [Time Series Benchmark Suite (TSBS)](https://github.com/timescale/tsbs) is an open-source performance benchmarking platform specifically designed for time-series data processing systems, such as databases. It provides a standardized approach to evaluating the performance of various databases by simulating typical use cases such as IoT and DevOps. +### How to run tests? + 1. Clone the code and run the tests locally on your machine. Ensure that your virtual machine supports the AVX instruction set: ```bash cd /usr/local/src && git clone https://github.com/taosdata/tsbs-internal.git tsbs && \ @@ -154,9 +156,3 @@ cd tests/script ./testTsbs.sh ``` 3. When the test is done, the result can be found in `/data2/` directory, which can also be configured in `test.ini`. - -### How to run tests? - -Need updates: must be run from public repo! - -TSBS Test is based on the TDengine Enterprise Edition and need private repositry privilege, you can refer the link for detail steps. [TSBS Test](https://github.com/taosdata/TDinternal/tree/main?tab=readme-ov-file#85-tsbs-test) From d4aec7a3d2b0ffa7a5c609ee857db9e76b7a37e1 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 16 Jan 2025 12:22:03 +0800 Subject: [PATCH 48/87] docs: add note for tsbs test --- tests/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/README.md b/tests/README.md index 0e870b1fea..1466285096 100644 --- a/tests/README.md +++ b/tests/README.md @@ -138,6 +138,10 @@ cd tests/script cd tsbs && git checkout enh/chr-td-33357 && \ cd scripts/tsdbComp && ./testTsbs.sh ``` +> [!NOTE] +> If you want to quickly verify the test environment, please set the `caseType` in `test.ini` file to `cputest`. The current default test is the CPU scenario. After running, the test will take a very long time. + + 2. When testing the client and server on separate machines, you should set up your environment as outlined in the steps below: 2.1. Modify IP and host of client and server in `test.ini` From 28af1cc214b8108b045b3c992774ff7590be8a51 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 16 Jan 2025 12:29:34 +0800 Subject: [PATCH 49/87] Update README.md --- tests/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 1466285096..da70c9078d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -139,7 +139,9 @@ cd tests/script cd scripts/tsdbComp && ./testTsbs.sh ``` > [!NOTE] -> If you want to quickly verify the test environment, please set the `caseType` in `test.ini` file to `cputest`. The current default test is the CPU scenario. After running, the test will take a very long time. +> The default case type, CPU test, takes a long time to run. +> +> If you want to run quick test to verify the test environment, please set `caseType=cputest` in test.ini . 2. When testing the client and server on separate machines, you should set up your environment as outlined in the steps below: From 64c6562be3c38d901ec443f5927aff3470db4c00 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 16 Jan 2025 15:02:32 +0800 Subject: [PATCH 50/87] update README to adjust the structure --- README.md | 127 +++++++++++++++++++++++++++--------------------- tests/README.md | 41 +++++----------- 2 files changed, 84 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index a39ddc0f78..495c28c2c3 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,28 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine 1. [Introduction](#1-introduction) 1. [Documentation](#2-documentation) 1. [Prerequisites](#3-prerequisites) -1. [Building](#4-building) -1. [Packaging](#5-packaging) -1. [Installation](#6-installation) -1. [Running](#7-running) -1. [Testing](#8-testing) -1. [Releasing](#9-releasing) -1. [CI/CD](#10-cicd) -1. [Coverage](#11-coverage) -1. [Contributing](#12-contributing) + - [3.1 Prerequisites On Linux](#31-on-linux) + - [3.2 Prerequisites On macOS](#32-on-macos) + - [3.3 Prerequisites On Windows](#33-on-windows) + - [3.4 Clone the repo](#34-clone-the-repo) +2. [Building](#4-building) + - [4.1 Build on Linux](#41-build-on-linux) + - [4.2 Build on macOS](#42-build-on-macos) + - [4.3 Build On Windows](#43-build-on-windows) +3. [Packaging](#5-packaging) +4. [Installation](#6-installation) + - [6.1 Install on Linux](#61-install-on-linux) + - [6.2 Install on macOS](#62-install-on-macos) + - [6.3 Install on Windows](#63-install-on-windows) +5. [Running](#7-running) + - [7.1 Run TDengine on Linux](#71-run-tdengine-on-linux) + - [7.2 Run TDengine on macOS](#72-run-tdengine-on-macos) + - [7.3 Run TDengine on Windows](#73-run-tdengine-on-windows) +6. [Testing](#8-testing) +7. [Releasing](#9-releasing) +8. [CI/CD](#10-cicd) +9. [Coverage](#11-coverage) +10. [Contributing](#12-contributing) # 1. Introduction @@ -65,7 +78,11 @@ For user manual, system design and architecture, please refer to [TDengine Docum ## 3.1 On Linux -### For Ubuntu 18.04 or Later +
+ +Install required tools on Linux + +### For Ubuntu 18.04、20.04、22.04 ```bash sudo apt-get udpate @@ -82,36 +99,38 @@ yum config-manager --set-enabled powertools yum install -y zlib-static xz-devel snappy-devel jansson-devel pkgconfig libatomic-static libstdc++-static ``` -### For Fedora or Rocky Linux - -```bash -sudo dnf install -y gcc gcc-c++ make cmake git perl -sudo dnf install -y zlib-devel xz-devel snappy-devel jansson-devel pkgconfig -``` - -To build the [taosTools](https://github.com/taosdata/taos-tools) on Fedora or Rocky, the following packages need to be installed. - -```bash -sudo dnf install -y dnf-plugins-core -sudo dnf config-manager --set-enabled powertools -sudo dnf install -y zlib-devel zlib-static xz-devel snappy-devel jansson \ - jansson-devel pkgconfig libatomic libatomic-static libstdc++-static -``` +
## 3.2 On macOS +
+ +Install required tools on macOS + Please intall the dependencies with [brew](https://brew.sh/). ```bash brew install argp-standalone gflags pkgconfig ``` +
+ ## 3.3 On Windows +
+ +Install required tools on Windows + Work in Progress. +
+ ## 3.4 Clone the repo +
+ +Clone the repo + Clone the repository to the target machine: ```bash @@ -122,6 +141,8 @@ cd TDengine > [!NOTE] > TDengine Connectors can be found in following repositories: [JDBC Connector](https://github.com/taosdata/taos-connector-jdbc), [Go Connector](https://github.com/taosdata/driver-go), [Python Connector](https://github.com/taosdata/taos-connector-python), [Node.js Connector](https://github.com/taosdata/taos-connector-node), [C# Connector](https://github.com/taosdata/taos-connector-dotnet), [Rust Connector](https://github.com/taosdata/taos-connector-rust). +
+ # 4. Building At the moment, TDengine server supports running on Linux/Windows/MacOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service. TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. Right now we don't support build with cross-compiling environment. @@ -307,29 +328,7 @@ Option `-c test/cfg` specifies the system configuration file directory. -## 7.2 Run TDengine on Windows - -
- -Detailed steps to run on windows - -You can start TDengine server on Windows platform with below commands: - -```cmd -.\build\bin\taosd.exe -c test\cfg -``` - -In another terminal, use the TDengine CLI to connect the server: - -```cmd -.\build\bin\taos.exe -c test\cfg -``` - -option "-c test/cfg" specifies the system configuration file directory. - -
- -## 7.3 Run TDengine on macOS +## 7.2 Run TDengine on macOS
@@ -351,6 +350,29 @@ If TDengine CLI connects the server successfully, welcome messages and version i
+ +## 7.3 Run TDengine on Windows + +
+ +Detailed steps to run on windows + +You can start TDengine server on Windows platform with below commands: + +```cmd +.\build\bin\taosd.exe -c test\cfg +``` + +In another terminal, use the TDengine CLI to connect the server: + +```cmd +.\build\bin\taos.exe -c test\cfg +``` + +option "-c test/cfg" specifies the system configuration file directory. + +
+ # 8. Testing For how to run different types of tests on TDengine, please see [Testing TDengine](./tests/README.md). @@ -359,14 +381,9 @@ For how to run different types of tests on TDengine, please see [Testing TDengin For the complete list of TDengine Releases, please see [Releases](https://github.com/taosdata/TDengine/releases). -# 10. CI/CD +# 10. Workflow -Now, Jenkins is mainly used to build CI/CD pipeline for TDengine. To run the tests in the CI/CD pipeline, please run following commands: - -```bash -cd tests -./run_all_ci_cases.sh -b main # on main branch -``` +Now, Jenkins is mainly used to build CI/CD pipeline for TDengine. TDengine build check workflow can be found in this [Github Action](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml). diff --git a/tests/README.md b/tests/README.md index da70c9078d..3333ae2489 100644 --- a/tests/README.md +++ b/tests/README.md @@ -8,12 +8,15 @@ 1. [Smoke Test](#33-smoke-test) 1. [Legacy Test](#34-legacy-test) 1. [Chaos Test](#35-chaos-test) - 1. [TSBS Test](#36-tsbs-test) + 1. [CI Test](#36-ci-test) # 1. Introduction This manual is intended to provide users with comprehensive guidance to help them verify the TDengine function efficiently. The document is divided into three main sections: introduction, prerequisites and testing guide. +> [!NOTICE] +> The below commands and test scripts are verified on linux (Ubuntu 18.04、20.04、22.04) locally. + # 2. Prerequisites - Install Python3 @@ -126,39 +129,19 @@ cd tests/script [Placeholder] -## 3.6. TSBS Test +## 3.6 CI Test -[Time Series Benchmark Suite (TSBS)](https://github.com/timescale/tsbs) is an open-source performance benchmarking platform specifically designed for time-series data processing systems, such as databases. It provides a standardized approach to evaluating the performance of various databases by simulating typical use cases such as IoT and DevOps. +[Desciprtion] ### How to run tests? -1. Clone the code and run the tests locally on your machine. Ensure that your virtual machine supports the AVX instruction set: +To run the CI tests, please run following commands: + ```bash - cd /usr/local/src && git clone https://github.com/taosdata/tsbs-internal.git tsbs && \ - cd tsbs && git checkout enh/chr-td-33357 && \ - cd scripts/tsdbComp && ./testTsbs.sh +cd tests +./run_all_ci_cases.sh -b main # on main branch ``` -> [!NOTE] -> The default case type, CPU test, takes a long time to run. -> -> If you want to run quick test to verify the test environment, please set `caseType=cputest` in test.ini . +### How to add new cases? -2. When testing the client and server on separate machines, you should set up your environment as outlined in the steps below: - - 2.1. Modify IP and host of client and server in `test.ini` - ```ini - clientIP="192.168.0.203" # client ip - clientHost="trd03" # client hostname - serverIP="192.168.0.204" # server ip - serverHost="trd04" # server hostname - ``` - 2.2. Set up passwordless login between the client and server; otherwise, you'll need to configure the server password: - ```ini - serverPass="taosdata123" # server root password - ``` - 2.3. Run the following command to start the test: - ```bash - ./testTsbs.sh - ``` -3. When the test is done, the result can be found in `/data2/` directory, which can also be configured in `test.ini`. +[Placeholder] From 250f410bbbc5d3ec340fc98114007a369dc9b1aa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Jan 2025 15:08:26 +0800 Subject: [PATCH 51/87] enh: remove stt_trigger limitation of community version. --- docs/en/14-reference/03-taos-sql/02-database.md | 4 ---- docs/zh/14-reference/03-taos-sql/02-database.md | 4 ---- include/util/tdef.h | 10 +++++----- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/docs/en/14-reference/03-taos-sql/02-database.md b/docs/en/14-reference/03-taos-sql/02-database.md index 54548fe297..6a46620418 100644 --- a/docs/en/14-reference/03-taos-sql/02-database.md +++ b/docs/en/14-reference/03-taos-sql/02-database.md @@ -144,10 +144,6 @@ You can view cacheload through show \.vgroups; If cacheload is very close to cachesize, then cachesize may be too small. If cacheload is significantly less than cachesize, then cachesize is sufficient. You can decide whether to modify cachesize based on this principle. The specific modification value can be determined based on the available system memory, whether to double it or increase it several times. -4. stt_trigger - -Please stop database writing before modifying the stt_trigger parameter. - :::note Other parameters are not supported for modification in version 3.0.0.0 diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 32df6c60c1..4e64c1257b 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -146,10 +146,6 @@ alter_database_option: { 如果 cacheload 非常接近 cachesize，则 cachesize 可能过小。 如果 cacheload 明显小于 cachesize 则 cachesize 是够用的。可以根据这个原则判断是否需要修改 cachesize 。具体修改值可以根据系统可用内存情况来决定是加倍或者是提高几倍。 -4. stt_trigger - -在修改 stt_trigger 参数之前请先停止数据库写入。 - :::note 其它参数在 3.0.0.0 中暂不支持修改 diff --git a/include/util/tdef.h b/include/util/tdef.h index e7f40f1092..0cfc7ab591 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -456,13 +456,13 @@ typedef enum ELogicConditionType { #define TSDB_DB_SCHEMALESS_OFF 0 #define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF #define TSDB_MIN_STT_TRIGGER 1 -#ifdef TD_ENTERPRISE +// #ifdef TD_ENTERPRISE #define TSDB_MAX_STT_TRIGGER 16 #define TSDB_DEFAULT_SST_TRIGGER 2 -#else -#define TSDB_MAX_STT_TRIGGER 1 -#define TSDB_DEFAULT_SST_TRIGGER 1 -#endif +// #else +// #define TSDB_MAX_STT_TRIGGER 1 +// #define TSDB_DEFAULT_SST_TRIGGER 1 +// #endif #define TSDB_STT_TRIGGER_ARRAY_SIZE 16 // maximum of TSDB_MAX_STT_TRIGGER of TD_ENTERPRISE and TD_COMMUNITY #define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN) #define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2) From c7337fe4a34d5ac1d7237b0e47162562bfe35fe2 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 16 Jan 2025 15:10:47 +0800 Subject: [PATCH 52/87] test:add test type --- tests/run_all_ci_cases.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/run_all_ci_cases.sh b/tests/run_all_ci_cases.sh index 41040f3c43..43127f75e2 100755 --- a/tests/run_all_ci_cases.sh +++ b/tests/run_all_ci_cases.sh @@ -23,16 +23,20 @@ function printHelp() { echo " -b [Build test branch] Build test branch (default: null)" echo " Options: " echo " e.g., -b main (pull main branch, build and install)" + echo " -t [Run test cases] Run test cases type(default: all)" + echo " Options: " + echo " e.g., -t all/python/legacy" echo " -s [Save cases log] Save cases log(default: notsave)" echo " Options:" - echo " e.g., -c notsave : do not save the log " - echo " -c save : default save ci case log in Project dir/tests/ci_bak" + echo " e.g., -s notsave : do not save the log " + echo " -s save : default save ci case log in Project dir/tests/ci_bak" exit 0 } # Initialization parameter PROJECT_DIR="" BRANCH="" +TEST_TYPE="all" SAVE_LOG="notsave" # Parse command line parameters @@ -44,6 +48,9 @@ while getopts "hb:d:s:" arg; do b) BRANCH=$OPTARG ;; + t) + TEST_TYPE=$OPTARG + ;; s) SAVE_LOG=$OPTARG ;; @@ -315,9 +322,9 @@ function runTest() { [ -d sim ] && rm -rf sim [ -f $TDENGINE_ALLCI_REPORT ] && rm $TDENGINE_ALLCI_REPORT - runUnitTest runSimCases runPythonCases + runUnitTest stopTaosd cd $TDENGINE_DIR/tests/script @@ -361,7 +368,13 @@ print_color "$GREEN" "Run all ci test cases" | tee -a $WORK_DIR/date.log stopTaosd -runTest +if [ -z "$TEST_TYPE" ]; then + runTest +elif [ "$TEST_TYPE" = "python" -o "$TEST_TYPE" = "PYTHON"]; then + runPythonCases +elif [ "$TEST_TYPE" = "legacy" -o "$TEST_TYPE" = "LEGACY"]; then + runSimCases +fi date >> $WORK_DIR/date.log print_color "$GREEN" "End of ci test cases" | tee -a $WORK_DIR/date.log \ No newline at end of file From 130e8a3e068d3037d3f38b885e898e1584556a14 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 16 Jan 2025 15:14:29 +0800 Subject: [PATCH 53/87] test:add test type --- tests/run_all_ci_cases.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_all_ci_cases.sh b/tests/run_all_ci_cases.sh index 43127f75e2..1edc279e57 100755 --- a/tests/run_all_ci_cases.sh +++ b/tests/run_all_ci_cases.sh @@ -40,7 +40,7 @@ TEST_TYPE="all" SAVE_LOG="notsave" # Parse command line parameters -while getopts "hb:d:s:" arg; do +while getopts "hb:d:t:s:" arg; do case $arg in d) PROJECT_DIR=$OPTARG From a5d327df412987b5f56f724b0da229d1f451e279 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 16 Jan 2025 15:17:03 +0800 Subject: [PATCH 54/87] remove CICD cotents of TDengine README and update testing sructure of testing guide --- README.md | 2 -- tests/README.md | 96 +++++++++++++++++++++++++------------------------ 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 495c28c2c3..baf6097c30 100644 --- a/README.md +++ b/README.md @@ -383,8 +383,6 @@ For the complete list of TDengine Releases, please see [Releases](https://github # 10. Workflow -Now, Jenkins is mainly used to build CI/CD pipeline for TDengine. - TDengine build check workflow can be found in this [Github Action](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml). # 11. Coverage diff --git a/tests/README.md b/tests/README.md index 3333ae2489..dcec10eb7a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,18 +3,19 @@ 1. [Introduction](#1-introduction) 1. [Prerequisites](#2-prerequisites) 1. [Testing Guide](#3-testing-guide) - 1. [Unit Test](#31-unit-test) - 1. [System Test](#32-system-test) - 1. [Smoke Test](#33-smoke-test) - 1. [Legacy Test](#34-legacy-test) - 1. [Chaos Test](#35-chaos-test) - 1. [CI Test](#36-ci-test) + - [3.1 CI Test](#31-ci-test) + - [3.1.1 Unit Test](#311-unit-test) + - [3.1.2 System Test](#312-system-test) + - [3.1.3 Legacy Test](#313-legacy-test) + - [3.2 Smoke Test](#32-smoke-test) + - [3.3 Chaos Test](#33-chaos-test) + # 1. Introduction This manual is intended to provide users with comprehensive guidance to help them verify the TDengine function efficiently. The document is divided into three main sections: introduction, prerequisites and testing guide. -> [!NOTICE] +> [!NOTE] > The below commands and test scripts are verified on linux (Ubuntu 18.04、20.04、22.04) locally. # 2. Prerequisites @@ -53,25 +54,44 @@ make && make install In `tests` directory, there are different types of tests for TDengine. Below is a brief introduction about how to run them and how to add new cases. -## 3.1. Unit Test + +## 3.1 CI Test + +[Desciprtion] + +### How to run tests? + +To run the CI tests, please run following commands: + +```bash +cd tests +./run_all_ci_cases.sh -b main # on main branch +``` + +### How to add new cases? + +[Placeholder] + + +### 3.1.1 Unit Test Unit test script is the smallest testable part and developed for some function, method or class of TDengine. -### How to run tests? +#### How to run tests? ```bash cd tests/unit-test/ bash test.sh -e 0 ``` -### How to add new cases? +#### How to add new cases? Copy from the old version, need updates: You can add sim test case under tests/script, python test case under tests/system-test or tests/army. When the case passes in the test branch, add the case to the cases.task file under tests/parallel_test, and then merge the pr into main branch to run in the future CI. -## 3.2. System Test +## 3.1.2 System Test -Python test script includes almost all of the functions of TDengine, so some test case maybe fail cause the function only +Python test script includes all of the functions of TDengine OSS, so some test case maybe fail cause the function only work for TDengine Enterprise Edition. ### How to run tests? @@ -85,22 +105,7 @@ python3 ./test.py -f 2-query/floor.py [Placeholder] -## 3.3. Smoke Test - -Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine. - -### How to run tests? - -```bash -cd /root/TDengine/packaging/smokeTest -./test_smoking_selfhost.sh -``` - -### How to add new cases? - -[Placeholder] - -## 3.4. Legacy Test +## 3.1.3 Legacy Test In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++. @@ -117,31 +122,30 @@ cd tests/script [Placeholder] -## 3.5. Chaos Test -[Desciprtion] +## 3.2 Smoke Test + +Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine. ### How to run tests? -[Placeholder] - -### How to add new cases? - -[Placeholder] - -## 3.6 CI Test - -[Desciprtion] - -### How to run tests? - -To run the CI tests, please run following commands: - ```bash -cd tests -./run_all_ci_cases.sh -b main # on main branch +cd /root/TDengine/packaging/smokeTest +./test_smoking_selfhost.sh ``` ### How to add new cases? [Placeholder] + +## 3.3 Chaos Test + +[Desciprtion] + +### How to run tests? + +[Placeholder] + +### How to add new cases? + +[Placeholder] From 11d6768ba5af60bd736a356ee0f6be3e4f6f61c2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jan 2025 15:19:47 +0800 Subject: [PATCH 55/87] Update 01-taosd.md --- docs/zh/14-reference/01-components/01-taosd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 4c3350df7c..0b7189897c 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -189,7 +189,7 @@ charset 的有效值是 UTF-8。 |numOfQnodeQueryThreads | |支持动态修改 重启生效 |qnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| |numOfSnodeSharedThreads | |支持动态修改 重启生效 |snode 的共享线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| |numOfSnodeUniqueThreads | |支持动态修改 重启生效 |snode 的独占线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| -|ratioOfVnodeStreamThreads | |支持动态修改 重启生效 |流计算使用 vnode 线程的比例，取值范围 0.01-4，默认值 4| +|ratioOfVnodeStreamThreads | |支持动态修改 重启生效 |流计算使用 vnode 线程的比例，取值范围 0.01-4，默认值 0.5| |ttlUnit | |不支持动态修改 |ttl 参数的单位，取值范围 1-31572500，单位为秒，默认值 86400| |ttlPushInterval | |支持动态修改 立即生效 |ttl 检测超时频率，取值范围 1-100000，单位为秒，默认值 10| |ttlChangeOnWrite | |支持动态修改 立即生效 |ttl 到期时间是否伴随表的修改操作改变；0：不改变，1：改变；默认值为 0| From ac10ed5af5c317f4538516c74d14c9a53ee73819 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 16 Jan 2025 15:25:06 +0800 Subject: [PATCH 56/87] test:add test type --- tests/run_all_ci_cases.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run_all_ci_cases.sh b/tests/run_all_ci_cases.sh index 1edc279e57..b8e68b5907 100755 --- a/tests/run_all_ci_cases.sh +++ b/tests/run_all_ci_cases.sh @@ -370,9 +370,9 @@ stopTaosd if [ -z "$TEST_TYPE" ]; then runTest -elif [ "$TEST_TYPE" = "python" -o "$TEST_TYPE" = "PYTHON"]; then +elif [ "$TEST_TYPE" = "python" -o "$TEST_TYPE" = "PYTHON" ]; then runPythonCases -elif [ "$TEST_TYPE" = "legacy" -o "$TEST_TYPE" = "LEGACY"]; then +elif [ "$TEST_TYPE" = "legacy" -o "$TEST_TYPE" = "LEGACY" ]; then runSimCases fi From 297f7bb5d5b54bcab14614a6ef4bec6f432c78ba Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Thu, 16 Jan 2025 15:33:08 +0800 Subject: [PATCH 57/87] fix(query)[TS-5907]: skip some decompressTest sub-cases on machines without AVX2 support - Check CPU instruction set before running AVX2 tests - Skip tests if AVX2 is not supported to avoid errors --- source/util/test/CMakeLists.txt | 4 ---- source/util/test/decompressTest.cpp | 18 ++++++------------ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index ec05a4e415..768e465fea 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -142,10 +142,6 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -IF(COMPILER_SUPPORT_AVX2) - MESSAGE(STATUS "AVX2 instructions is ACTIVATED") - set_source_files_properties(decompressTest.cpp PROPERTIES COMPILE_FLAGS -mavx2) -ENDIF() add_executable(decompressTest "decompressTest.cpp") target_link_libraries(decompressTest os util common gtest_main) add_test( diff --git a/source/util/test/decompressTest.cpp b/source/util/test/decompressTest.cpp index e508c489df..b1f7f7e85c 100644 --- a/source/util/test/decompressTest.cpp +++ b/source/util/test/decompressTest.cpp @@ -524,23 +524,20 @@ static void decompressBasicTest(size_t dataSize, const CompF& compress, const De decltype(origData) decompData(origData.size()); // test simple implementation without SIMD instructions - tsSIMDEnable = 0; + tsAVX2Supported = 0; cnt = decompress(compData.data(), compData.size(), decompData.size(), decompData.data(), decompData.size(), ONE_STAGE_COMP, nullptr, 0); ASSERT_EQ(cnt, compData.size() - 1); EXPECT_EQ(origData, decompData); -#ifdef __AVX2__ - if (DataTypeSupportAvx::value) { + taosGetSystemInfo(); + if (DataTypeSupportAvx::value && tsAVX2Supported) { // test AVX2 implementation - tsSIMDEnable = 1; - tsAVX2Supported = 1; cnt = decompress(compData.data(), compData.size(), decompData.size(), decompData.data(), decompData.size(), ONE_STAGE_COMP, nullptr, 0); ASSERT_EQ(cnt, compData.size() - 1); EXPECT_EQ(origData, decompData); } -#endif } template @@ -557,7 +554,7 @@ static void decompressPerfTest(const char* typname, const CompF& compress, const << "; Compression ratio: " << 1.0 * (compData.size() - 1) / cnt << "\n"; decltype(origData) decompData(origData.size()); - tsSIMDEnable = 0; + tsAVX2Supported = 0; auto ms = measureRunTime( [&]() { decompress(compData.data(), compData.size(), decompData.size(), decompData.data(), decompData.size(), @@ -567,10 +564,8 @@ static void decompressPerfTest(const char* typname, const CompF& compress, const std::cout << "Decompression of " << NROUND * DATA_SIZE << " " << typname << " without SIMD costs " << ms << " ms, avg speed: " << NROUND * DATA_SIZE * 1000 / ms << " tuples/s\n"; -#ifdef __AVX2__ - if (DataTypeSupportAvx::value) { - tsSIMDEnable = 1; - tsAVX2Supported = 1; + taosGetSystemInfo(); + if (DataTypeSupportAvx::value && tsAVX2Supported) { ms = measureRunTime( [&]() { decompress(compData.data(), compData.size(), decompData.size(), decompData.data(), decompData.size(), @@ -580,7 +575,6 @@ static void decompressPerfTest(const char* typname, const CompF& compress, const std::cout << "Decompression of " << NROUND * DATA_SIZE << " " << typname << " using AVX2 costs " << ms << " ms, avg speed: " << NROUND * DATA_SIZE * 1000 / ms << " tuples/s\n"; } -#endif } #define RUN_PERF_TEST(typname, comp, decomp, min, max) \ From b2ac2d51c59e5bbbad8bbce040236231b4ced5de Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 16 Jan 2025 15:37:57 +0800 Subject: [PATCH 58/87] test:add test type --- tests/run_all_ci_cases.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run_all_ci_cases.sh b/tests/run_all_ci_cases.sh index b8e68b5907..1374fc1431 100755 --- a/tests/run_all_ci_cases.sh +++ b/tests/run_all_ci_cases.sh @@ -36,7 +36,7 @@ function printHelp() { # Initialization parameter PROJECT_DIR="" BRANCH="" -TEST_TYPE="all" +TEST_TYPE="" SAVE_LOG="notsave" # Parse command line parameters @@ -368,7 +368,7 @@ print_color "$GREEN" "Run all ci test cases" | tee -a $WORK_DIR/date.log stopTaosd -if [ -z "$TEST_TYPE" ]; then +if [ -z "$TEST_TYPE" -o "$TEST_TYPE" = "all" -o "$TEST_TYPE" = "ALL" ]; then runTest elif [ "$TEST_TYPE" = "python" -o "$TEST_TYPE" = "PYTHON" ]; then runPythonCases From d92870e6818749c3df90f30cbd8c5331334ccaa7 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 16 Jan 2025 16:01:17 +0800 Subject: [PATCH 59/87] test:add test type --- tests/README.md | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tests/README.md b/tests/README.md index dcec10eb7a..225bd2797d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -61,7 +61,7 @@ In `tests` directory, there are different types of tests for TDengine. Below is ### How to run tests? -To run the CI tests, please run following commands: +If this is the first time to run all the CI tests, it is recommended to add the test branch, please run like following commands: ```bash cd tests @@ -70,14 +70,21 @@ cd tests ### How to add new cases? -[Placeholder] +[You can add sim test case under tests/script, python test case under tests/system-test or tests/army. When the case passes in the test branch, add the case to the cases.task file under tests/parallel_test, and then merge the pr into main branch to run in the future CI.] ### 3.1.1 Unit Test Unit test script is the smallest testable part and developed for some function, method or class of TDengine. -#### How to run tests? +### How to run single test case? + +```bash +cd debug/build/bin +./osTimeTests +``` + +### How to run all unit test cases? ```bash cd tests/unit-test/ @@ -87,21 +94,28 @@ bash test.sh -e 0 #### How to add new cases? Copy from the old version, need updates: -You can add sim test case under tests/script, python test case under tests/system-test or tests/army. When the case passes in the test branch, add the case to the cases.task file under tests/parallel_test, and then merge the pr into main branch to run in the future CI. + ## 3.1.2 System Test Python test script includes all of the functions of TDengine OSS, so some test case maybe fail cause the function only work for TDengine Enterprise Edition. -### How to run tests? +### How to run single test case? ```bash cd tests/system-test python3 ./test.py -f 2-query/floor.py ``` -### How to add new cases? +### How to run all system test cases? + +```bash +cd tests +./run_all_ci_cases.sh -t python # all python cases +``` + +### How to add new case? [Placeholder] @@ -109,7 +123,7 @@ python3 ./test.py -f 2-query/floor.py In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++. -### How to run tests? +### How to run single test case? To run the legacy test cases, please execute the following commands: @@ -118,7 +132,14 @@ cd tests/script ./test.sh -f tsim/db/basic1.sim ``` -### How to add new cases? +### How to run all legacy test cases? + +```bash +cd tests +./run_all_ci_cases.sh -t legacy # all legacy cases +``` + +### How to add new case? [Placeholder] @@ -127,14 +148,14 @@ cd tests/script Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine. -### How to run tests? +### How to run test? ```bash cd /root/TDengine/packaging/smokeTest ./test_smoking_selfhost.sh ``` -### How to add new cases? +### How to add new case? [Placeholder] @@ -142,10 +163,10 @@ cd /root/TDengine/packaging/smokeTest [Desciprtion] -### How to run tests? +### How to run test? [Placeholder] -### How to add new cases? +### How to add new case? [Placeholder] From 9cd1113c93d06ce22d7d00c718bb7ef1db60dc71 Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 16 Jan 2025 16:11:49 +0800 Subject: [PATCH 60/87] update TDengine README workflow link --- README.md | 18 +++++++++--------- tests/README.md | 2 ++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index baf6097c30..91232136c5 100644 --- a/README.md +++ b/README.md @@ -33,24 +33,24 @@ English | [简体中文](README-CN.md) | [TDengine Cloud](https://cloud.tdengine - [3.2 Prerequisites On macOS](#32-on-macos) - [3.3 Prerequisites On Windows](#33-on-windows) - [3.4 Clone the repo](#34-clone-the-repo) -2. [Building](#4-building) +1. [Building](#4-building) - [4.1 Build on Linux](#41-build-on-linux) - [4.2 Build on macOS](#42-build-on-macos) - [4.3 Build On Windows](#43-build-on-windows) -3. [Packaging](#5-packaging) -4. [Installation](#6-installation) +1. [Packaging](#5-packaging) +1. [Installation](#6-installation) - [6.1 Install on Linux](#61-install-on-linux) - [6.2 Install on macOS](#62-install-on-macos) - [6.3 Install on Windows](#63-install-on-windows) -5. [Running](#7-running) +1. [Running](#7-running) - [7.1 Run TDengine on Linux](#71-run-tdengine-on-linux) - [7.2 Run TDengine on macOS](#72-run-tdengine-on-macos) - [7.3 Run TDengine on Windows](#73-run-tdengine-on-windows) -6. [Testing](#8-testing) -7. [Releasing](#9-releasing) -8. [CI/CD](#10-cicd) -9. [Coverage](#11-coverage) -10. [Contributing](#12-contributing) +1. [Testing](#8-testing) +1. [Releasing](#9-releasing) +1. [Workflow](#10-workflow) +1. [Coverage](#11-coverage) +1. [Contributing](#12-contributing) # 1. Introduction diff --git a/tests/README.md b/tests/README.md index 225bd2797d..e51a6a0e49 100644 --- a/tests/README.md +++ b/tests/README.md @@ -143,6 +143,8 @@ cd tests [Placeholder] +> [!NOTE] +> TSIM test framwork is replaced by system test currently, suggest to add new test scripts to system test. ## 3.2 Smoke Test From 008233ccf6e8efec22613ad84c31b496b1504c5a Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Thu, 16 Jan 2025 16:22:17 +0800 Subject: [PATCH 61/87] doc: add Chaos Test to README and update some functions --- tests/README.md | 12 ++++++--- tests/pytest/auto_crash_gen.py | 15 ++++++----- tests/pytest/auto_crash_gen_valgrind.py | 14 +++++------ .../pytest/auto_crash_gen_valgrind_cluster.py | 18 ++++++------- tests/system-test/test.py | 25 ++++++++++++------- 5 files changed, 47 insertions(+), 37 deletions(-) diff --git a/tests/README.md b/tests/README.md index e51a6a0e49..e990d622dc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -81,7 +81,7 @@ Unit test script is the smallest testable part and developed for some function, ```bash cd debug/build/bin -./osTimeTests +./osTimeTests ``` ### How to run all unit test cases? @@ -163,12 +163,16 @@ cd /root/TDengine/packaging/smokeTest ## 3.3 Chaos Test -[Desciprtion] +A simple tool to exercise various functions of the system in a randomized fashion, hoping to expose maximum number of problems, hopefully without a pre-determined scenario. ### How to run test? -[Placeholder] +```bash +cd tests/pytest +python3 auto_crash_gen.py +``` ### How to add new case? -[Placeholder] +Add a function, such as TaskCreateNewFunction, to pytest/crash_gen/crash_gen_main.py. +Integrate TaskCreateNewFunction into the balance_pickTaskType function in crash_gen_main.py. diff --git a/tests/pytest/auto_crash_gen.py b/tests/pytest/auto_crash_gen.py index 316f2ead0f..a35beb3395 100755 --- a/tests/pytest/auto_crash_gen.py +++ b/tests/pytest/auto_crash_gen.py @@ -244,7 +244,7 @@ def start_taosd(): else: pass - start_cmd = 'cd %s && python3 test.py >>/dev/null '%(start_path) + start_cmd = 'cd %s && python3 test.py -G >>/dev/null '%(start_path) os.system(start_cmd) def get_cmds(args_list): @@ -371,7 +371,7 @@ Result: {msg_dict[status]} Details Owner: Jayden Jia Start time: {starttime} -End time: {endtime} +End time: {endtime} Hostname: {hostname} Commit: {git_commit} Cmd: {cmd} @@ -380,14 +380,13 @@ Core dir: {core_dir} ''' text_result=text.split("Result: ")[1].split("Details")[0].strip() print(text_result) - if text_result == "success": - send_msg(notification_robot_url, get_msg(text)) + send_msg(notification_robot_url, get_msg(text)) else: - send_msg(alert_robot_url, get_msg(text)) - send_msg(notification_robot_url, get_msg(text)) - - #send_msg(get_msg(text)) + send_msg(alert_robot_url, get_msg(text)) + send_msg(notification_robot_url, get_msg(text)) + + #send_msg(get_msg(text)) except Exception as e: print("exception:", e) exit(status) diff --git a/tests/pytest/auto_crash_gen_valgrind.py b/tests/pytest/auto_crash_gen_valgrind.py index b7af68cd2f..0bd70ebf3f 100755 --- a/tests/pytest/auto_crash_gen_valgrind.py +++ b/tests/pytest/auto_crash_gen_valgrind.py @@ -245,7 +245,7 @@ def start_taosd(): else: pass - start_cmd = 'cd %s && python3 test.py '%(start_path) + start_cmd = 'cd %s && python3 test.py -G'%(start_path) os.system(start_cmd +">>/dev/null") def get_cmds(args_list): @@ -404,24 +404,24 @@ Result: {msg_dict[status]} Details Owner: Jayden Jia Start time: {starttime} -End time: {endtime} +End time: {endtime} Hostname: {hostname} Commit: {git_commit} Cmd: {cmd} Log dir: {log_dir} Core dir: {core_dir} ''' - + text_result=text.split("Result: ")[1].split("Details")[0].strip() print(text_result) - + if text_result == "success": send_msg(notification_robot_url, get_msg(text)) else: - send_msg(alert_robot_url, get_msg(text)) + send_msg(alert_robot_url, get_msg(text)) send_msg(notification_robot_url, get_msg(text)) - - #send_msg(get_msg(text)) + + #send_msg(get_msg(text)) except Exception as e: print("exception:", e) exit(status) diff --git a/tests/pytest/auto_crash_gen_valgrind_cluster.py b/tests/pytest/auto_crash_gen_valgrind_cluster.py index df40b60967..b4b90e1f5e 100755 --- a/tests/pytest/auto_crash_gen_valgrind_cluster.py +++ b/tests/pytest/auto_crash_gen_valgrind_cluster.py @@ -236,7 +236,7 @@ def start_taosd(): else: pass - start_cmd = 'cd %s && python3 test.py -N 4 -M 1 '%(start_path) + start_cmd = 'cd %s && python3 test.py -N 4 -M 1 -G '%(start_path) os.system(start_cmd +">>/dev/null") def get_cmds(args_list): @@ -388,28 +388,28 @@ def main(): text = f''' Result: {msg_dict[status]} - + Details Owner: Jayden Jia Start time: {starttime} -End time: {endtime} +End time: {endtime} Hostname: {hostname} Commit: {git_commit} Cmd: {cmd} Log dir: {log_dir} Core dir: {core_dir} ''' - + text_result=text.split("Result: ")[1].split("Details")[0].strip() print(text_result) - + if text_result == "success": send_msg(notification_robot_url, get_msg(text)) else: - send_msg(alert_robot_url, get_msg(text)) - send_msg(notification_robot_url, get_msg(text)) - - #send_msg(get_msg(text)) + send_msg(alert_robot_url, get_msg(text)) + send_msg(notification_robot_url, get_msg(text)) + + #send_msg(get_msg(text)) except Exception as e: print("exception:", e) exit(status) diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 0d40544be8..ab1bdc21d3 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -58,12 +58,12 @@ def checkRunTimeError(): if hwnd: os.system("TASKKILL /F /IM taosd.exe") -# +# # run case on previous cluster # def runOnPreviousCluster(host, config, fileName): print("enter run on previeous") - + # load case module sep = "/" if platform.system().lower() == 'windows': @@ -113,8 +113,9 @@ if __name__ == "__main__": asan = False independentMnode = False previousCluster = False - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) + crashGen = False + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP:G', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous',"crashGen"]) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -141,6 +142,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-a address sanitizer mode') tdLog.printNoPrefix('-P run case with [P]revious cluster, do not create new cluster to run case.') + tdLog.printNoPrefix('-G crashGen mode') sys.exit(0) @@ -208,7 +210,7 @@ if __name__ == "__main__": if key in ['-R', '--restful']: restful = True - + if key in ['-W', '--websocket']: websocket = True @@ -228,6 +230,10 @@ if __name__ == "__main__": if key in ['-P', '--previous']: previousCluster = True + if key in ['-G', '--crashGen']: + crashGen = True + + # # do exeCmd command # @@ -405,7 +411,7 @@ if __name__ == "__main__": for dnode in tdDnodes.dnodes: tdDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) - + if restful or websocket: tAdapter.deploy(adapter_cfg_dict) tAdapter.start() @@ -450,7 +456,7 @@ if __name__ == "__main__": else: tdLog.debug(res) tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") - + if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True: conn = None else: @@ -640,7 +646,7 @@ if __name__ == "__main__": else: tdLog.debug(res) tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") - + # run case if testCluster: @@ -692,6 +698,7 @@ if __name__ == "__main__": # tdDnodes.StopAllSigint() tdLog.info("Address sanitizer mode finished") else: - tdDnodes.stopAll() + if not crashGen: + tdDnodes.stopAll() tdLog.info("stop all td process finished") sys.exit(0) From 376a30f20de579a29264587a2fc9c5f5d4f3ebdd Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 16 Jan 2025 17:00:39 +0800 Subject: [PATCH 62/87] update testing README by charles --- tests/README.md | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/tests/README.md b/tests/README.md index e990d622dc..78edcac6ac 100644 --- a/tests/README.md +++ b/tests/README.md @@ -57,11 +57,11 @@ In `tests` directory, there are different types of tests for TDengine. Below is ## 3.1 CI Test -[Desciprtion] +CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability. TDengine CI testing includes three part of test cases: unit test、system test and legacy test -### How to run tests? +### How to run all CI test cases? -If this is the first time to run all the CI tests, it is recommended to add the test branch, please run like following commands: +If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands: ```bash cd tests @@ -70,8 +70,7 @@ cd tests ### How to add new cases? -[You can add sim test case under tests/script, python test case under tests/system-test or tests/army. When the case passes in the test branch, add the case to the cases.task file under tests/parallel_test, and then merge the pr into main branch to run in the future CI.] - +You can refer the below child sections to add new test cases for CI test. ### 3.1.1 Unit Test @@ -93,8 +92,27 @@ bash test.sh -e 0 #### How to add new cases? -Copy from the old version, need updates: +The Google test framwork is used for unit testing to specific function module, you can refer below steps to add one test case: +##### Create test case file and develop the test scripts + +In the test directory corresponding to the target function module, create test files in CPP format and write corresponding test cases. + +##### Update build configuration + +Modify the CMakeLists.txt file in this directory to ensure that the new test files are properly included in the compilation process. See the source\os\test\CMakeLists.txt file for configuration examples. + +##### Compile test code + +In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., cmake .. -DBUILD_TEST=1) to generate a compilation file, and then run a compilation command (e.g. make) to complete the compilation of the test code. + +##### Execute the test program + +Find the executable file in the compiled directory(e.g. TDengine/debug/build/bin/) and run it. + +##### Integrate into CI tests + +Use the add_test command to add new compiled test cases into CI test collection, ensure that the new added test cases can be run for every build. ## 3.1.2 System Test @@ -141,14 +159,12 @@ cd tests ### How to add new case? -[Placeholder] - > [!NOTE] -> TSIM test framwork is replaced by system test currently, suggest to add new test scripts to system test. +> TSIM test framwork is replaced by system test currently, suggest to add new test scripts to system test, you can refer [System Test](#312-system-test) for detail steps. ## 3.2 Smoke Test -Smoke test script is known as sanity testing to ensure that the critical functionalities of TDengine. +Smoke test script is from system test and known as sanity testing to ensure that the critical functionalities of TDengine. ### How to run test? @@ -163,7 +179,7 @@ cd /root/TDengine/packaging/smokeTest ## 3.3 Chaos Test -A simple tool to exercise various functions of the system in a randomized fashion, hoping to expose maximum number of problems, hopefully without a pre-determined scenario. +A simple tool to exercise various functions of the system in a randomized fashion, hoping to expose maximum number of problems without a pre-determined scenario. ### How to run test? From b63842177cf89269ced40229d6449625d6022fa3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Jan 2025 17:13:49 +0800 Subject: [PATCH 63/87] doc: update stt_trigger --- docs/en/14-reference/03-taos-sql/02-database.md | 2 +- docs/zh/14-reference/03-taos-sql/02-database.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-taos-sql/02-database.md b/docs/en/14-reference/03-taos-sql/02-database.md index 6a46620418..c6865fd162 100644 --- a/docs/en/14-reference/03-taos-sql/02-database.md +++ b/docs/en/14-reference/03-taos-sql/02-database.md @@ -65,7 +65,7 @@ database_option: { - MINROWS: The minimum number of records in a file block, default is 100. - KEEP: Indicates the number of days data files are kept, default value is 3650, range [1, 365000], and must be greater than or equal to 3 times the DURATION parameter value. The database will automatically delete data that has been saved for longer than the KEEP value to free up storage space. KEEP can use unit-specified formats, such as KEEP 100h, KEEP 10d, etc., supports m (minutes), h (hours), and d (days) three units. It can also be written without a unit, like KEEP 50, where the default unit is days. The enterprise version supports multi-tier storage feature, thus, multiple retention times can be set (multiple separated by commas, up to 3, satisfying keep 0 \<= keep 1 \<= keep 2, such as KEEP 100h,100d,3650d); the community version does not support multi-tier storage feature (even if multiple retention times are configured, it will not take effect, KEEP will take the longest retention time). - KEEP_TIME_OFFSET: Effective from version 3.2.0.0. The delay execution time for deleting or migrating data that has been saved for longer than the KEEP value, default value is 0 (hours). After the data file's save time exceeds KEEP, the deletion or migration operation will not be executed immediately, but will wait an additional interval specified by this parameter, to avoid peak business periods. -- STT_TRIGGER: Indicates the number of file merges triggered by disk files. The open-source version is fixed at 1, the enterprise version can be set from 1 to 16. For scenarios with few tables and high-frequency writing, this parameter is recommended to use the default configuration; for scenarios with many tables and low-frequency writing, this parameter is recommended to be set to a larger value. +- STT_TRIGGER: Indicates the number of file merges triggered by disk files. For scenarios with few tables and high-frequency writing, this parameter is recommended to use the default configuration; for scenarios with many tables and low-frequency writing, this parameter is recommended to be set to a larger value. - SINGLE_STABLE: Indicates whether only one supertable can be created in this database, used in cases where the supertable has a very large number of columns. - 0: Indicates that multiple supertables can be created. - 1: Indicates that only one supertable can be created. diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 4e64c1257b..9f60b51efd 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -67,7 +67,7 @@ database_option: { - KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 3 倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](../../operation/planning/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。了解更多，请点击 [关于主键时间戳](https://docs.taosdata.com/reference/taos-sql/insert/) - KEEP_TIME_OFFSET：自 3.2.0.0 版本生效。删除或迁移保存时间超过 KEEP 值的数据的延迟执行时间，默认值为 0 (小时)。在数据文件保存时间超过 KEEP 后，删除或迁移操作不会立即执行，而会额外等待本参数指定的时间间隔，以实现与业务高峰期错开的目的。 -- STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。 +- STT_TRIGGER：表示落盘文件触发文件合并的个数。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 - 0：表示可以创建多张超级表。 - 1：表示只可以创建一张超级表。 From b72e6189dac55857bf2d87d0d27a0f8e260ddf9c Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 16 Jan 2025 17:19:06 +0800 Subject: [PATCH 64/87] udpate smoke test part of testing README by charles --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 78edcac6ac..ff42c9bb3d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -175,7 +175,7 @@ cd /root/TDengine/packaging/smokeTest ### How to add new case? -[Placeholder] +You can update python commands part of test_smoking_selfhost.sh file to add any system test case into smoke test. ## 3.3 Chaos Test From ce384e325a9ed7f64e5e8dd3af78b509fa251935 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Jan 2025 17:50:54 +0800 Subject: [PATCH 65/87] ci:modify test readme --- tests/README.md | 45 +++++++++++++-- .../0-others/test_case_template.py | 55 +++++++++++++++++++ 2 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 tests/system-test/0-others/test_case_template.py diff --git a/tests/README.md b/tests/README.md index ff42c9bb3d..f684c8977c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -92,28 +92,34 @@ bash test.sh -e 0 #### How to add new cases? +
+ +Detailed steps to add new unit test case + The Google test framwork is used for unit testing to specific function module, you can refer below steps to add one test case: -##### Create test case file and develop the test scripts +##### 1. Create test case file and develop the test scripts In the test directory corresponding to the target function module, create test files in CPP format and write corresponding test cases. -##### Update build configuration +##### 2. Update build configuration Modify the CMakeLists.txt file in this directory to ensure that the new test files are properly included in the compilation process. See the source\os\test\CMakeLists.txt file for configuration examples. -##### Compile test code +##### 3. Compile test code In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., cmake .. -DBUILD_TEST=1) to generate a compilation file, and then run a compilation command (e.g. make) to complete the compilation of the test code. -##### Execute the test program +##### 4. Execute the test program Find the executable file in the compiled directory(e.g. TDengine/debug/build/bin/) and run it. -##### Integrate into CI tests +##### 5. Integrate into CI tests Use the add_test command to add new compiled test cases into CI test collection, ensure that the new added test cases can be run for every build. +
+ ## 3.1.2 System Test Python test script includes all of the functions of TDengine OSS, so some test case maybe fail cause the function only @@ -135,7 +141,34 @@ cd tests ### How to add new case? -[Placeholder] +
+ +Detailed steps to add new system test case + +you can refer below steps to add one test case: + +##### 1. Create test case file and develop the test scripts + +In the test directory corresponding to the target function module, create test files in CPP format and write corresponding test cases. + +##### 2. Update build configuration + +Modify the CMakeLists.txt file in this directory to ensure that the new test files are properly included in the compilation process. See the source\os\test\CMakeLists.txt file for configuration examples. + +##### 3. Compile test code + +In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., cmake .. -DBUILD_TEST=1) to generate a compilation file, and then run a compilation command (e.g. make) to complete the compilation of the test code. + +##### 4. Execute the test program + +Find the executable file in the compiled directory(e.g. TDengine/debug/build/bin/) and run it. + +##### 5. Integrate into CI tests + +Use the add_test command to add new compiled test cases into CI test collection, ensure that the new added test cases can be run for every build. + +
+ ## 3.1.3 Legacy Test diff --git a/tests/system-test/0-others/test_case_template.py b/tests/system-test/0-others/test_case_template.py new file mode 100644 index 0000000000..53acbade66 --- /dev/null +++ b/tests/system-test/0-others/test_case_template.py @@ -0,0 +1,55 @@ + +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from util.dnodes import * +from util.common import * + + +class TDTestCase: + + """ + Here is the class description for the whole file cases + """ + + # add the configuration of the client and server here + clientCfgDict = {'debugFlag': 131} + updatecfgDict = { + "debugFlag" : "131", + "queryBufferSize" : 10240, + 'clientCfg' : clientCfgDict + } + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.replicaVar = int(replicaVar) + + + def test_function(): # case function should be named start with test_ + """ + Here is the function description for single test: + Test case for custom function + """ + tdLog.info(f"Test case test custom function") + # excute the sql + tdLog.execute(f"create database db_test_function") + tdSql.execute(f"create table db_test_function.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned);") + # qury the result and return the result + tdSql.query(f"show databases") + # print result and check the result + database_info = tdLog.info(f"{tdSql.queryResult}") + tdSql.checkRows(3) + tdSql.checkData(2,0,"db_test_function") + + + def run(self): + self.test_5dnode_3mnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='vnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 24db39d4ce4a938baf8f5dce3b9ef427aa502fde Mon Sep 17 00:00:00 2001 From: Feng Chao Date: Thu, 16 Jan 2025 18:08:07 +0800 Subject: [PATCH 66/87] update testing guide structure by charles --- tests/README.md | 61 +++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/tests/README.md b/tests/README.md index f684c8977c..4349e4bc65 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,13 +3,12 @@ 1. [Introduction](#1-introduction) 1. [Prerequisites](#2-prerequisites) 1. [Testing Guide](#3-testing-guide) - - [3.1 CI Test](#31-ci-test) - - [3.1.1 Unit Test](#311-unit-test) - - [3.1.2 System Test](#312-system-test) - - [3.1.3 Legacy Test](#313-legacy-test) - - [3.2 Smoke Test](#32-smoke-test) - - [3.3 Chaos Test](#33-chaos-test) - + - [3.1 Unit Test](#31-unit-test) + - [3.2 System Test](#32-system-test) + - [3.3 Legacy Test](#33-legacy-test) + - [3.4 Smoke Test](#34-smoke-test) + - [3.5 Chaos Test](#35-chaos-test) + - [3.6 CI Test](#36-ci-test) # 1. Introduction @@ -54,25 +53,7 @@ make && make install In `tests` directory, there are different types of tests for TDengine. Below is a brief introduction about how to run them and how to add new cases. - -## 3.1 CI Test - -CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability. TDengine CI testing includes three part of test cases: unit test、system test and legacy test - -### How to run all CI test cases? - -If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands: - -```bash -cd tests -./run_all_ci_cases.sh -b main # on main branch -``` - -### How to add new cases? - -You can refer the below child sections to add new test cases for CI test. - -### 3.1.1 Unit Test +### 3.1 Unit Test Unit test script is the smallest testable part and developed for some function, method or class of TDengine. @@ -90,7 +71,7 @@ cd tests/unit-test/ bash test.sh -e 0 ``` -#### How to add new cases? +#### How to add new cases? {#test-id}
@@ -120,7 +101,7 @@ Use the add_test command to add new compiled test cases into CI test collection,
-## 3.1.2 System Test +## 3.2 System Test Python test script includes all of the functions of TDengine OSS, so some test case maybe fail cause the function only work for TDengine Enterprise Edition. @@ -169,8 +150,7 @@ Use the add_test command to add new compiled test cases into CI test collection, - -## 3.1.3 Legacy Test +## 3.3 Legacy Test In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++. @@ -195,7 +175,7 @@ cd tests > [!NOTE] > TSIM test framwork is replaced by system test currently, suggest to add new test scripts to system test, you can refer [System Test](#312-system-test) for detail steps. -## 3.2 Smoke Test +## 3.4 Smoke Test Smoke test script is from system test and known as sanity testing to ensure that the critical functionalities of TDengine. @@ -210,7 +190,7 @@ cd /root/TDengine/packaging/smokeTest You can update python commands part of test_smoking_selfhost.sh file to add any system test case into smoke test. -## 3.3 Chaos Test +## 3.5 Chaos Test A simple tool to exercise various functions of the system in a randomized fashion, hoping to expose maximum number of problems without a pre-determined scenario. @@ -225,3 +205,20 @@ python3 auto_crash_gen.py Add a function, such as TaskCreateNewFunction, to pytest/crash_gen/crash_gen_main.py. Integrate TaskCreateNewFunction into the balance_pickTaskType function in crash_gen_main.py. + +## 3.6 CI Test + +CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability. TDengine CI testing includes three part of test cases: unit test、system test and legacy test + +### How to run all CI test cases? + +If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands: + +```bash +cd tests +./run_all_ci_cases.sh -b main # on main branch +``` + +### How to add new cases? + +You can refer the [Unit Test](#31-unit-test)、[System Test](#32-system-test) and [Legacy Test](#33-legacy-test) sections for detail steps to add new test cases for CI test. From 59ea64e1a8623ff6c54b7f9e45fb2241bf0190d9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Jan 2025 18:19:19 +0800 Subject: [PATCH 67/87] ci:modify test readme --- tests/README.md | 23 +++++++++---------- .../0-others/test_case_template.py | 6 ++--- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/README.md b/tests/README.md index f684c8977c..9b903c2cb5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -145,27 +145,26 @@ cd tests Detailed steps to add new system test case -you can refer below steps to add one test case: +test.py is the test case execution and monitoring of the entry program. you can refer below steps to add one test case: -##### 1. Create test case file and develop the test scripts +##### 1.Create a test case file and develop the test cases -In the test directory corresponding to the target function module, create test files in CPP format and write corresponding test cases. +Create a file in `tests/system-test` containing each functional directory and refer to the use case template `tests/system-test/0-others/test_case_template.py` to add a new test case. -##### 2. Update build configuration +##### 2.Execute the test case -Modify the CMakeLists.txt file in this directory to ensure that the new test files are properly included in the compilation process. See the source\os\test\CMakeLists.txt file for configuration examples. +cd tests/system-test & python3 ./test.py -f 0-others/test_case_template.py -##### 3. Compile test code +##### 3.Integrate into CI tests -In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., cmake .. -DBUILD_TEST=1) to generate a compilation file, and then run a compilation command (e.g. make) to complete the compilation of the test code. +Edit `tests/parallel_test/cases.task` and add the testcase path and executions in the specified format. The third column indicates whether to use Address Sanitizer mode for testing. -##### 4. Execute the test program -Find the executable file in the compiled directory(e.g. TDengine/debug/build/bin/) and run it. -##### 5. Integrate into CI tests - -Use the add_test command to add new compiled test cases into CI test collection, ensure that the new added test cases can be run for every build. +```bash +#caseID,rerunTimes,Run with Sanitizer,casePath,caseCommand +,,n,system-test, python3 ./test.py -f 0-others/test_case_template.py +``` diff --git a/tests/system-test/0-others/test_case_template.py b/tests/system-test/0-others/test_case_template.py index 53acbade66..fa1a9b5ade 100644 --- a/tests/system-test/0-others/test_case_template.py +++ b/tests/system-test/0-others/test_case_template.py @@ -27,14 +27,14 @@ class TDTestCase: self.replicaVar = int(replicaVar) - def test_function(): # case function should be named start with test_ + def test_function(self): # case function should be named start with test_ """ Here is the function description for single test: Test case for custom function """ tdLog.info(f"Test case test custom function") # excute the sql - tdLog.execute(f"create database db_test_function") + tdSql.execute(f"create database db_test_function") tdSql.execute(f"create table db_test_function.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned);") # qury the result and return the result tdSql.query(f"show databases") @@ -45,7 +45,7 @@ class TDTestCase: def run(self): - self.test_5dnode_3mnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='vnode') + self.test_function() def stop(self): tdSql.close() From 31e69ea2b2c8ac2db72563f8aaf332c0a80cb69c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Jan 2025 18:25:29 +0800 Subject: [PATCH 68/87] ci:modify test readme --- tests/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 9b903c2cb5..fac0babe24 100644 --- a/tests/README.md +++ b/tests/README.md @@ -145,7 +145,8 @@ cd tests Detailed steps to add new system test case -test.py is the test case execution and monitoring of the entry program. you can refer below steps to add one test case: +The Python test framework is developed by TDengine teams, and test.py is the test case execution and monitoring of the entry program, Use `python3 ./test.py -h` to view more features. +you can refer below steps to add one test case: ##### 1.Create a test case file and develop the test cases From d155025d9d5067c79de7cb844bb99f5ae3118a1c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Jan 2025 18:32:07 +0800 Subject: [PATCH 69/87] ci:modify test readme --- tests/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/README.md b/tests/README.md index fac0babe24..3bc57d67e3 100644 --- a/tests/README.md +++ b/tests/README.md @@ -154,7 +154,10 @@ Create a file in `tests/system-test` containing each functional directory and re ##### 2.Execute the test case +Ensure the test case execution is successful. +```bash cd tests/system-test & python3 ./test.py -f 0-others/test_case_template.py +``` ##### 3.Integrate into CI tests From 20f911d009ab106487d8b848d9f1fd2edcffae7e Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 16 Jan 2025 18:52:43 +0800 Subject: [PATCH 70/87] test:add note --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 91232136c5..8ceab44b3f 100644 --- a/README.md +++ b/README.md @@ -387,7 +387,10 @@ TDengine build check workflow can be found in this [Github Action](https://githu # 11. Coverage -Latest TDengine test coverage report can be found on [coveralls.io](https://coveralls.io/github/taosdata/TDengine). To create the test coverage report (in HTML format) locally, please run following commands: +
+ +Latest TDengine test coverage report can be found on [coveralls.io](https://coveralls.io/github/taosdata/TDengine). +To create the test coverage report (in HTML format) locally, please run following commands: ```bash cd tests @@ -395,9 +398,12 @@ bash setup-lcov.sh -v 1.16 && ./run_local_coverage.sh -b main -c task # on main branch and run cases in longtimeruning_cases.task # for more infomation about options please refer to ./run_local_coverage.sh -h ``` -> [!NOTE] +> [**NOTE**] > Please note that the -b and -i options will recompile TDengine with the -DCOVER=true option, which may take a amount of time. +
+ + # 12. Contributing Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to TDengine. From 469cbcbf332b972433f544d8340b8a8c2b16160d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Jan 2025 18:56:49 +0800 Subject: [PATCH 71/87] ci:modify test readme --- README.md | 5 ++-- tests/README.md | 73 +++++++++++++++++++++++-------------------------- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 91232136c5..1111fc150f 100644 --- a/README.md +++ b/README.md @@ -138,8 +138,7 @@ git clone https://github.com/taosdata/TDengine.git cd TDengine ``` -> [!NOTE] -> TDengine Connectors can be found in following repositories: [JDBC Connector](https://github.com/taosdata/taos-connector-jdbc), [Go Connector](https://github.com/taosdata/driver-go), [Python Connector](https://github.com/taosdata/taos-connector-python), [Node.js Connector](https://github.com/taosdata/taos-connector-node), [C# Connector](https://github.com/taosdata/taos-connector-dotnet), [Rust Connector](https://github.com/taosdata/taos-connector-rust). +> NOTE: TDengine Connectors can be found in following repositories: [JDBC Connector](https://github.com/taosdata/taos-connector-jdbc), [Go Connector](https://github.com/taosdata/driver-go), [Python Connector](https://github.com/taosdata/taos-connector-python), [Node.js Connector](https://github.com/taosdata/taos-connector-node), [C# Connector](https://github.com/taosdata/taos-connector-dotnet), [Rust Connector](https://github.com/taosdata/taos-connector-rust). @@ -147,7 +146,7 @@ cd TDengine At the moment, TDengine server supports running on Linux/Windows/MacOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service. TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. Right now we don't support build with cross-compiling environment. -You can choose to install through source code, [container](https://docs.tdengine.com/get-started/docker/), [installation package](https://docs.tdengine.com/get-started/package/) or [Kubernetes](https://docs.tdengine.com/deployment/k8s/). This quick guide only applies to install from source. +You can choose to install through source code, [container](https://docs.tdengine.com/get-started/deploy-in-docker/), [installation package](https://docs.tdengine.com/get-started/deploy-from-package/) or [Kubernetes](https://docs.tdengine.com/operations-and-maintenance/deploy-your-cluster/#kubernetes-deployment). This quick guide only applies to install from source. TDengine provide a few useful tools such as taosBenchmark (was named taosdemo) and taosdump. They were part of TDengine. By default, TDengine compiling does not include taosTools. You can use `cmake .. -DBUILD_TOOLS=true` to make them be compiled with TDengine. diff --git a/tests/README.md b/tests/README.md index 3bc57d67e3..aef27c7a6e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,13 +3,12 @@ 1. [Introduction](#1-introduction) 1. [Prerequisites](#2-prerequisites) 1. [Testing Guide](#3-testing-guide) - - [3.1 CI Test](#31-ci-test) - - [3.1.1 Unit Test](#311-unit-test) - - [3.1.2 System Test](#312-system-test) - - [3.1.3 Legacy Test](#313-legacy-test) - - [3.2 Smoke Test](#32-smoke-test) - - [3.3 Chaos Test](#33-chaos-test) - + - [3.1 Unit Test](#31-unit-test) + - [3.2 System Test](#32-system-test) + - [3.3 Legacy Test](#33-legacy-test) + - [3.4 Smoke Test](#34-smoke-test) + - [3.5 Chaos Test](#35-chaos-test) + - [3.6 CI Test](#36-ci-test) # 1. Introduction @@ -54,25 +53,7 @@ make && make install In `tests` directory, there are different types of tests for TDengine. Below is a brief introduction about how to run them and how to add new cases. - -## 3.1 CI Test - -CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability. TDengine CI testing includes three part of test cases: unit test、system test and legacy test - -### How to run all CI test cases? - -If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands: - -```bash -cd tests -./run_all_ci_cases.sh -b main # on main branch -``` - -### How to add new cases? - -You can refer the below child sections to add new test cases for CI test. - -### 3.1.1 Unit Test +### 3.1 Unit Test Unit test script is the smallest testable part and developed for some function, method or class of TDengine. @@ -90,7 +71,7 @@ cd tests/unit-test/ bash test.sh -e 0 ``` -#### How to add new cases? +#### How to add new cases?
@@ -104,15 +85,15 @@ In the test directory corresponding to the target function module, create test f ##### 2. Update build configuration -Modify the CMakeLists.txt file in this directory to ensure that the new test files are properly included in the compilation process. See the source\os\test\CMakeLists.txt file for configuration examples. +Modify the CMakeLists.txt file in this directory to ensure that the new test files are properly included in the compilation process. See the `source/os/test/CMakeLists.txt` file for configuration examples. ##### 3. Compile test code -In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., cmake .. -DBUILD_TEST=1) to generate a compilation file, and then run a compilation command (e.g. make) to complete the compilation of the test code. +In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., `cmake .. -DBUILD_TEST=1`) to generate a compilation file, and then run a compilation command (e.g. make) to complete the compilation of the test code. ##### 4. Execute the test program -Find the executable file in the compiled directory(e.g. TDengine/debug/build/bin/) and run it. +Find the executable file in the compiled directory(e.g. `TDengine/debug/build/bin/`) and run it. ##### 5. Integrate into CI tests @@ -120,7 +101,7 @@ Use the add_test command to add new compiled test cases into CI test collection,
-## 3.1.2 System Test +## 3.2 System Test Python test script includes all of the functions of TDengine OSS, so some test case maybe fail cause the function only work for TDengine Enterprise Edition. @@ -154,10 +135,7 @@ Create a file in `tests/system-test` containing each functional directory and re ##### 2.Execute the test case -Ensure the test case execution is successful. -```bash cd tests/system-test & python3 ./test.py -f 0-others/test_case_template.py -``` ##### 3.Integrate into CI tests @@ -172,8 +150,7 @@ Edit `tests/parallel_test/cases.task` and add the testcase path and executions i - -## 3.1.3 Legacy Test +## 3.3 Legacy Test In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++. @@ -195,10 +172,10 @@ cd tests ### How to add new case? -> [!NOTE] +> [!NOTE] > TSIM test framwork is replaced by system test currently, suggest to add new test scripts to system test, you can refer [System Test](#312-system-test) for detail steps. -## 3.2 Smoke Test +## 3.4 Smoke Test Smoke test script is from system test and known as sanity testing to ensure that the critical functionalities of TDengine. @@ -213,7 +190,7 @@ cd /root/TDengine/packaging/smokeTest You can update python commands part of test_smoking_selfhost.sh file to add any system test case into smoke test. -## 3.3 Chaos Test +## 3.5 Chaos Test A simple tool to exercise various functions of the system in a randomized fashion, hoping to expose maximum number of problems without a pre-determined scenario. @@ -227,4 +204,22 @@ python3 auto_crash_gen.py ### How to add new case? Add a function, such as TaskCreateNewFunction, to pytest/crash_gen/crash_gen_main.py. + Integrate TaskCreateNewFunction into the balance_pickTaskType function in crash_gen_main.py. + +## 3.6 CI Test + +CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability. TDengine CI testing includes three part of test cases: unit test、system test and legacy test + +### How to run all CI test cases? + +If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands: + +```bash +cd tests +./run_all_ci_cases.sh -b main # on main branch +``` + +### How to add new cases? + +You can refer the [Unit Test](#31-unit-test)、[System Test](#32-system-test) and [Legacy Test](#33-legacy-test) sections for detail steps to add new test cases for CI test. From 51f989833bba2ae7495d24ab4c4e3cbdd2da1fae Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 16 Jan 2025 18:59:37 +0800 Subject: [PATCH 72/87] test:add note --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8ceab44b3f..872842423c 100644 --- a/README.md +++ b/README.md @@ -398,7 +398,7 @@ bash setup-lcov.sh -v 1.16 && ./run_local_coverage.sh -b main -c task # on main branch and run cases in longtimeruning_cases.task # for more infomation about options please refer to ./run_local_coverage.sh -h ``` -> [**NOTE**] +> **NOTE:** > Please note that the -b and -i options will recompile TDengine with the -DCOVER=true option, which may take a amount of time. From 687fa1d5c272d4959bb62f4afb4ebd809bc82f85 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Thu, 16 Jan 2025 19:01:09 +0800 Subject: [PATCH 73/87] test:add note --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 872842423c..8ef4f1e887 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ git clone https://github.com/taosdata/TDengine.git cd TDengine ``` -> [!NOTE] +> **NOTE:** > TDengine Connectors can be found in following repositories: [JDBC Connector](https://github.com/taosdata/taos-connector-jdbc), [Go Connector](https://github.com/taosdata/driver-go), [Python Connector](https://github.com/taosdata/taos-connector-python), [Node.js Connector](https://github.com/taosdata/taos-connector-node), [C# Connector](https://github.com/taosdata/taos-connector-dotnet), [Rust Connector](https://github.com/taosdata/taos-connector-rust). From 4f0335eb9976d5b88bbf515b7f75f924f22217ad Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Jan 2025 19:03:14 +0800 Subject: [PATCH 74/87] ci:modify test readme --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index aef27c7a6e..b9fa40f93b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -173,7 +173,7 @@ cd tests ### How to add new case? > [!NOTE] -> TSIM test framwork is replaced by system test currently, suggest to add new test scripts to system test, you can refer [System Test](#312-system-test) for detail steps. +> TSIM test framwork is replaced by system test currently, suggest to add new test scripts to system test, you can refer [System Test](#32-system-test) for detail steps. ## 3.4 Smoke Test From 0af9f1005435f003606914d62e560e0836a370ba Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Jan 2025 19:10:10 +0800 Subject: [PATCH 75/87] Merge latest changes from origin/docs/chr1-readme --- tests/README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/README.md b/tests/README.md index b9fa40f93b..bc97d74afd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -12,7 +12,9 @@ # 1. Introduction -This manual is intended to provide users with comprehensive guidance to help them verify the TDengine function efficiently. The document is divided into three main sections: introduction, prerequisites and testing guide. +This manual is intended to provide users with comprehensive guidance to help them verify the TDengine function efficiently. + +The document is divided into three main sections: introduction, prerequisites and testing guide. > [!NOTE] > The below commands and test scripts are verified on linux (Ubuntu 18.04、20.04、22.04) locally. @@ -89,7 +91,9 @@ Modify the CMakeLists.txt file in this directory to ensure that the new test fil ##### 3. Compile test code -In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., `cmake .. -DBUILD_TEST=1`) to generate a compilation file, and then run a compilation command (e.g. make) to complete the compilation of the test code. +In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., `cmake .. -DBUILD_TEST=1`) to generate a compilation file, + +and then run a compilation command (e.g. make) to complete the compilation of the test code. ##### 4. Execute the test program @@ -110,7 +114,7 @@ work for TDengine Enterprise Edition. ```bash cd tests/system-test -python3 ./test.py -f 2-query/floor.py +python3 ./test.py -f 2-query/avg.py ``` ### How to run all system test cases? @@ -127,6 +131,7 @@ cd tests Detailed steps to add new system test case The Python test framework is developed by TDengine teams, and test.py is the test case execution and monitoring of the entry program, Use `python3 ./test.py -h` to view more features. + you can refer below steps to add one test case: ##### 1.Create a test case file and develop the test cases @@ -209,7 +214,9 @@ Integrate TaskCreateNewFunction into the balance_pickTaskType function in crash_ ## 3.6 CI Test -CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability. TDengine CI testing includes three part of test cases: unit test、system test and legacy test +CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability. + +TDengine CI testing includes three part of test cases: unit test、system test and legacy test ### How to run all CI test cases? From 35f8fb8153f615d71cb2277d381ab75ea57464d9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Jan 2025 19:11:36 +0800 Subject: [PATCH 76/87] ci:modify test readme --- tests/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/README.md b/tests/README.md index bc97d74afd..5574a16743 100644 --- a/tests/README.md +++ b/tests/README.md @@ -140,7 +140,10 @@ Create a file in `tests/system-test` containing each functional directory and re ##### 2.Execute the test case +Ensure the test case execution is successful. +``` bash cd tests/system-test & python3 ./test.py -f 0-others/test_case_template.py +``` ##### 3.Integrate into CI tests From 7d2d44cbe328110c12b87cfca340ed6ca33573b4 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 16 Jan 2025 19:25:12 +0800 Subject: [PATCH 77/87] ci:modify test readme --- README.md | 4 +++- tests/README.md | 46 +++++++++++++++++++++++----------------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 787c5c6b7a..a7009fb6f2 100644 --- a/README.md +++ b/README.md @@ -388,9 +388,11 @@ TDengine build check workflow can be found in this [Github Action](https://githu # 11. Coverage +Latest TDengine test coverage report can be found on [coveralls.io](https://coveralls.io/github/taosdata/TDengine) +
-Latest TDengine test coverage report can be found on [coveralls.io](https://coveralls.io/github/taosdata/TDengine). + how to run the coverage report locally. To create the test coverage report (in HTML format) locally, please run following commands: ```bash diff --git a/tests/README.md b/tests/README.md index 5574a16743..dd1f8ba9a1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -59,21 +59,21 @@ In `tests` directory, there are different types of tests for TDengine. Below is Unit test script is the smallest testable part and developed for some function, method or class of TDengine. -### How to run single test case? +### 3.1.1 How to run single test case? ```bash cd debug/build/bin ./osTimeTests ``` -### How to run all unit test cases? +### 3.1.2 How to run all unit test cases? ```bash cd tests/unit-test/ bash test.sh -e 0 ``` -#### How to add new cases? +### 3.1.3 How to add new cases?
@@ -81,25 +81,25 @@ bash test.sh -e 0 The Google test framwork is used for unit testing to specific function module, you can refer below steps to add one test case: -##### 1. Create test case file and develop the test scripts +##### a. Create test case file and develop the test scripts In the test directory corresponding to the target function module, create test files in CPP format and write corresponding test cases. -##### 2. Update build configuration +##### b. Update build configuration Modify the CMakeLists.txt file in this directory to ensure that the new test files are properly included in the compilation process. See the `source/os/test/CMakeLists.txt` file for configuration examples. -##### 3. Compile test code +##### c. Compile test code In the root directory of the project, create a compilation directory (e.g., debug), switch to the directory and run CMake commands (e.g., `cmake .. -DBUILD_TEST=1`) to generate a compilation file, and then run a compilation command (e.g. make) to complete the compilation of the test code. -##### 4. Execute the test program +##### d. Execute the test program Find the executable file in the compiled directory(e.g. `TDengine/debug/build/bin/`) and run it. -##### 5. Integrate into CI tests +##### e. Integrate into CI tests Use the add_test command to add new compiled test cases into CI test collection, ensure that the new added test cases can be run for every build. @@ -110,21 +110,21 @@ Use the add_test command to add new compiled test cases into CI test collection, Python test script includes all of the functions of TDengine OSS, so some test case maybe fail cause the function only work for TDengine Enterprise Edition. -### How to run single test case? +### 3.2.1 How to run single test case? ```bash cd tests/system-test python3 ./test.py -f 2-query/avg.py ``` -### How to run all system test cases? +### 3.2.2 How to run all system test cases? ```bash cd tests ./run_all_ci_cases.sh -t python # all python cases ``` -### How to add new case? +### 3.2.3 How to add new case?
@@ -134,18 +134,18 @@ The Python test framework is developed by TDengine teams, and test.py is the tes you can refer below steps to add one test case: -##### 1.Create a test case file and develop the test cases +##### a. Create a test case file and develop the test cases Create a file in `tests/system-test` containing each functional directory and refer to the use case template `tests/system-test/0-others/test_case_template.py` to add a new test case. -##### 2.Execute the test case +##### b. Execute the test case Ensure the test case execution is successful. ``` bash cd tests/system-test & python3 ./test.py -f 0-others/test_case_template.py ``` -##### 3.Integrate into CI tests +##### c. Integrate into CI tests Edit `tests/parallel_test/cases.task` and add the testcase path and executions in the specified format. The third column indicates whether to use Address Sanitizer mode for testing. @@ -162,7 +162,7 @@ Edit `tests/parallel_test/cases.task` and add the testcase path and executions i In the early stage of TDengine development, test cases are run by an internal test framework called TSIM, which is developed in C++. -### How to run single test case? +### 3.3.1 How to run single test case? To run the legacy test cases, please execute the following commands: @@ -171,14 +171,14 @@ cd tests/script ./test.sh -f tsim/db/basic1.sim ``` -### How to run all legacy test cases? +### 3.3.2 How to run all legacy test cases? ```bash cd tests ./run_all_ci_cases.sh -t legacy # all legacy cases ``` -### How to add new case? +### 3.3.3 How to add new case? > [!NOTE] > TSIM test framwork is replaced by system test currently, suggest to add new test scripts to system test, you can refer [System Test](#32-system-test) for detail steps. @@ -187,14 +187,14 @@ cd tests Smoke test script is from system test and known as sanity testing to ensure that the critical functionalities of TDengine. -### How to run test? +### 3.4.1 How to run test? ```bash cd /root/TDengine/packaging/smokeTest ./test_smoking_selfhost.sh ``` -### How to add new case? +### 3.4.2 How to add new case? You can update python commands part of test_smoking_selfhost.sh file to add any system test case into smoke test. @@ -202,14 +202,14 @@ You can update python commands part of test_smoking_selfhost.sh file to add any A simple tool to exercise various functions of the system in a randomized fashion, hoping to expose maximum number of problems without a pre-determined scenario. -### How to run test? +### 3.5.1 How to run test? ```bash cd tests/pytest python3 auto_crash_gen.py ``` -### How to add new case? +### 3.5.2 How to add new case? Add a function, such as TaskCreateNewFunction, to pytest/crash_gen/crash_gen_main.py. @@ -221,7 +221,7 @@ CI testing (Continuous Integration testing), is an important practice in softwar TDengine CI testing includes three part of test cases: unit test、system test and legacy test -### How to run all CI test cases? +### 3.6.1 How to run all CI test cases? If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands: @@ -230,6 +230,6 @@ cd tests ./run_all_ci_cases.sh -b main # on main branch ``` -### How to add new cases? +### 3.6.2 How to add new cases? You can refer the [Unit Test](#31-unit-test)、[System Test](#32-system-test) and [Legacy Test](#33-legacy-test) sections for detail steps to add new test cases for CI test. From b992f27ca0154740950f21bb0dc2483683571b37 Mon Sep 17 00:00:00 2001 From: sheyanjie-qq <249478495@qq.com> Date: Thu, 16 Jan 2025 19:31:17 +0800 Subject: [PATCH 78/87] jdbc update to 3.5.2 --- docs/en/07-develop/01-connect.md | 2 +- docs/en/14-reference/05-connector/14-java.md | 1 + docs/examples/JDBC/JDBCDemo/pom.xml | 2 +- docs/examples/JDBC/SpringJdbcTemplate/pom.xml | 2 +- docs/examples/JDBC/connectionPools/pom.xml | 2 +- docs/examples/JDBC/consumer-demo/pom.xml | 2 +- docs/examples/JDBC/mybatisplus-demo/pom.xml | 2 +- docs/examples/JDBC/springbootdemo/pom.xml | 2 +- docs/examples/JDBC/taosdemo/pom.xml | 2 +- docs/examples/java/pom.xml | 2 +- docs/zh/07-develop/01-connect/index.md | 2 +- docs/zh/14-reference/05-connector/14-java.mdx | 1 + 12 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/en/07-develop/01-connect.md b/docs/en/07-develop/01-connect.md index c14eed311a..b6725ed7a4 100644 --- a/docs/en/07-develop/01-connect.md +++ b/docs/en/07-develop/01-connect.md @@ -109,7 +109,7 @@ If you are using Maven to manage your project, simply add the following dependen com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 ``` diff --git a/docs/en/14-reference/05-connector/14-java.md b/docs/en/14-reference/05-connector/14-java.md index c28702440a..0c7fbbdda4 100644 --- a/docs/en/14-reference/05-connector/14-java.md +++ b/docs/en/14-reference/05-connector/14-java.md @@ -33,6 +33,7 @@ The JDBC driver implementation for TDengine strives to be consistent with relati | taos-jdbcdriver Version | Major Changes | TDengine Version | | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | +| 3.5.2 | Fixed WebSocket result set free bug. | - | | 3.5.1 | Fixed the getObject issue in data subscription. | - | | 3.5.0 | 1. Optimized the performance of WebSocket connection parameter binding, supporting parameter binding queries using binary data.
2. Optimized the performance of small queries in WebSocket connection.
3. Added support for setting time zone and app info on WebSocket connection. | 3.3.5.0 and higher | | 3.4.0 | 1. Replaced fastjson library with jackson.
2. WebSocket uses a separate protocol identifier.
3. Optimized background thread usage to avoid user misuse leading to timeouts. | - | diff --git a/docs/examples/JDBC/JDBCDemo/pom.xml b/docs/examples/JDBC/JDBCDemo/pom.xml index 78262712e9..a80f7a9cdf 100644 --- a/docs/examples/JDBC/JDBCDemo/pom.xml +++ b/docs/examples/JDBC/JDBCDemo/pom.xml @@ -19,7 +19,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 org.locationtech.jts diff --git a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml index 7ff4a72f5e..20da9bfae8 100644 --- a/docs/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/docs/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 diff --git a/docs/examples/JDBC/connectionPools/pom.xml b/docs/examples/JDBC/connectionPools/pom.xml index 70be6ed527..86e6fb04a4 100644 --- a/docs/examples/JDBC/connectionPools/pom.xml +++ b/docs/examples/JDBC/connectionPools/pom.xml @@ -18,7 +18,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 diff --git a/docs/examples/JDBC/consumer-demo/pom.xml b/docs/examples/JDBC/consumer-demo/pom.xml index c9537a93bf..e439c22224 100644 --- a/docs/examples/JDBC/consumer-demo/pom.xml +++ b/docs/examples/JDBC/consumer-demo/pom.xml @@ -17,7 +17,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 com.google.guava diff --git a/docs/examples/JDBC/mybatisplus-demo/pom.xml b/docs/examples/JDBC/mybatisplus-demo/pom.xml index effb13cfe8..8b4777bfb0 100644 --- a/docs/examples/JDBC/mybatisplus-demo/pom.xml +++ b/docs/examples/JDBC/mybatisplus-demo/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 diff --git a/docs/examples/JDBC/springbootdemo/pom.xml b/docs/examples/JDBC/springbootdemo/pom.xml index 25b503b0e6..825f6fb9c2 100644 --- a/docs/examples/JDBC/springbootdemo/pom.xml +++ b/docs/examples/JDBC/springbootdemo/pom.xml @@ -70,7 +70,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 diff --git a/docs/examples/JDBC/taosdemo/pom.xml b/docs/examples/JDBC/taosdemo/pom.xml index a80deeff94..c8f9c73d9d 100644 --- a/docs/examples/JDBC/taosdemo/pom.xml +++ b/docs/examples/JDBC/taosdemo/pom.xml @@ -67,7 +67,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 diff --git a/docs/examples/java/pom.xml b/docs/examples/java/pom.xml index 63ce3159e6..4569742ab4 100644 --- a/docs/examples/java/pom.xml +++ b/docs/examples/java/pom.xml @@ -22,7 +22,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index fa22f750f5..a6e30ccb9c 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -89,7 +89,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 com.taosdata.jdbc taos-jdbcdriver - 3.5.1 + 3.5.2 ``` diff --git a/docs/zh/14-reference/05-connector/14-java.mdx b/docs/zh/14-reference/05-connector/14-java.mdx index 7d5096bb66..9325396eb8 100644 --- a/docs/zh/14-reference/05-connector/14-java.mdx +++ b/docs/zh/14-reference/05-connector/14-java.mdx @@ -33,6 +33,7 @@ TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致 | taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 | | ------------------| ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| 3.5.2 | 解决了 WebSocket 查询结果集释放 bug | - | | 3.5.1 | 解决了数据订阅获取时间戳对象类型问题 | - | | 3.5.0 | 1. 优化了 WebSocket 连接参数绑定性能，支持参数绑定查询使用二进制数据
2. 优化了 WebSocket 连接在小查询上的性能
3. WebSocket 连接上支持设置时区和应用信息 | 3.3.5.0 及更高版本 | | 3.4.0 | 1. 使用 jackson 库替换 fastjson 库
2. WebSocket 采用独立协议标识
3. 优化后台拉取线程使用，避免用户误用导致超时 | - | From f54ee09915172a2545c1facfa6ba5086d79ea17c Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 16 Jan 2025 20:51:37 +0800 Subject: [PATCH 79/87] Update README.md --- tests/README.md | 53 +++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/tests/README.md b/tests/README.md index dd1f8ba9a1..26ced3da17 100644 --- a/tests/README.md +++ b/tests/README.md @@ -12,12 +12,10 @@ # 1. Introduction -This manual is intended to provide users with comprehensive guidance to help them verify the TDengine function efficiently. - -The document is divided into three main sections: introduction, prerequisites and testing guide. +This manual is intended to give developers a comprehensive guidance to test TDengine efficiently. It is divided into three main sections: introduction, prerequisites and testing guide. > [!NOTE] -> The below commands and test scripts are verified on linux (Ubuntu 18.04、20.04、22.04) locally. +> The commands and scripts below are verified on Linux (Ubuntu 18.04/20.04/22.04). # 2. Prerequisites @@ -43,7 +41,7 @@ pip3 install taospy taos-ws-py - Building -Please make sure building operation with option `-DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true` has been finished, otherwise execute commands below: +Before testting, please make sure the building operation with option `-DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true` has been done, otherwise execute commands below: ```bash cd debug @@ -57,7 +55,7 @@ In `tests` directory, there are different types of tests for TDengine. Below is ### 3.1 Unit Test -Unit test script is the smallest testable part and developed for some function, method or class of TDengine. +Unit tests are the smallest testable units, which are used to test functions, methods or classes in TDengine code. ### 3.1.1 How to run single test case? @@ -79,7 +77,7 @@ bash test.sh -e 0 Detailed steps to add new unit test case -The Google test framwork is used for unit testing to specific function module, you can refer below steps to add one test case: +The Google test framwork is used for unit testing to specific function module, please refer to steps below to add a new test case: ##### a. Create test case file and develop the test scripts @@ -107,10 +105,11 @@ Use the add_test command to add new compiled test cases into CI test collection, ## 3.2 System Test -Python test script includes all of the functions of TDengine OSS, so some test case maybe fail cause the function only -work for TDengine Enterprise Edition. +System tests are end-to-end test cases written in Python from a system point of view. Some of them are designed to test features only in enterprise ediiton, so when running on community edition, they may fail. We'll fix this issue by separating the cases into different gruops in the future. -### 3.2.1 How to run single test case? +### 3.2.1 How to run a single test case? + +Take test file `system-test/2-query/avg.py` for example: ```bash cd tests/system-test @@ -130,9 +129,9 @@ cd tests Detailed steps to add new system test case -The Python test framework is developed by TDengine teams, and test.py is the test case execution and monitoring of the entry program, Use `python3 ./test.py -h` to view more features. +The Python test framework is developed by TDengine team, and test.py is the test case execution and monitoring of the entry program, Use `python3 ./test.py -h` to view more features. -you can refer below steps to add one test case: +Please refer to steps below for how to add a new test case: ##### a. Create a test case file and develop the test cases @@ -141,16 +140,15 @@ Create a file in `tests/system-test` containing each functional directory and re ##### b. Execute the test case Ensure the test case execution is successful. + ``` bash -cd tests/system-test & python3 ./test.py -f 0-others/test_case_template.py +cd tests/system-test && python3 ./test.py -f 0-others/test_case_template.py ``` ##### c. Integrate into CI tests Edit `tests/parallel_test/cases.task` and add the testcase path and executions in the specified format. The third column indicates whether to use Address Sanitizer mode for testing. - - ```bash #caseID,rerunTimes,Run with Sanitizer,casePath,caseCommand ,,n,system-test, python3 ./test.py -f 0-others/test_case_template.py @@ -178,14 +176,14 @@ cd tests ./run_all_ci_cases.sh -t legacy # all legacy cases ``` -### 3.3.3 How to add new case? +### 3.3.3 How to add new cases? > [!NOTE] -> TSIM test framwork is replaced by system test currently, suggest to add new test scripts to system test, you can refer [System Test](#32-system-test) for detail steps. +> TSIM test framwork is deprecated by system test now, it is encouraged to add new test cases in system test, please refer to [System Test](#32-system-test) for details. ## 3.4 Smoke Test -Smoke test script is from system test and known as sanity testing to ensure that the critical functionalities of TDengine. +Smoke test is a group of test cases selected from system test, which is also known as sanity test to ensure the critical functionalities of TDengine. ### 3.4.1 How to run test? @@ -194,13 +192,13 @@ cd /root/TDengine/packaging/smokeTest ./test_smoking_selfhost.sh ``` -### 3.4.2 How to add new case? +### 3.4.2 How to add new cases? -You can update python commands part of test_smoking_selfhost.sh file to add any system test case into smoke test. +New cases can be added by updating the value of `commands` variable in `test_smoking_selfhost.sh`. ## 3.5 Chaos Test -A simple tool to exercise various functions of the system in a randomized fashion, hoping to expose maximum number of problems without a pre-determined scenario. +A simple tool to execute various functions of the system in a randomized way, hoping to expose potential problems without a pre-defined test scenario. ### 3.5.1 How to run test? @@ -209,21 +207,20 @@ cd tests/pytest python3 auto_crash_gen.py ``` -### 3.5.2 How to add new case? +### 3.5.2 How to add new cases? -Add a function, such as TaskCreateNewFunction, to pytest/crash_gen/crash_gen_main.py. - -Integrate TaskCreateNewFunction into the balance_pickTaskType function in crash_gen_main.py. +1. Add a function, such as `TaskCreateNewFunction` in `pytest/crash_gen/crash_gen_main.py`. +2. Integrate `TaskCreateNewFunction` into the `balance_pickTaskType` function in `crash_gen_main.py`. ## 3.6 CI Test CI testing (Continuous Integration testing), is an important practice in software development that aims to automate frequent integration of code into a shared codebase, build and test it to ensure code quality and stability. -TDengine CI testing includes three part of test cases: unit test、system test and legacy test +TDengine CI testing will run all the test cases from the following three types of tests: unit test, system test and legacy test. ### 3.6.1 How to run all CI test cases? -If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands: +If this is the first time to run all the CI test cases, it is recommended to add the test branch, please run it with following commands: ```bash cd tests @@ -232,4 +229,4 @@ cd tests ### 3.6.2 How to add new cases? -You can refer the [Unit Test](#31-unit-test)、[System Test](#32-system-test) and [Legacy Test](#33-legacy-test) sections for detail steps to add new test cases for CI test. +Please refer to the [Unit Test](#31-unit-test)、[System Test](#32-system-test) and [Legacy Test](#33-legacy-test) sections for detailed steps to add new test cases, when new cases are added in aboved tests, they will be run automatically by CI test. From a6d343cdb8ce1d7de3c4cde6e21285b1a63663a0 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 16 Jan 2025 20:56:47 +0800 Subject: [PATCH 80/87] Update README.md --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 26ced3da17..23b2ca8700 100644 --- a/tests/README.md +++ b/tests/README.md @@ -41,7 +41,7 @@ pip3 install taospy taos-ws-py - Building -Before testting, please make sure the building operation with option `-DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true` has been done, otherwise execute commands below: +Before testing, please make sure the building operation with option `-DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true` has been done, otherwise execute commands below: ```bash cd debug From f2ae4a042565db54011de9909f861bde65a15f23 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 16 Jan 2025 21:11:39 +0800 Subject: [PATCH 81/87] Update README.md --- tests/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 23b2ca8700..58747d93f7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -15,7 +15,8 @@ This manual is intended to give developers a comprehensive guidance to test TDengine efficiently. It is divided into three main sections: introduction, prerequisites and testing guide. > [!NOTE] -> The commands and scripts below are verified on Linux (Ubuntu 18.04/20.04/22.04). +> - The commands and scripts below are verified on Linux (Ubuntu 18.04/20.04/22.04). +> - The commands and steps described below are to run the tests on a single host. # 2. Prerequisites From 1fb80a3b2580855cd7632b9a079f722375644eb4 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 16 Jan 2025 21:15:15 +0800 Subject: [PATCH 82/87] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index a7009fb6f2..53ddc85bf9 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,7 @@ nmake # 5. Packaging -TDengine packaging scripts depends on some private repositries currently, you can refer the link for detail steps. [Packaging](https://github.com/taosdata/TDinternal/tree/main?tab=readme-ov-file#5-packaging) +[Placeholder] # 6. Installation @@ -406,7 +406,6 @@ bash setup-lcov.sh -v 1.16 && ./run_local_coverage.sh -b main -c task
- # 12. Contributing Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to TDengine. From d95a1ca89a661d68f21a16f8b21569872ccd700e Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Thu, 16 Jan 2025 21:16:56 +0800 Subject: [PATCH 83/87] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 53ddc85bf9..a09b9cfea3 100644 --- a/README.md +++ b/README.md @@ -384,7 +384,7 @@ For the complete list of TDengine Releases, please see [Releases](https://github # 10. Workflow -TDengine build check workflow can be found in this [Github Action](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml). +TDengine build check workflow can be found in this [Github Action](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml). More workflows will be available soon. # 11. Coverage From 25f5b8673f473704ff17126572bdf2974770e94e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Jan 2025 10:53:02 +0800 Subject: [PATCH 84/87] fix: compile errors --- source/libs/parser/test/parAlterToBalanceTest.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index 12390cc9d5..2d6ab13c1f 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -206,10 +206,6 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbReplica(3); setAlterDbWalRetentionPeriod(10); setAlterDbWalRetentionSize(20); -#ifndef _STORAGE - run("ALTER DATABASE test BUFFER 16 CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 PAGES 128 " - "REPLICA 3 WAL_LEVEL 1 WAL_RETENTION_PERIOD 10 WAL_RETENTION_SIZE 20"); -#else run("ALTER DATABASE test BUFFER 16 CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 PAGES 128 " "REPLICA 3 WAL_LEVEL 1 " #ifdef TD_ENTERPRISE From 2312ed4c284ecd4e9f3b12768bb7cebe404a06be Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 17 Jan 2025 04:12:57 +0000 Subject: [PATCH 85/87] fix/insert-when-2-replicas --- source/libs/sync/src/syncMain.c | 3 ++- tests/army/cluster/arbitrator.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4862a4b963..0933fd48c7 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3428,7 +3428,8 @@ _out:; ths->pLogBuf->matchIndex, ths->pLogBuf->endIndex); if (code == 0 && ths->state == TAOS_SYNC_STATE_ASSIGNED_LEADER) { - TAOS_CHECK_RETURN(syncNodeUpdateAssignedCommitIndex(ths, matchIndex)); + int64_t index = syncNodeUpdateAssignedCommitIndex(ths, matchIndex); + sTrace("vgId:%d, update assigned commit index %" PRId64 "", ths->vgId, index); if (ths->fsmState != SYNC_FSM_STATE_INCOMPLETE && syncLogBufferCommit(ths->pLogBuf, ths, ths->assignedCommitIndex) < 0) { diff --git a/tests/army/cluster/arbitrator.py b/tests/army/cluster/arbitrator.py index 9fd8e7b1f3..385358e5cc 100644 --- a/tests/army/cluster/arbitrator.py +++ b/tests/army/cluster/arbitrator.py @@ -35,6 +35,12 @@ class TDTestCase(TBase): time.sleep(1) + tdSql.execute("use db;") + + tdSql.execute("CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);") + + tdSql.execute("CREATE TABLE d0 USING meters TAGS (\"California.SanFrancisco\", 2);"); + count = 0 while count < 100: @@ -72,6 +78,8 @@ class TDTestCase(TBase): count += 1 + tdSql.execute("INSERT INTO d0 VALUES (NOW, 10.3, 219, 0.31);") + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") From d4ceb37f8c9a74bf43afba3280895f9525a2d466 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Jan 2025 13:44:21 +0800 Subject: [PATCH 86/87] fix: ut errors --- source/libs/parser/test/parAlterToBalanceTest.cpp | 10 ---------- source/libs/parser/test/parInitialCTest.cpp | 8 -------- 2 files changed, 18 deletions(-) diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index 2d6ab13c1f..172c729f34 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -196,11 +196,7 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbFsync(200); setAlterDbWal(1); setAlterDbCacheModel(TSDB_CACHE_MODEL_LAST_ROW); -#ifdef TD_ENTERPRISE setAlterDbSttTrigger(16); -#else - setAlterDbSttTrigger(1); -#endif setAlterDbBuffer(16); setAlterDbPages(128); setAlterDbReplica(3); @@ -208,11 +204,7 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbWalRetentionSize(20); run("ALTER DATABASE test BUFFER 16 CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 PAGES 128 " "REPLICA 3 WAL_LEVEL 1 " -#ifdef TD_ENTERPRISE "STT_TRIGGER 16 " -#else - "STT_TRIGGER 1 " -#endif "WAL_RETENTION_PERIOD 10 WAL_RETENTION_SIZE 20"); clearAlterDbReq(); @@ -300,12 +292,10 @@ TEST_F(ParserInitialATest, alterDatabase) { initAlterDb("test"); setAlterDbSttTrigger(1); run("ALTER DATABASE test STT_TRIGGER 1"); -#ifdef TD_ENTERPRISE setAlterDbSttTrigger(4); run("ALTER DATABASE test STT_TRIGGER 4"); setAlterDbSttTrigger(16); run("ALTER DATABASE test STT_TRIGGER 16"); -#endif clearAlterDbReq(); #endif diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 0f72f9d882..2412bf4e78 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -292,11 +292,7 @@ TEST_F(ParserInitialCTest, createDatabase) { setDbWalRetentionSize(-1); setDbWalRollPeriod(10); setDbWalSegmentSize(20); -#ifdef TD_ENTERPRISE - setDbSstTrigger(16); -#else setDbSstTrigger(1); -#endif setDbHashPrefix(3); setDbHashSuffix(4); setDbTsdbPageSize(32); @@ -354,11 +350,7 @@ TEST_F(ParserInitialCTest, createDatabase) { "WAL_RETENTION_SIZE -1 " "WAL_ROLL_PERIOD 10 " "WAL_SEGMENT_SIZE 20 " -#ifdef TD_ENTERPRISE - "STT_TRIGGER 16 " -#else "STT_TRIGGER 1 " -#endif "TABLE_PREFIX 3 " "TABLE_SUFFIX 4 " "TSDB_PAGESIZE 32"); From 5b95b519ee4f9553e81ff69176222ee1a6589aea Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Fri, 17 Jan 2025 14:05:22 +0800 Subject: [PATCH 87/87] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a09b9cfea3..f540b1cc43 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,7 @@ nmake # 5. Packaging -[Placeholder] +The TDengine community installer can NOT be created by this repository only, due to some component dependencies. We are still working on this improvement. # 6. Installation @@ -392,7 +392,7 @@ Latest TDengine test coverage report can be found on [coveralls.io](https://cove
- how to run the coverage report locally. +How to run the coverage report locally? To create the test coverage report (in HTML format) locally, please run following commands: ```bash