From 79580b80c9249133c8902b457d1488efe5b1fea9 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 28 Jan 2021 14:36:12 +0800 Subject: [PATCH 01/11] [TD-2863] : change the unit of measurement for config tsQueryBufferSize. --- documentation20/webdocs/markdowndocs/administrator-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 7c8344d9ca..97be21e5ad 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -102,7 +102,7 @@ taosd -C - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。 - stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。 -- queryBufferSize: 为所有并发查询占用保留的内存大小。计算规则可以根据实际应用可能的最大并发数和表的数字相乘，再乘 170 。单位为字节。 +- queryBufferSize: 为所有并发查询占用保留的内存大小。计算规则可以根据实际应用可能的最大并发数和表的数字相乘，再乘 170 。单位为 MB（2.0.15 以前的版本中，此参数的单位是字节）。 - ratioOfQueryCores: 设置查询线程的最大数量。最小值0 表示只有1个查询线程；最大值2表示最大建立2倍CPU核数的查询线程。默认为1，表示最大和CPU核数相等的查询线程。该值可以为小数，即0.5表示最大建立CPU核数一半的查询线程。 **注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030到6042共13个端口，而且必须TCP和UDP都打开。 From 766dc0bb4fb1326327cdbf7ba9d935536d2cf121 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 18 Feb 2021 14:48:09 +0800 Subject: [PATCH 02/11] [TD-2568] : describe new commands about client log management. --- documentation20/cn/12.taos-sql/docs.md | 1 + documentation20/cn/13.faq/docs.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 6db747bb23..7ef6fe2060 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -267,6 +267,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ``` ## 超级表 STable 中 TAG 管理 + - **添加标签** ```mysql diff --git a/documentation20/cn/13.faq/docs.md b/documentation20/cn/13.faq/docs.md index 8d4811805a..96ef8a4ec0 100644 --- a/documentation20/cn/13.faq/docs.md +++ b/documentation20/cn/13.faq/docs.md @@ -140,3 +140,20 @@ TDengine是根据hostname唯一标志一台机器的，在数据文件从机器A - 2.0.7.0 及以后的版本，到/var/lib/taos/dnode下，修复dnodeEps.json的dnodeId对应的FQDN，重启。确保机器内所有机器的此文件是完全相同的。 - 1.x 和 2.x 版本的存储结构不兼容，需要使用迁移工具或者自己开发应用导出导入数据。 +## 17. 如何在命令行程序 taos 中临时调整日志级别 + +为了调试方便，从 2.0.16 版本开始，命令行程序 taos 新增了与日志记录相关的两条指令： + +```mysql +ALTER LOCAL flag_name flag_value; +``` + +其含义是，在当前的命令行程序下，修改一个特定模块的日志记录级别（只对当前命令行程序有效，如果 taos 命令行程序重启，则需要重新设置）： +- flag_name 的取值可以是：debugFlag，cDebugFlag，tmrDebugFlag，uDebugFlag，rpcDebugFlag +- flag_value 的取值可以是：131（输出错误和警告日志），135（ 输出错误、警告和调试日志），143（ 输出错误、警告、调试和跟踪日志） + +```mysql +ALTER LOCAL RESETLOG; +``` + +其含义是，清空本机所有由客户端生成的日志文件。 From aedccee5162a49c7a0077e2fe71936fe21f3c34b Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 18 Feb 2021 15:25:35 +0800 Subject: [PATCH 03/11] test date befor 1970 --- tests/pytest/pytest_1.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh index 6905f0c61e..729319c231 100755 --- a/tests/pytest/pytest_1.sh +++ b/tests/pytest/pytest_1.sh @@ -19,7 +19,7 @@ python3 ./test.py -f insert/randomNullCommit.py #python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py -#python3 ./test.py -f insert/before_1970.py +python3 ./test.py -f insert/before_1970.py python3 bug2265.py #table From a586784c5d97c79cb26c89e47649187d646e1558 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 18 Feb 2021 16:00:04 +0800 Subject: [PATCH 04/11] [TD-2977]: add test case --- tests/perftest-scripts/perftest-query.sh | 10 +++-- tests/pytest/fulltest.sh | 2 +- tests/pytest/tag_lite/alter_tag.py | 52 ++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 tests/pytest/tag_lite/alter_tag.py diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index 8498094181..07055d64b0 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -39,6 +39,8 @@ function buildTDengine { cd $WORK_DIR/TDengine git remote update > /dev/null + git reset --hard HEAD + git checkout develop REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` LOCAL_COMMIT=`git rev-parse --short @` @@ -54,15 +56,16 @@ function buildTDengine { cd debug rm -rf * cmake .. > /dev/null - make > /dev/null - make install + make && make install > /dev/null fi } function runQueryPerfTest { [ -f $PERFORMANCE_TEST_REPORT ] && rm $PERFORMANCE_TEST_REPORT nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taosperf/ > /dev/null 2>&1 & - echoInfo "Run Performance Test" + echoInfo "Wait TDengine to start" + sleep 60 + echoInfo "Run Performance Test" cd $WORK_DIR/TDengine/tests/pytest python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT @@ -104,6 +107,7 @@ function sendReport { stopTaosd buildTDengine runQueryPerfTest +stopTaosd echoInfo "Send Report" sendReport diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 9c5ee6aae0..6b710732ca 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -66,7 +66,7 @@ python3 ./test.py -f tag_lite/int.py python3 ./test.py -f tag_lite/set.py python3 ./test.py -f tag_lite/smallint.py python3 ./test.py -f tag_lite/tinyint.py - +python3 ./test.py -f tag_lite/alter_tag.py #python3 ./test.py -f dbmgmt/database-name-boundary.py python3 ./test.py -f import_merge/importBlock1HO.py diff --git a/tests/pytest/tag_lite/alter_tag.py b/tests/pytest/tag_lite/alter_tag.py new file mode 100644 index 0000000000..aafffc19c3 --- /dev/null +++ b/tests/pytest/tag_lite/alter_tag.py @@ -0,0 +1,52 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "CREATE TABLE IF NOT EXISTS ampere (ts TIMESTAMP(8),ampere DOUBLE(8)) TAGS (device_name BINARY(50),build_id BINARY(50),project_id BINARY(50),alias BINARY(50))") + tdSql.execute("insert into d1001 using ampere tags('test', '2', '2', '2') VALUES (now, 123)") + tdSql.execute("ALTER TABLE ampere ADD TAG variable_id BINARY(50)") + + print("==============step2") + + tdSql.execute("insert into d1002 using ampere tags('test', '2', '2', '2', 'test') VALUES (now, 124)") + + tdSql.query("select * from ampere") + tdSql.checkRows(2) + tdSql.checkData(0, 6, None) + tdSql.checkData(1, 6, 'test') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 1e1d038f731f0368b5da6548ddfedc0fcf22a382 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 18 Feb 2021 16:28:45 +0800 Subject: [PATCH 05/11] [TD-2980]add nodejs connector test into CI --- tests/Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index f5b0cf1478..aa1815fc63 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -109,6 +109,13 @@ pipeline { java --class-path=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 ''' } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cp -rf ${WKC}/tests/examples/nodejs ${JENKINS_HOME}/workspace/ + cd ${JENKINS_HOME}/workspace/nodejs + node nodejsChecker.js host=localhost + ''' + } catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# From 31ef1fef0088bba56b1a18b72fb99865a8d2eede Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 18 Feb 2021 08:34:26 +0000 Subject: [PATCH 06/11] [TD-2982] : disable rpmbuild auto detect dependency --- packaging/rpm/tdengine.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index d20a6c91cd..a080876c60 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -9,6 +9,7 @@ Summary: tdengine from taosdata Group: Application/Database License: AGPL URL: www.taosdata.com +AutoReqProv: no #BuildRoot: %_topdir/BUILDROOT BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root From bc8141e1a90ce81da97e0cfc37257cad919d70ee Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 18 Feb 2021 16:38:17 +0800 Subject: [PATCH 07/11] [TD-203] : emphasize the best practice about connection sharing in multi threads. --- documentation20/cn/08.connector/docs.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 1176c0fbcd..e473588782 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -24,6 +24,7 @@ TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、 * 在没有安装TDengine服务端软件的系统中使用连接器（除RESTful外）访问 TDengine 数据库，需要安装相应版本的客户端安装包来使应用驱动（Linux系统中文件名为libtaos.so，Windows系统中为taos.dll）被安装在系统中，否则会产生无法找到相应库文件的错误。 * 所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。 * 升级到TDengine到2.0.8.0版本的用户，必须更新JDBC连接TDengine必须升级taos-jdbcdriver到2.0.12及以上。 +* 无论选用何种编程语言的连接器，2.0 及以上版本的 TDengine 推荐数据库应用的每个线程都建立一个独立的连接，或基于线程建立连接池，以避免连接内的“USE statement”状态量在线程之间相互干扰（但连接的查询和写入操作都是线程安全的）。 ## 安装连接器驱动步骤 @@ -238,13 +239,13 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine 获取查询结果集每列数据的属性（数据类型、名字、字节数），与taos_num_fileds配合使用，可用来解析`taos_fetch_row`返回的一个元组(一行)的数据。 `TAOS_FIELD` 的结构如下： - ```c - typedef struct taosField { - char name[65]; // 列名 - uint8_t type; // 数据类型 - int16_t bytes; // 字节数 - } TAOS_FIELD; - ``` +```c +typedef struct taosField { + char name[65]; // 列名 + uint8_t type; // 数据类型 + int16_t bytes; // 字节数 +} TAOS_FIELD; +``` - `void taos_stop_query(TAOS_RES *res)` @@ -266,7 +267,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine ### 异步查询API -同步API之外，TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下，异步API处理数据插入的速度比同步API快2\~4倍。异步API采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步API在网络延迟严重的情况下，优点尤为突出。 +同步API之外，TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下，异步API处理数据插入的速度比同步API快2～4倍。异步API采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步API在网络延迟严重的情况下，优点尤为突出。 异步API都需要应用提供相应的回调函数，回调函数参数设置如下：前两个参数都是一致的，第三个参数依不同的API而定。第一个参数param是应用调用异步API时提供给系统的，用于回调时，应用能够找回具体操作的上下文，依具体实现而定。第二个参数是SQL操作的结果集，如果为空，比如insert操作，表示没有记录返回，如果不为空，比如select操作，表示有记录返回。 @@ -896,7 +897,7 @@ Node-example-raw.js 验证方法： -1. 新建安装验证目录，例如：\~/tdengine-test，拷贝github上nodejsChecker.js源程序。下载地址：（https://github.com/taosdata/TDengine/tree/develop/tests/examples/nodejs/nodejsChecker.js）。 +1. 新建安装验证目录，例如：`~/tdengine-test`，拷贝github上nodejsChecker.js源程序。下载地址：（https://github.com/taosdata/TDengine/tree/develop/tests/examples/nodejs/nodejsChecker.js）。 2. 在命令中执行以下命令： From 889c6be4dde6aed3cfc31cdd6e8fd00d2b5413e5 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 18 Feb 2021 16:45:41 +0800 Subject: [PATCH 08/11] fix Jenkinsfile error --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d7d2ab6a9a..25245edd6c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ node { git url: 'https://github.com/taosdata/TDengine.git' } -def kipstage=0 +def skipstage=0 def abortPreviousBuilds() { def currentJobName = env.JOB_NAME def currentBuildNumber = env.BUILD_NUMBER.toInteger() From 98fd766007eff5cbf8efa7b2f59780e06ffaf768 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 18 Feb 2021 17:00:08 +0800 Subject: [PATCH 09/11] modify dockerManifest.sh to push latest tag --- packaging/docker/dockerManifest.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh index ca2c3c66c9..b52580cfa6 100755 --- a/packaging/docker/dockerManifest.sh +++ b/packaging/docker/dockerManifest.sh @@ -35,10 +35,11 @@ done echo "verNumber=${verNumber}" -docker manifest create -a tdengine/tdengine:${verNumber} tdengine/tdengine-amd64:${verNumber} tdengine/tdengine-aarch64:${verNumber} tdengine/tdengine-aarch32:${verNumber} +#docker manifest create -a tdengine/tdengine:${verNumber} tdengine/tdengine-amd64:${verNumber} tdengine/tdengine-aarch64:${verNumber} tdengine/tdengine-aarch32:${verNumber} +docker manifest create -a tdengine/tdengine tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest docker login -u tdengine -p ${passWord} #replace the docker registry username and password -docker manifest push tdengine/tdengine:${verNumber} +docker manifest push tdengine/tdengine # how set latest version ??? From a154d1328ea081d0f0ef3f779945ca951c2c0cb5 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 18 Feb 2021 17:57:00 +0800 Subject: [PATCH 10/11] balance --- Jenkinsfile | 3 ++- tests/script/jenkins/basic_2.txt | 1 - tests/script/jenkins/basic_4.txt | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 25245edd6c..571d776a96 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -88,8 +88,9 @@ pipeline { git checkout -qf FETCH_HEAD ''' script{ - skipstage=sh(script:"git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 0 ",returnStdout:true) + env.skipstage=sh(script:"cd ${WORKSPACE}.tes && git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 0 ",returnStdout:true) } + println env.skipstage sh''' rm -rf ${WORKSPACE}.tes ''' diff --git a/tests/script/jenkins/basic_2.txt b/tests/script/jenkins/basic_2.txt index 20e574711a..5a2a6f4062 100644 --- a/tests/script/jenkins/basic_2.txt +++ b/tests/script/jenkins/basic_2.txt @@ -72,4 +72,3 @@ cd ../../../debug; make ./test.sh -f unique/cluster/cache.sim ./test.sh -f unique/cluster/vgroup100.sim -./test.sh -f unique/column/replica3.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_4.txt b/tests/script/jenkins/basic_4.txt index 2f2ed147a2..7735c5b3d5 100644 --- a/tests/script/jenkins/basic_4.txt +++ b/tests/script/jenkins/basic_4.txt @@ -35,4 +35,6 @@ ./test.sh -f general/stable/refcount.sim ./test.sh -f general/stable/show.sim ./test.sh -f general/stable/values.sim -./test.sh -f general/stable/vnode3.sim \ No newline at end of file +./test.sh -f general/stable/vnode3.sim + +./test.sh -f unique/column/replica3.sim \ No newline at end of file From 38f6e4a6ac31f39f56ff1394c3a10e8db205aecd Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 18 Feb 2021 18:19:22 +0800 Subject: [PATCH 11/11] fix --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 571d776a96..d007adf736 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -102,7 +102,7 @@ pipeline { when { changeRequest() expression { - skipstage != 0 + env.skipstage != 0 } } parallel {