From c5815a51d6e1dd2460e7f87decc7fcf361cdd01d Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 17 Sep 2020 23:26:33 +0800 Subject: [PATCH 01/12] [TD-1245] add test case for interval --- tests/pytest/query/queryInterval.py | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/pytest/query/queryInterval.py diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py new file mode 100644 index 0000000000..45f6988bbe --- /dev/null +++ b/tests/pytest/query/queryInterval.py @@ -0,0 +1,51 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table st (ts timestamp, voltage int) tags (loc nchar(30))") + tdSql.execute("insert into t0 using st tags('beijing') values(now, 220) (now - 15d, 221) (now - 30d, 225) (now - 35d, 228) (now - 45d, 222)" ) + + tdSql.query("select avg(voltage) from st interval(1n)") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 225.0) + tdSql.checkData(1, 1, 220.5) + + tdSql.query("select avg(voltage) from st interval(1n, 15d)") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 225.0) + tdSql.checkData(1, 1, 223.0) + tdSql.checkData(2, 1, 220.0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 601342b333c4da7706b5fe7252a7b87a956c2212 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 18 Sep 2020 10:40:02 +0800 Subject: [PATCH 02/12] [TD-1245] add test case for interval --- tests/pytest/fulltest.sh | 3 ++- tests/pytest/query/queryInterval.py | 19 +++++++++++++++---- tests/pytest/util/sql.py | 4 ++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index c74fb33c3b..b92a7319de 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -187,7 +187,8 @@ python3 ./test.py -f functions/function_top.py #python3 ./test.py -f functions/function_twa.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py -python3 client/twoClients.py +python3 client/twoClients.python3 +python3 test.py -f query/queryInterval.py # tools python3 test.py -f tools/taosdemo.py diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index 45f6988bbe..db2c3fdeec 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -29,18 +29,29 @@ class TDTestCase: tdSql.prepare() tdSql.execute("create table st (ts timestamp, voltage int) tags (loc nchar(30))") - tdSql.execute("insert into t0 using st tags('beijing') values(now, 220) (now - 15d, 221) (now - 30d, 225) (now - 35d, 228) (now - 45d, 222)" ) + tdSql.execute("insert into t0 using st tags('beijing') values(now, 220) (now - 15d, 221) (now - 30d, 225) (now - 35d, 228) (now - 45d, 222)") + tdSql.execute("insert into t1 using st tags('shanghai') values(now, 220) (now - 60d, 221) (now - 50d, 225) (now - 40d, 228) (now - 20d, 222)") tdSql.query("select avg(voltage) from st interval(1n)") - tdSql.checkRows(2) - tdSql.checkData(0, 1, 225.0) - tdSql.checkData(1, 1, 220.5) + tdSql.checkRows(3) + tdSql.checkData(0, 1, 223.0) + tdSql.checkData(1, 1, 225.0) + tdSql.checkData(2, 1, 220.333333) tdSql.query("select avg(voltage) from st interval(1n, 15d)") tdSql.checkRows(3) + tdSql.checkData(0, 1, 224.8) + tdSql.checkData(1, 1, 222.666666) + tdSql.checkData(2, 1, 220.0) + + tdSql.query("select avg(voltage) from st interval(1n, 15d) group by loc") + tdSql.checkRows(6) tdSql.checkData(0, 1, 225.0) tdSql.checkData(1, 1, 223.0) tdSql.checkData(2, 1, 220.0) + tdSql.checkData(3, 1, 224.666666) + tdSql.checkData(4, 1, 222.0) + tdSql.checkData(5, 1, 220.0) def stop(self): tdSql.close() diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 7de8efdfe9..9abec354c6 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -123,8 +123,8 @@ class TDSql: def checkData(self, row, col, data): self.checkRowCol(row, col) - if self.queryResult[row][col] != data: - if str(self.queryResult[row][col]) != str(data): + if self.queryResult[row][col] != data: + if str(self.queryResult[row][col]) == str(data): tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) return From 347a45e24fdc5a00e157b1507d94f011ba2ee20e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 18 Sep 2020 10:53:58 +0800 Subject: [PATCH 03/12] minor changes --- src/plugins/http/src/httpSql.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 881fa55fb7..883fa574ff 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -257,20 +257,20 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo HttpEncodeMethod *encode = pContext->encodeMethod; if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd, - pContext->user, pContext->session->taos, tstrerror(code), (SSqlObj *)result); + httpError("context:%p, fd:%d, user:%s, query error, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd, + pContext->user, tstrerror(code), (SSqlObj *)result); return; } if (code < 0) { SSqlObj *pObj = (SSqlObj *)result; if (code == TSDB_CODE_TSC_INVALID_SQL) { - httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s, sqlObj:%p, error:%s", pContext, - pContext->fd, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload); + httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, + pContext->fd, pContext->user, tstrerror(code), pObj, pObj->cmd.payload); httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload); } else { - httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s, sqlObj:%p", pContext, pContext->fd, - pContext->user, pContext->session->taos, tstrerror(code), pObj); + httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, + pContext->user, tstrerror(code), pObj); httpSendErrorResp(pContext, code); } taos_free_result(result); From 0914c9787cb89fad8857247a5a14315a145af71b Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 18 Sep 2020 11:19:14 +0800 Subject: [PATCH 04/12] Update cluster-ch.md --- documentation20/webdocs/markdowndocs/cluster-ch.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 193de729ba..0b0221f20d 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -2,13 +2,13 @@ 多个TDengine服务器，也就是多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。而且在安装集群之前，先请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验单节点功能。 -集群的每个数据节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令`hostname -f`获取,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。端口是这个数据节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。一个物理节点可能配置了多个hostname, TDengine会自动获取第一个，但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问，可以将参数fqdn设置为本节点的IP地址。 +集群的每个数据节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令`hostname -f`获取（如何配置FQDN，请参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。端口是这个数据节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。一个物理节点可能配置了多个hostname, TDengine会自动获取第一个，但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问，可以将参数fqdn设置为本节点的IP地址。 TDengine的集群管理极其简单，除添加和删除节点需要人工干预之外，其他全部是自动完成，最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 ## 准备工作 -**第零步**：规划集群所有物理节点的FQDN，将规划好的FQDN分别添加到每个物理节点的/etc/hostname；修改每个物理节点的/etc/hosts，将所有集群物理节点的IP与FQDN的对应添加好【如部署了DNS，请联系网络管理员在DNS上做好相关配置】； +**第零步**：如果没有部署DNS服务，请规划集群所有物理节点的FQDN，然后按照[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)里的步骤，将所有集群物理节点的IP与FQDN的对应关系添加好。 **第一步**：如果搭建集群的物理节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) **注意1：**因为FQDN的信息会写进文件，如果之前没有配置或者更改FQDN，且启动了TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（rm -rf /var/lib/taos/）； @@ -22,7 +22,8 @@ TDengine的集群管理极其简单，除添加和删除节点需要人工干预 1. 每个物理节点上执行命令`hostname -f`，查看和确认所有节点的hostname是不相同的(应用驱动所在节点无需做此项检查)； 2. 每个物理节点上执行`ping host`, 其中host是其他物理节点的hostname, 看能否ping通其它物理节点; 如果不能ping通，需要检查网络设置, 或/etc/hosts文件(Windows系统默认路径为C:\Windows\system32\drivers\etc\hosts)，或DNS的配置。如果无法ping通，是无法组成集群的； -3. 每个数据节点的End Point就是输出的hostname外加端口号，比如h1.taosdata.com:6030 +3. 从应用运行的物理节点，ping taosd运行的数据节点，如果无法平通，应用是无法连接taosd的，请检查应用所在物理节点的DNS设置或hosts文件； +4. 每个数据节点的End Point就是输出的hostname外加端口号，比如h1.taosdata.com:6030 **第五步**：修改TDengine的配置文件（所有节点的文件/etc/taos/taos.cfg都需要修改）。假设准备启动的第一个数据节点End Point为 h1.taosdata.com:6030, 其与集群配置相关参数如下： @@ -114,6 +115,8 @@ taos> ## 数据节点管理 +上面已经介绍如何从零开始搭建集群。集群组建完后，还可以随时添加新的数据节点进行扩容，或删除数据节点，并检查集群当前状态。 + ### 添加数据节点 执行CLI程序taos, 使用root账号登录进系统, 执行: From f43b7db39b89d928de78037a3e98c8c30e23fd52 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 18 Sep 2020 11:39:52 +0800 Subject: [PATCH 05/12] Update cluster-ch.md --- documentation20/webdocs/markdowndocs/cluster-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 0b0221f20d..24592d4114 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -2,13 +2,13 @@ 多个TDengine服务器，也就是多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。而且在安装集群之前，先请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验单节点功能。 -集群的每个数据节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令`hostname -f`获取（如何配置FQDN，请参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。端口是这个数据节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。一个物理节点可能配置了多个hostname, TDengine会自动获取第一个，但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问，可以将参数fqdn设置为本节点的IP地址。 +集群的每个数据节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令`hostname -f`获取（如何配置FQDN，请参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)）。端口是这个数据节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。一个物理节点可能配置了多个hostname, TDengine会自动获取第一个，但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问，可以将参数fqdn设置为本节点的IP地址。 TDengine的集群管理极其简单，除添加和删除节点需要人工干预之外，其他全部是自动完成，最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 ## 准备工作 -**第零步**：如果没有部署DNS服务，请规划集群所有物理节点的FQDN，然后按照[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)里的步骤，将所有集群物理节点的IP与FQDN的对应关系添加好。 +**第零步**：如果没有部署DNS服务，请规划集群所有物理节点的FQDN，然后按照《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》里的步骤，将所有集群物理节点的IP与FQDN的对应关系添加好。 **第一步**：如果搭建集群的物理节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) **注意1：**因为FQDN的信息会写进文件，如果之前没有配置或者更改FQDN，且启动了TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（rm -rf /var/lib/taos/）； From 5c1362815cc25fecb1482767a8c38b4b177c1d16 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 18 Sep 2020 11:54:37 +0800 Subject: [PATCH 06/12] Update cluster-ch.md --- documentation20/webdocs/markdowndocs/cluster-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 24592d4114..25ab9b9107 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -22,7 +22,7 @@ TDengine的集群管理极其简单，除添加和删除节点需要人工干预 1. 每个物理节点上执行命令`hostname -f`，查看和确认所有节点的hostname是不相同的(应用驱动所在节点无需做此项检查)； 2. 每个物理节点上执行`ping host`, 其中host是其他物理节点的hostname, 看能否ping通其它物理节点; 如果不能ping通，需要检查网络设置, 或/etc/hosts文件(Windows系统默认路径为C:\Windows\system32\drivers\etc\hosts)，或DNS的配置。如果无法ping通，是无法组成集群的； -3. 从应用运行的物理节点，ping taosd运行的数据节点，如果无法平通，应用是无法连接taosd的，请检查应用所在物理节点的DNS设置或hosts文件； +3. 从应用运行的物理节点，ping taosd运行的数据节点，如果无法ping通，应用是无法连接taosd的，请检查应用所在物理节点的DNS设置或hosts文件； 4. 每个数据节点的End Point就是输出的hostname外加端口号，比如h1.taosdata.com:6030 **第五步**：修改TDengine的配置文件（所有节点的文件/etc/taos/taos.cfg都需要修改）。假设准备启动的第一个数据节点End Point为 h1.taosdata.com:6030, 其与集群配置相关参数如下： From dd800e772c2bdc0cf5581b2e942ab46427a38ea5 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 18 Sep 2020 12:02:47 +0800 Subject: [PATCH 07/12] Update architecture-ch.md --- documentation20/webdocs/markdowndocs/architecture-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index a279875649..460e655f5f 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -67,7 +67,7 @@ TDengine 分布式架构的逻辑结构图如下：
图 1 TDengine架构示意图
一个完整的 TDengine 系统是运行在一到多个物理节点上的，逻辑上，它包含数据节点(dnode)、TDengine客户端(taosc)以及应用(app)。系统中存在一到多个数据节点，这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。 -**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。 +**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》。 **数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例，一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE)，零或者至多一个逻辑的管理节点(mnode)。dnode在系统中的唯一标识由实例的End Point (EP )决定。EP是dnode所在物理节点的FQDN (Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口，一个物理节点(一台物理机、虚拟机或容器）可以运行多个实例，或有多个数据节点。 From f4f4a07865c4f7f2715e56e45604d4d21542fc22 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 18 Sep 2020 12:51:14 +0800 Subject: [PATCH 08/12] update TAOS SQL --- documentation20/webdocs/markdowndocs/TAOS SQL-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index 3c7f6987c7..e0acaee137 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -1019,5 +1019,5 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER - 表名最大长度为193，每行数据最大长度16k个字符 - 列名最大长度为65，最多允许1024列，最少需要2列，第一列必须是时间戳 - 标签最多允许128个，可以0个，标签总长度不超过16k个字符 -- SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改，最长可配置为8M +- SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改，最长可配置为1M - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 From 4c834753ad3c419e88c1258ad5b52bf2f0e65fd1 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 21 Sep 2020 10:37:20 +0800 Subject: [PATCH 09/12] Update cluster-ch.md --- documentation20/webdocs/markdowndocs/cluster-ch.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 25ab9b9107..c36819e5c7 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -31,8 +31,8 @@ TDengine的集群管理极其简单，除添加和删除节点需要人工干预 // firstEp 是每个数据节点首次启动后连接的第一个数据节点 firstEp h1.taosdata.com:6030 -// 配置本数据节点的FQDN，如果本机只有一个hostname, 无需配置 -fqdn h1.taosdata.com +// 必须配置为本数据节点的FQDN，如果本机只有一个hostname, 可注释掉本配置 +fqdn h1.taosdata.com // 配置本数据节点的端口号，缺省是6030 serverPort 6030 @@ -41,7 +41,7 @@ serverPort 6030 arbitrator ha.taosdata.com:6042 ``` -一定要修改的参数是firstEp和fqdn, 其他参数可不做任何修改，除非你很清楚为什么要修改。 +一定要修改的参数是firstEp和fqdn。在每个数据节点，firstEp需全部配置成一样，**但fqdn一定要配置成其所在数据节点的值**。其他参数可不做任何修改，除非你很清楚为什么要修改。 **加入到集群中的数据节点dnode，涉及集群相关的下表11项参数必须完全相同，否则不能成功加入到集群中。** From 1f73c106b91e711880d772bbdae9d432bddcac69 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 21 Sep 2020 11:56:33 +0800 Subject: [PATCH 10/12] TD-1529: stream async create db connection and also fix failed test cases related to stream. --- src/client/src/tscSql.c | 10 +++++++++- src/client/src/tscStream.c | 5 +++++ src/cq/CMakeLists.txt | 2 ++ src/cq/src/cqMain.c | 18 ++++++++++++------ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 69bc69cd4a..1df77d1850 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -236,13 +236,21 @@ TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t us return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); } +static void asyncConnCallback(void *param, TAOS_RES *tres, int code) { + SSqlObj *pSql = (SSqlObj *) tres; + assert(pSql != NULL); + + pSql->fetchFp(pSql->param, tres, code); +} + TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { - SSqlObj* pSql = taosConnectImpl(ip, user, pass, NULL, db, port, fp, param, taos); + SSqlObj* pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, taos); if (pSql == NULL) { return NULL; } + pSql->fetchFp = fp; pSql->res.code = tscProcessSql(pSql); tscDebug("%p DB async connection is opening", taos); return taos; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 4a1f4d9d87..93a865a78b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -515,6 +515,10 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { return; } + uint64_t handle = (uint64_t) pSql; + pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); + T_REF_INC(pSql->pTscObj); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -608,6 +612,7 @@ void taos_close_stream(TAOS_STREAM *handle) { * Here, we need a check before release memory */ if (pSql->signature == pSql) { + T_REF_DEC(pSql->pTscObj); tscRemoveFromStreamList(pStream, pSql); taosTmrStopA(&(pStream->pTimer)); diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index db366639ef..e631397348 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -2,6 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) IF (TD_LINUX) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index d8f68f66a5..889cc84374 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -21,6 +21,7 @@ #include #include "taos.h" +#include "tsclient.h" #include "taosdef.h" #include "taosmsg.h" #include "ttimer.h" @@ -238,18 +239,23 @@ void cqDrop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } +static void doCreateStream(void *param, TAOS_RES *result, int code) { + SCqObj* pObj = (SCqObj*)param; + SCqContext* pContext = pObj->pContext; + SSqlObj* pSql = (SSqlObj*)result; + pContext->dbConn = pSql->pTscObj; + cqCreateStream(pContext, pObj); +} + static void cqProcessCreateTimer(void *param, void *tmrId) { SCqObj* pObj = (SCqObj*)param; SCqContext* pContext = pObj->pContext; if (pContext->dbConn == NULL) { - pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, pContext->db, 0); - if (pContext->dbConn == NULL) { - cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno)); - } + taos_connect_a(NULL, pContext->user, pContext->pass, pContext->db, 0, doCreateStream, param, NULL); + } else { + cqCreateStream(pContext, pObj); } - - cqCreateStream(pContext, pObj); } static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { From d5f21eda176d13f91414406ebfd6c0dd0295657e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 21 Sep 2020 11:56:33 +0800 Subject: [PATCH 11/12] TD-1529: stream async create db connection and also fix failed test cases related to stream. --- src/client/src/tscSql.c | 10 +++++++++- src/client/src/tscStream.c | 5 +++++ src/cq/CMakeLists.txt | 2 ++ src/cq/src/cqMain.c | 18 ++++++++++++------ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 69bc69cd4a..1df77d1850 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -236,13 +236,21 @@ TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t us return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); } +static void asyncConnCallback(void *param, TAOS_RES *tres, int code) { + SSqlObj *pSql = (SSqlObj *) tres; + assert(pSql != NULL); + + pSql->fetchFp(pSql->param, tres, code); +} + TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { - SSqlObj* pSql = taosConnectImpl(ip, user, pass, NULL, db, port, fp, param, taos); + SSqlObj* pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, taos); if (pSql == NULL) { return NULL; } + pSql->fetchFp = fp; pSql->res.code = tscProcessSql(pSql); tscDebug("%p DB async connection is opening", taos); return taos; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 4a1f4d9d87..93a865a78b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -515,6 +515,10 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { return; } + uint64_t handle = (uint64_t) pSql; + pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000); + T_REF_INC(pSql->pTscObj); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -608,6 +612,7 @@ void taos_close_stream(TAOS_STREAM *handle) { * Here, we need a check before release memory */ if (pSql->signature == pSql) { + T_REF_DEC(pSql->pTscObj); tscRemoveFromStreamList(pStream, pSql); taosTmrStopA(&(pStream->pTimer)); diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index db366639ef..e631397348 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -2,6 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) IF (TD_LINUX) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index d8f68f66a5..889cc84374 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -21,6 +21,7 @@ #include #include "taos.h" +#include "tsclient.h" #include "taosdef.h" #include "taosmsg.h" #include "ttimer.h" @@ -238,18 +239,23 @@ void cqDrop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } +static void doCreateStream(void *param, TAOS_RES *result, int code) { + SCqObj* pObj = (SCqObj*)param; + SCqContext* pContext = pObj->pContext; + SSqlObj* pSql = (SSqlObj*)result; + pContext->dbConn = pSql->pTscObj; + cqCreateStream(pContext, pObj); +} + static void cqProcessCreateTimer(void *param, void *tmrId) { SCqObj* pObj = (SCqObj*)param; SCqContext* pContext = pObj->pContext; if (pContext->dbConn == NULL) { - pContext->dbConn = taos_connect("localhost", pContext->user, pContext->pass, pContext->db, 0); - if (pContext->dbConn == NULL) { - cError("vgId:%d, failed to connect to TDengine(%s)", pContext->vgId, tstrerror(terrno)); - } + taos_connect_a(NULL, pContext->user, pContext->pass, pContext->db, 0, doCreateStream, param, NULL); + } else { + cqCreateStream(pContext, pObj); } - - cqCreateStream(pContext, pObj); } static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { From 1a5272a49f85994c6e7e10685e5ea5e5938fd692 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Sep 2020 04:34:29 +0000 Subject: [PATCH 12/12] TD-1520 --- src/client/src/tscSQLParser.c | 1 + src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 14 +++++++++++++- src/cq/src/cqMain.c | 4 ++-- src/mnode/src/mnodeTable.c | 4 ++-- tests/script/sh/deploy.sh | 1 + tests/script/tmp/prepare.sim | 4 ++-- tests/script/unique/cluster/vgroup100.sim | 4 +++- 8 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4555b0e08d..a6efffe436 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4873,6 +4873,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { {"cDebugFlag", 10}, {"httpDebugFlag", 13}, {"qDebugflag", 10}, {"sdbDebugFlag", 12}, {"uDebugFlag", 10}, {"tsdbDebugFlag", 13}, {"sDebugflag", 10}, {"rpcDebugFlag", 12}, {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, + {"cqDebugFlag", 11}, }; SStrToken* pOptionToken = &pOptions->a[1]; diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 77e8b76456..798e265455 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -174,6 +174,7 @@ extern int32_t rpcDebugFlag; extern int32_t odbcDebugFlag; extern int32_t qDebugFlag; extern int32_t wDebugFlag; +extern int32_t cqDebugFlag; extern int32_t debugFlag; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 7e46f58a93..94ec6fde0f 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -203,6 +203,7 @@ int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; +int32_t cqDebugFlag = 135; int32_t (*monitorStartSystemFp)() = NULL; void (*monitorStopSystemFp)() = NULL; @@ -222,12 +223,13 @@ void taosSetAllDebugFlag() { httpDebugFlag = debugFlag; mqttDebugFlag = debugFlag; monitorDebugFlag = debugFlag; + qDebugFlag = debugFlag; rpcDebugFlag = debugFlag; uDebugFlag = debugFlag; sDebugFlag = debugFlag; wDebugFlag = debugFlag; tsdbDebugFlag = debugFlag; - qDebugFlag = debugFlag; + cqDebugFlag = debugFlag; uInfo("all debug flag are set to %d", debugFlag); } } @@ -1209,6 +1211,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "cqDebugFlag"; + cfg.ptr = &cqDebugFlag; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; + cfg.minValue = 0; + cfg.maxValue = 255; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "tscEnableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 889cc84374..1a99a84b8e 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -66,8 +66,6 @@ typedef struct SCqObj { SCqContext * pContext; } SCqObj; -int cqDebugFlag = 135; - static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row); static void cqCreateStream(SCqContext *pContext, SCqObj *pObj); @@ -252,6 +250,7 @@ static void cqProcessCreateTimer(void *param, void *tmrId) { SCqContext* pContext = pObj->pContext; if (pContext->dbConn == NULL) { + cDebug("vgId:%d, try connect to TDengine", pContext->vgId); taos_connect_a(NULL, pContext->user, pContext->pass, pContext->db, 0, doCreateStream, param, NULL); } else { cqCreateStream(pContext, pObj); @@ -262,6 +261,7 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { pObj->pContext = pContext; if (pContext->dbConn == NULL) { + cDebug("vgId:%d, create dbConn after 1000 ms", pContext->vgId); pObj->tmrId = taosTmrStart(cqProcessCreateTimer, 1000, pObj, pContext->tmrCtrl); return; } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 4400927e9b..a35f09cd8d 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2122,8 +2122,8 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { } pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId); - mDebug("app:%p:%p, table:%s, uid:%" PRIu64 " table meta is retrieved", pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId, pTable->uid); + mDebug("app:%p:%p, table:%s, uid:%" PRIu64 " table meta is retrieved, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId, pTable->uid, pTable->vgId, pTable->sid); return TSDB_CODE_SUCCESS; } diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 0d444a5a6e..8fccb1442f 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -128,6 +128,7 @@ echo "tmrDebugFlag 131" >> $TAOS_CFG echo "udebugFlag 135" >> $TAOS_CFG echo "sdebugFlag 135" >> $TAOS_CFG echo "wdebugFlag 135" >> $TAOS_CFG +echo "cqdebugFlag 135" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index 8b8f206233..343c422e9f 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -34,11 +34,11 @@ system sh/cfg.sh -n dnode4 -c http -v 1 return # for crash_gen -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 10 system sh/cfg.sh -n dnode1 -c rpcMaxTime -v 101 system sh/cfg.sh -n dnode1 -c cache -v 2 system sh/cfg.sh -n dnode1 -c keep -v 36500 -system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 # for windows diff --git a/tests/script/unique/cluster/vgroup100.sim b/tests/script/unique/cluster/vgroup100.sim index cddb38cefd..bde6dd2462 100644 --- a/tests/script/unique/cluster/vgroup100.sim +++ b/tests/script/unique/cluster/vgroup100.sim @@ -42,9 +42,11 @@ $count = 2 while $count < 102 $db = d . $count $tb = $db . .t + $tb2 = $db . .t2 sql create database $db replica 3 cache 1 blocks 3 sql create table $tb (ts timestamp, i int) sql insert into $tb values(now, 1) + sql create table $tb2 as select count(*) from $tb interval(10s) $count = $count + 1 print insert into $tb values(now, 1) ==> finished endw @@ -74,7 +76,7 @@ print ============================== step6 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 10000 print ============================== step7