From c2fcab0a0d95f15bc736db0cf8bbb76658a44c80 Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 7 Jun 2021 20:44:07 +0800 Subject: [PATCH 001/115] [TD-4312]: add testcase of nested query with interval function --- .../pytest/query/nestedQuery/insertData.json | 62 ++++++++++++++ .../query/nestedQuery/nestedQueryJson.py | 81 +++++++++++++++++++ .../pytest/query/nestedQuery/queryInterval.py | 76 +++++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 tests/pytest/query/nestedQuery/insertData.json create mode 100644 tests/pytest/query/nestedQuery/nestedQueryJson.py create mode 100644 tests/pytest/query/nestedQuery/queryInterval.py diff --git a/tests/pytest/query/nestedQuery/insertData.json b/tests/pytest/query/nestedQuery/insertData.json new file mode 100644 index 0000000000..d4ef8dbe97 --- /dev/null +++ b/tests/pytest/query/nestedQuery/insertData.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file":"./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100000, + "childtable_limit": -1, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1, + "timestamp_step": 1000, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/sample.csv", + "tags_file": "", + "columns": [{"type": "INT", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BOOL"}], + "tags": [{"type": "TINYINT", "count":1}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/query/nestedQuery/nestedQueryJson.py b/tests/pytest/query/nestedQuery/nestedQueryJson.py new file mode 100644 index 0000000000..36a231a916 --- /dev/null +++ b/tests/pytest/query/nestedQuery/nestedQueryJson.py @@ -0,0 +1,81 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + os.system("%staosdemo -f query/nestedQuery/insertData.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1000) + tdSql.query("select count (tbname) from stb1") + tdSql.checkData(0, 0, 1000) + tdSql.query("select count(*) from stb00_0") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 100000) + tdSql.query("select count(*) from stb01_1") + tdSql.checkData(0, 0, 200) + tdSql.query("select count(*) from stb1") + tdSql.checkData(0, 0, 200000) + + + + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf query/nestedQuery/%s.sql" % testcaseFilename ) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/nestedQuery/queryInterval.py b/tests/pytest/query/nestedQuery/queryInterval.py new file mode 100644 index 0000000000..fc94cdf4eb --- /dev/null +++ b/tests/pytest/query/nestedQuery/queryInterval.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +import random + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts1 = 1593548685000 + self.ts2 = 1593548785000 + + + def run(self): + # tdSql.execute("drop database db ") + tdSql.prepare() + tdSql.execute("create table st (ts timestamp, num int, value int , t_instance int) tags (loc nchar(30))") + number = 20 + for n in range(number): + dt= n*300000 # collecting'frequency is 10s + args1=(self.ts1+dt,n,100+n,10+n) + args2=(self.ts2+dt,n,120+n,15+n) + tdSql.execute("insert into t0 using st tags('beijing') values(%d, %d, %d, %d)" % args1) + tdSql.execute("insert into t1 using st tags('shanghai') values(%d, %d, %d, %d)" % args2) + + + tdSql.query("select avg(value) from st interval(10m)") + print(tdSql.queryResult) + tdSql.checkRows(11) + tdSql.checkData(0, 0, "2020-07-01 04:20:00") + tdSql.query("select avg_val from(select avg(value) as avg_val from st where loc='beijing' interval(10m));") + # tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing' interval(10m));") + print(tdSql.queryResult) + tdSql.checkData(0, 0, 109.5) + + + # tdSql.query("select avg(voltage) from st interval(1n, 15d)") + + # tdSql.query("select avg(voltage) from st interval(1n, 15d) group by loc") + + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("select last(*) from t interval(1s)") + + + # tdSql.query("select first(ts),twa(c) from tb interval(14a)") + # tdSql.checkRows(6) + + # tdSql.query("select twa(c) from tb group by c") + # tdSql.checkRows(4) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 3f940428cb8e9f6dd7b9ca69536d7ed907964c04 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 9 Jun 2021 16:25:49 +0800 Subject: [PATCH 002/115] modify nestedquery testcase --- .../pytest/query/nestedQuery/queryInterval.py | 67 +++++++++++++++---- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/tests/pytest/query/nestedQuery/queryInterval.py b/tests/pytest/query/nestedQuery/queryInterval.py index fc94cdf4eb..a7c06a18f9 100644 --- a/tests/pytest/query/nestedQuery/queryInterval.py +++ b/tests/pytest/query/nestedQuery/queryInterval.py @@ -32,25 +32,64 @@ class TDTestCase: # tdSql.execute("drop database db ") tdSql.prepare() tdSql.execute("create table st (ts timestamp, num int, value int , t_instance int) tags (loc nchar(30))") - number = 20 - for n in range(number): - dt= n*300000 # collecting'frequency is 10s - args1=(self.ts1+dt,n,100+n,10+n) - args2=(self.ts2+dt,n,120+n,15+n) - tdSql.execute("insert into t0 using st tags('beijing') values(%d, %d, %d, %d)" % args1) - tdSql.execute("insert into t1 using st tags('shanghai') values(%d, %d, %d, %d)" % args2) + node = 5 + number = 10 + for n in range(node): + for m in range(number): + dt= m*300000+n*60000 # collecting'frequency is 10s + args1=(n,n,self.ts1+dt,n,100+2*m+2*n,10+m+n) + # args2=(n,self.ts2+dt,n,120+n,15+n) + tdSql.execute("insert into t%d using st tags('beijing%d') values(%d, %d, %d, %d)" % args1) + # tdSql.execute("insert into t1 using st tags('shanghai') values(%d, %d, %d, %d)" % args2) - + # interval function tdSql.query("select avg(value) from st interval(10m)") print(tdSql.queryResult) - tdSql.checkRows(11) + tdSql.checkRows(6) tdSql.checkData(0, 0, "2020-07-01 04:20:00") - tdSql.query("select avg_val from(select avg(value) as avg_val from st where loc='beijing' interval(10m));") - # tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing' interval(10m));") - print(tdSql.queryResult) - tdSql.checkData(0, 0, 109.5) - + tdSql.checkData(1, 1, 107.4) + + # subquery with interval + tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing0' interval(10m));") + tdSql.checkData(0, 0, 109.0) + + # subquery with interval and select two Column in parent query + # tdSql.query("select ts,avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing0' interval(10m));") + + # subquery with interval and sliding + tdSql.query("select avg(value) as avg_val from st where loc='beijing0' interval(8m) sliding(30s) limit 1;") + tdSql.checkData(0, 0, "2020-07-01 04:17:00") + tdSql.checkData(0, 1, 107.4) + tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing1' interval(8m) sliding(30s));") + tdSql.checkData(0, 0, 111) + # subquery with interval and offset + tdSql.query("select avg(value) as avg_val from st where loc='beijing0' interval(5m,1m);") + tdSql.checkData(0, 0, "2020-07-01 04:21:00") + tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing0' interval(5m,1m));") + tdSql.checkData(0, 0, 111) + + # subquery with interval,sliding and group by ; parent query with interval + tdSql.query("select avg(value) as avg_val from st where loc='beijing0' interval(8m) sliding(1m) group by loc limit 1 offset 52 ;") + tdSql.checkData(0, 0, "2020-07-01 05:09:00") + tdSql.checkData(0, 1, 118) + tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc!='beijing0' interval(8m) sliding(1m) group by loc )interval(5m);") + tdSql.checkData(0, 0, 111) + + # # subquery and parent query with interval and sliding + # tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing1' interval(8m) sliding(30s)) interval(5m) sliding(1s);") + # tdSql.checkData(0, 0, 111) + + # subquery and parent query with top and bottom + tdSql.query("select top(avg_val,2) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num);") + tdSql.checkData(0, 0, 111) + tdSql.query("select bottom(avg_val,2) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num);") + tdSql.checkData(0, 0, 111) + + + + + # tdSql.query("select avg(voltage) from st interval(1n, 15d)") # tdSql.query("select avg(voltage) from st interval(1n, 15d) group by loc") From 2c83206d657dd94ddd16b160575e0bd8018d3d87 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 9 Jun 2021 19:26:17 +0800 Subject: [PATCH 003/115] [TD-4314]: add testcase of nested query with top/bottom function --- .../pytest/query/nestedQuery/queryInterval.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/pytest/query/nestedQuery/queryInterval.py b/tests/pytest/query/nestedQuery/queryInterval.py index a7c06a18f9..e346b26c60 100644 --- a/tests/pytest/query/nestedQuery/queryInterval.py +++ b/tests/pytest/query/nestedQuery/queryInterval.py @@ -44,7 +44,7 @@ class TDTestCase: # interval function tdSql.query("select avg(value) from st interval(10m)") - print(tdSql.queryResult) + # print(tdSql.queryResult) tdSql.checkRows(6) tdSql.checkData(0, 0, "2020-07-01 04:20:00") tdSql.checkData(1, 1, 107.4) @@ -54,37 +54,38 @@ class TDTestCase: tdSql.checkData(0, 0, 109.0) # subquery with interval and select two Column in parent query - # tdSql.query("select ts,avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing0' interval(10m));") + tdSql.error("select ts,avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing0' interval(10m));") # subquery with interval and sliding tdSql.query("select avg(value) as avg_val from st where loc='beijing0' interval(8m) sliding(30s) limit 1;") tdSql.checkData(0, 0, "2020-07-01 04:17:00") - tdSql.checkData(0, 1, 107.4) + tdSql.checkData(0, 1, 100) tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing1' interval(8m) sliding(30s));") tdSql.checkData(0, 0, 111) # subquery with interval and offset tdSql.query("select avg(value) as avg_val from st where loc='beijing0' interval(5m,1m);") tdSql.checkData(0, 0, "2020-07-01 04:21:00") - tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing0' interval(5m,1m));") - tdSql.checkData(0, 0, 111) + tdSql.checkData(0, 1, 100) + tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing0' interval(5m,1m) group by loc);") + tdSql.checkData(0, 0, 109) # subquery with interval,sliding and group by ; parent query with interval tdSql.query("select avg(value) as avg_val from st where loc='beijing0' interval(8m) sliding(1m) group by loc limit 1 offset 52 ;") tdSql.checkData(0, 0, "2020-07-01 05:09:00") tdSql.checkData(0, 1, 118) - tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc!='beijing0' interval(8m) sliding(1m) group by loc )interval(5m);") - tdSql.checkData(0, 0, 111) + tdSql.query("select avg(avg_val) as ncst from(select avg(value) as avg_val from st where loc!='beijing0' interval(8m) sliding(1m) group by loc ) interval(5m);") + tdSql.checkData(1, 1, 105) # # subquery and parent query with interval and sliding - # tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing1' interval(8m) sliding(30s)) interval(5m) sliding(1s);") - # tdSql.checkData(0, 0, 111) + tdSql.query("select avg(avg_val) from(select avg(value) as avg_val from st where loc='beijing1' interval(8m) sliding(5m)) interval(10m) sliding(2m);") + tdSql.checkData(29, 0, "2020-07-01 05:10:00.000") # subquery and parent query with top and bottom tdSql.query("select top(avg_val,2) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num);") - tdSql.checkData(0, 0, 111) + tdSql.checkData(0, 1, 115) tdSql.query("select bottom(avg_val,2) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num);") - tdSql.checkData(0, 0, 111) + tdSql.checkData(1, 1, 111) From 176f1e018d406a720a3b05ad1ca6c3ce61ccc3fe Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 10 Jun 2021 15:51:55 +0800 Subject: [PATCH 004/115] [TD-4314]: add testcase of nested query with top/bottom function --- .../pytest/query/nestedQuery/queryInterval.py | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/tests/pytest/query/nestedQuery/queryInterval.py b/tests/pytest/query/nestedQuery/queryInterval.py index e346b26c60..3ddf751b7f 100644 --- a/tests/pytest/query/nestedQuery/queryInterval.py +++ b/tests/pytest/query/nestedQuery/queryInterval.py @@ -12,6 +12,7 @@ # -*- coding: utf-8 -*- import sys +import os import taos from util.log import tdLog from util.cases import tdCases @@ -84,28 +85,15 @@ class TDTestCase: # subquery and parent query with top and bottom tdSql.query("select top(avg_val,2) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num);") tdSql.checkData(0, 1, 115) - tdSql.query("select bottom(avg_val,2) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num);") - tdSql.checkData(1, 1, 111) + tdSql.query("select bottom(avg_val,3) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num);") + tdSql.checkData(0, 1, 125) + # + tdSql.query("select top(avg_val,2) from(select avg(value) as avg_val from st where loc='beijing1' interval(8m) sliding(3m));") - - - - # tdSql.query("select avg(voltage) from st interval(1n, 15d)") - - # tdSql.query("select avg(voltage) from st interval(1n, 15d) group by loc") - - # tdDnodes.stop(1) - # tdDnodes.start(1) - # tdSql.query("select last(*) from t interval(1s)") - - - # tdSql.query("select first(ts),twa(c) from tb interval(14a)") - # tdSql.checkRows(6) - - # tdSql.query("select twa(c) from tb group by c") - # tdSql.checkRows(4) - + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf wal/%s.sql" % testcaseFilename ) def stop(self): tdSql.close() From 9e72c69c8913ddeb82c1db171d41b76218c7bde2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 16 Jun 2021 18:10:15 +0800 Subject: [PATCH 005/115] change c++ std option --- cmake/define.inc | 14 +++++++------- cmake/env.inc | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index a15e0aecbb..d6a3f2b915 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -57,7 +57,7 @@ IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) MESSAGE(STATUS "linux64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) IF (JEMALLOC_ENABLED) @@ -70,7 +70,7 @@ IF (TD_LINUX_32) ADD_DEFINITIONS(-D_TD_LINUX_32) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "linux32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_64) @@ -78,7 +78,7 @@ IF (TD_ARM_64) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_32) @@ -86,7 +86,7 @@ IF (TD_ARM_32) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") ENDIF () IF (TD_MIPS_64) @@ -94,7 +94,7 @@ IF (TD_MIPS_64) ADD_DEFINITIONS(-D_TD_MIPS_64) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_MIPS_32) @@ -102,7 +102,7 @@ IF (TD_MIPS_32) ADD_DEFINITIONS(-D_TD_MIPS_32) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_APLHINE) @@ -147,7 +147,7 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "darwin64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") IF (TD_MEMORY_SANITIZER) SET(DEBUG_FLAGS "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -O0 -g3 -DDEBUG") ELSE () diff --git a/cmake/env.inc b/cmake/env.inc index 6c1ce8fd89..15cd5e1880 100755 --- a/cmake/env.inc +++ b/cmake/env.inc @@ -39,9 +39,9 @@ SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}") SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}") # Set c++ compiler options -# SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11") -# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}") -# SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") +SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11") +SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}") +SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") IF (${CMAKE_BUILD_TYPE} MATCHES "Debug") SET(CMAKE_BUILD_TYPE "Debug") From c6d6ba3fd1da2e0d92e66417d023ae0dfd60a427 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 16 Jun 2021 20:54:08 +0800 Subject: [PATCH 006/115] [TD-45145]: add testcase of stat_window function --- tests/pytest/query/queryStateWindow.py | 92 ++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/pytest/query/queryStateWindow.py diff --git a/tests/pytest/query/queryStateWindow.py b/tests/pytest/query/queryStateWindow.py new file mode 100644 index 0000000000..027ece89af --- /dev/null +++ b/tests/pytest/query/queryStateWindow.py @@ -0,0 +1,92 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.rowNum = 100000 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "create table if not exists st (ts timestamp, t1 int, t2 timestamp, t3 bigint, t4 float, t5 double, t6 binary(10), t7 smallint, t8 tinyint, t9 bool, t10 nchar(10), t11 int unsigned, t12 bigint unsigned, t13 smallint unsigned, t14 tinyint unsigned ) tags(dev nchar(50), tag2 binary(16))") + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags("dev_01", "tag_01")') + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev_02", "tag_02")') + + print("==============step2") + + tdSql.execute( + "INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.000', 1, '2020-05-13 10:00:00.000', 10, 3.1, 3.14, 'test', -10, -126, true, '测试', 15, 10, 65534, 254)('2020-05-13 10:00:01.000', 1, '2020-05-13 10:00:01.000', 10, 3.1, 3.14, 'test', -10, -126, true, '测试', 15, 10, 65534, 253)('2020-05-13 10:00:02.000', 10, '2020-05-13 10:00:00.000', 11, 3.1, 3.14, 'test', 10, -127, true, '测试', 15, 10, 65534, 253)('2020-05-13 10:00:03.000', 1, '2020-05-13 10:00:00.000', 11, 3.1, 3.14, 'test', -10, -126, true, '测试', 14, 12, 65532, 254)") + + for i in range(self.rowNum): + tdSql.execute("insert into dev_002 (ts,t1) values(%d, %d,)" % (self.ts + i, i + 1)) + + tdSql.query("select count(ts) from dev_001 state_window(t1)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 2) + tdSql.query("select count(ts) from dev_001 state_window(t3)") + tdSql.checkRows(2) + tdSql.checkData(1, 0, 2) + tdSql.query("select count(ts) from dev_001 state_window(t7)") + tdSql.checkRows(3) + tdSql.checkData(1, 0, 1) + tdSql.query("select count(ts) from dev_001 state_window(t8)") + tdSql.checkRows(3) + tdSql.checkData(2, 0, 1) + tdSql.query("select count(ts) from dev_001 state_window(t11)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 3) + tdSql.query("select count(ts) from dev_001 state_window(t12)") + tdSql.checkRows(2) + tdSql.checkData(1, 0, 1) + tdSql.query("select count(ts) from dev_001 state_window(t13)") + tdSql.checkRows(2) + tdSql.checkData(1, 0, 1) + tdSql.query("select count(ts) from dev_001 state_window(t14)") + tdSql.checkRows(3) + tdSql.checkData(1, 0, 2) + tdSql.query("select count(ts) from dev_002 state_window(t1)") + tdSql.checkRows(100000) + + # error + tdSql.query("select count(ts) from dev_001 state_window(t7)") + tdSql.error("select count(*) from dev_001 state_window(t2)") + tdSql.error("select count(*) from st state_window(t3)") + tdSql.error("select count(*) from dev_001 state_window(t4)") + tdSql.error("select count(*) from dev_001 state_window(t5)") + tdSql.error("select count(*) from dev_001 state_window(t6)") + tdSql.error("select count(*) from dev_001 state_window(t9)") + tdSql.error("select count(*) from dev_001 state_window(t10)") + tdSql.error("select count(*) from dev_001 state_window(tag2)") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 0364437022c4882a4e6803b028cfe13558ac56c0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Jun 2021 12:03:12 +0800 Subject: [PATCH 007/115] [TD-4672]: remove peer fqdn IP caching --- src/sync/inc/syncInt.h | 1 + src/sync/src/syncMain.c | 32 ++++++++++++++++++++++++-------- src/sync/src/syncRetrieve.c | 5 ++++- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index ec6dfcbc82..411e706fb1 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -132,6 +132,7 @@ void * syncRestoreData(void *param); int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); void syncRestartConnection(SSyncPeer *pPeer); void syncBroadcastStatus(SSyncNode *pNode); +uint32_t syncResolvePeerFqdn(SSyncPeer *pPeer); SSyncPeer *syncAcquirePeer(int64_t rid); void syncReleasePeer(SSyncPeer *pPeer); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 8ce37cbf78..829c71a0a5 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -559,7 +559,7 @@ static void syncClosePeerConn(SSyncPeer *pPeer) { static void syncRemovePeer(SSyncPeer *pPeer) { sInfo("%s, it is removed", pPeer->id); - pPeer->ip = 0; + //pPeer->ip = 0; syncClosePeerConn(pPeer); //taosRemoveRef(tsPeerRefId, pPeer->rid); syncReleasePeer(pPeer); @@ -585,20 +585,31 @@ static void syncStopCheckPeerConn(SSyncPeer *pPeer) { sDebug("%s, stop check peer connection", pPeer->id); } +uint32_t syncResolvePeerFqdn(SSyncPeer *pPeer) { + uint32_t ip = taosGetIpv4FromFqdn(pPeer->fqdn); + if (ip == 0xFFFFFFFF) { + sError("failed to resolve peer fqdn:%s since %s", pPeer->fqdn, strerror(errno)); + terrno = TSDB_CODE_RPC_FQDN_ERROR; + return 0; + } + + return ip; +} + static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { - uint32_t ip = taosGetIpv4FromFqdn(pInfo->nodeFqdn); + /*uint32_t ip = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ip == 0xFFFFFFFF) { sError("failed to add peer, can resolve fqdn:%s since %s", pInfo->nodeFqdn, strerror(errno)); terrno = TSDB_CODE_RPC_FQDN_ERROR; return NULL; } - + */ SSyncPeer *pPeer = calloc(1, sizeof(SSyncPeer)); if (pPeer == NULL) return NULL; pPeer->nodeId = pInfo->nodeId; tstrncpy(pPeer->fqdn, pInfo->nodeFqdn, sizeof(pPeer->fqdn)); - pPeer->ip = ip; + //pPeer->ip = ip; pPeer->port = pInfo->nodePort; pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0; snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, nodeId:%d", pNode->vgId, pPeer->nodeId); @@ -864,7 +875,8 @@ static void syncRestartPeer(SSyncPeer *pPeer) { } void syncRestartConnection(SSyncPeer *pPeer) { - if (pPeer->ip == 0) return; + //if (pPeer->ip == 0) return; + if (!syncResolvePeerFqdn(pPeer)) return; if (syncAcquirePeer(pPeer->rid) == NULL) return; @@ -878,7 +890,8 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; sInfo("%s, sync-req is received", pPeer->id); - if (pPeer->ip == 0) return; + //if (pPeer->ip == 0) return; + if (!syncResolvePeerFqdn(pPeer)) return; if (nodeRole != TAOS_SYNC_ROLE_MASTER) { sError("%s, I am not master anymore", pPeer->id); @@ -1090,7 +1103,7 @@ static int32_t syncProcessPeerMsg(int64_t rid, void *buffer) { } static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { - if (pPeer->peerFd < 0 || pPeer->ip == 0) { + if (pPeer->peerFd < 0/* || pPeer->ip == 0*/) { sDebug("%s, failed to send status msg, restart fd:%d", pPeer->id, pPeer->peerFd); syncRestartConnection(pPeer); return -1; @@ -1135,7 +1148,10 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + uint32_t ip = syncResolvePeerFqdn(pPeer); + if (!ip) return; + + SOCKET connFd = taosOpenTcpClientSocket(ip, pPeer->port, 0); if (connFd <= 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 505ba68c41..6775328ba0 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -422,6 +422,9 @@ void *syncRetrieveData(void *param) { return NULL; } + uint32_t ip = syncResolvePeerFqdn(pPeer); + if (!ip) return NULL; + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); @@ -430,7 +433,7 @@ void *syncRetrieveData(void *param) { if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, pPeer->numOfRetrieves); - pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + pPeer->syncFd = taosOpenTcpClientSocket(ip, pPeer->port, 0); if (pPeer->syncFd < 0) { sError("%s, failed to open socket to sync", pPeer->id); } else { From 58f49c5dc56946e7c9f39adeb01df428aba898f3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Jun 2021 12:03:12 +0800 Subject: [PATCH 008/115] [TD-4672]: remove peer fqdn IP caching --- src/sync/inc/syncInt.h | 1 + src/sync/src/syncMain.c | 33 +++++++++++++++++++++++++-------- src/sync/src/syncRetrieve.c | 8 +++++++- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index ec6dfcbc82..411e706fb1 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -132,6 +132,7 @@ void * syncRestoreData(void *param); int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); void syncRestartConnection(SSyncPeer *pPeer); void syncBroadcastStatus(SSyncNode *pNode); +uint32_t syncResolvePeerFqdn(SSyncPeer *pPeer); SSyncPeer *syncAcquirePeer(int64_t rid); void syncReleasePeer(SSyncPeer *pPeer); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 8ce37cbf78..7abbd60b6c 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -559,7 +559,8 @@ static void syncClosePeerConn(SSyncPeer *pPeer) { static void syncRemovePeer(SSyncPeer *pPeer) { sInfo("%s, it is removed", pPeer->id); - pPeer->ip = 0; + //pPeer->ip = 0; + pPeer->fqdn[0] = '\0'; syncClosePeerConn(pPeer); //taosRemoveRef(tsPeerRefId, pPeer->rid); syncReleasePeer(pPeer); @@ -585,20 +586,31 @@ static void syncStopCheckPeerConn(SSyncPeer *pPeer) { sDebug("%s, stop check peer connection", pPeer->id); } +uint32_t syncResolvePeerFqdn(SSyncPeer *pPeer) { + uint32_t ip = taosGetIpv4FromFqdn(pPeer->fqdn); + if (ip == 0xFFFFFFFF) { + sError("failed to resolve peer fqdn:%s since %s", pPeer->fqdn, strerror(errno)); + terrno = TSDB_CODE_RPC_FQDN_ERROR; + return 0; + } + + return ip; +} + static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { - uint32_t ip = taosGetIpv4FromFqdn(pInfo->nodeFqdn); + /*uint32_t ip = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ip == 0xFFFFFFFF) { sError("failed to add peer, can resolve fqdn:%s since %s", pInfo->nodeFqdn, strerror(errno)); terrno = TSDB_CODE_RPC_FQDN_ERROR; return NULL; } - + */ SSyncPeer *pPeer = calloc(1, sizeof(SSyncPeer)); if (pPeer == NULL) return NULL; pPeer->nodeId = pInfo->nodeId; tstrncpy(pPeer->fqdn, pInfo->nodeFqdn, sizeof(pPeer->fqdn)); - pPeer->ip = ip; + //pPeer->ip = ip; pPeer->port = pInfo->nodePort; pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0; snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, nodeId:%d", pNode->vgId, pPeer->nodeId); @@ -864,7 +876,8 @@ static void syncRestartPeer(SSyncPeer *pPeer) { } void syncRestartConnection(SSyncPeer *pPeer) { - if (pPeer->ip == 0) return; + //if (pPeer->ip == 0) return; + if (pPeer->fqdn[0] == '\0') return; if (syncAcquirePeer(pPeer->rid) == NULL) return; @@ -878,7 +891,8 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; sInfo("%s, sync-req is received", pPeer->id); - if (pPeer->ip == 0) return; + //if (pPeer->ip == 0) return; + if (!syncResolvePeerFqdn(pPeer)) return; if (nodeRole != TAOS_SYNC_ROLE_MASTER) { sError("%s, I am not master anymore", pPeer->id); @@ -1090,7 +1104,7 @@ static int32_t syncProcessPeerMsg(int64_t rid, void *buffer) { } static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { - if (pPeer->peerFd < 0 || pPeer->ip == 0) { + if (pPeer->peerFd < 0/* || pPeer->ip == 0*/) { sDebug("%s, failed to send status msg, restart fd:%d", pPeer->id, pPeer->peerFd); syncRestartConnection(pPeer); return -1; @@ -1135,7 +1149,10 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + uint32_t ip = syncResolvePeerFqdn(pPeer); + if (!ip) return; + + SOCKET connFd = taosOpenTcpClientSocket(ip, pPeer->port, 0); if (connFd <= 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 505ba68c41..c86ab85499 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -422,6 +422,12 @@ void *syncRetrieveData(void *param) { return NULL; } + uint32_t ip = syncResolvePeerFqdn(pPeer); + if (!ip) { + syncReleasePeer(pPeer); + return NULL; + } + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); @@ -430,7 +436,7 @@ void *syncRetrieveData(void *param) { if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, pPeer->numOfRetrieves); - pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + pPeer->syncFd = taosOpenTcpClientSocket(ip, pPeer->port, 0); if (pPeer->syncFd < 0) { sError("%s, failed to open socket to sync", pPeer->id); } else { From c50a85e9da868894ab3cac5f8b2302ba67d77493 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Jun 2021 15:14:01 +0800 Subject: [PATCH 009/115] sync/cleanup: remove unused ip checking --- src/sync/src/syncMain.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index aea46eccad..d6b81cdf36 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -876,7 +876,6 @@ static void syncRestartPeer(SSyncPeer *pPeer) { } void syncRestartConnection(SSyncPeer *pPeer) { - //if (pPeer->ip == 0) return; if (pPeer->fqdn[0] == '\0') return; if (syncAcquirePeer(pPeer->rid) == NULL) return; @@ -891,7 +890,6 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; sInfo("%s, sync-req is received", pPeer->id); - //if (pPeer->ip == 0) return; if (pPeer->fqdn[0] == '\0') return; if (nodeRole != TAOS_SYNC_ROLE_MASTER) { From df5d62ed41b4dd0352cba813a9e193b33aae53b4 Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Fri, 18 Jun 2021 13:26:30 +0800 Subject: [PATCH 010/115] add test case for jdbc nano precision --- .../TimestampPrecisionInNanoInJniTest.java | 618 ++++++++++++++++++ 1 file changed, 618 insertions(+) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java new file mode 100644 index 0000000000..d733209f25 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java @@ -0,0 +1,618 @@ +package com.taosdata.jdbc.cases; + + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; +import java.text.Format; +import java.text.SimpleDateFormat; + +public class TimestampPrecisionInNanoInJniTest { + + private static final String host = "127.0.0.1"; + private static final String ns_timestamp_db = "ns_precision_test"; + private static final long timestamp1 = System.currentTimeMillis(); + private static final long timestamp2 = timestamp1 * 1000_000 + 123455; + private static final long timestamp3 = (timestamp1 + 10) * 1000_000 + 123456; + private static final Format format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + private static final String date1 = format.format(new Date(timestamp1)); + private static final String date4 = format.format(new Date(timestamp1 + 10l)); + private static final String date2 = date1 + "123455"; + private static final String date3 = date4 + "123456"; + + + private static Connection conn; + + @BeforeClass + public static void beforeClass() throws SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url, properties); + + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + ns_timestamp_db); + stmt.execute("create database if not exists " + ns_timestamp_db + " precision 'ns'"); + stmt.execute("create table " + ns_timestamp_db + ".weather(ts timestamp, ts2 timestamp, f1 int)"); + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(\"" + date3 + "\", \"" + date3 + "\", 128)"); + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(" + timestamp2 + "," + timestamp2 + ", 127)"); + stmt.close(); + } + + @After + public void afterEach() throws SQLException { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + ns_timestamp_db); + stmt.execute("create database if not exists " + ns_timestamp_db + " precision 'ns'"); + stmt.execute("create table " + ns_timestamp_db + ".weather(ts timestamp, ts2 timestamp, f1 int)"); + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(\"" + date3 + "\", \"" + date3 + "\", 128)"); + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(" + timestamp2 + "," + timestamp2 + ", 127)"); + stmt.close(); + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private void checkCount(long count, ResultSet rs) throws SQLException { + rs.next(); + long test_count = rs.getLong(1); + Assert.assertEquals(count, test_count); + } + + private void checkTime(long ts, ResultSet rs) throws SQLException { + rs.next(); + int nanos = rs.getTimestamp(1).getNanos(); + Assert.assertEquals(ts % 1000_000_000l, nanos); + long test_ts = rs.getLong(1); + Assert.assertEquals(ts / 1000_000l, test_ts); + } + + @Test + public void canInsertTimestampAndQueryByEqualToInDateTypeInBothFirstAndSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts = '" + date3 + "'"); + checkTime(timestamp3, rs); + rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 = '" + date3 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canImportTimestampAndQueryByEqualToInDateTypeInBothFirstAndSecondCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("import into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(\"" + date1 + "123123\", \"" + date1 + "123123\", 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + date1 + "123123'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts = '" + date1 + "123123'"); + checkTime(timestamp1 * 1000_000l + 123123l, rs); + rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + date1 + "123123'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 = '" + date1 + "123123'"); + checkTime(timestamp1 * 1000_000l + 123123l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canInsertTimestampAndQueryByEqualToInNumberTypeInBothFirstAndSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts = '" + timestamp2 + "'"); + checkTime(timestamp2, rs); + rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 = '" + timestamp2 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canImportTimestampAndQueryByEqualToInNumberTypeInBothFirstAndSecondCol() { + try (Statement stmt = conn.createStatement()) { + long timestamp4 = timestamp1 * 1000_000 + 123123; + stmt.executeUpdate("import into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(" + timestamp4 + ", " + timestamp4 + ", 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + timestamp4 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts = '" + timestamp4 + "'"); + checkTime(timestamp4, rs); + rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + timestamp4 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 = '" + timestamp4 + "'"); + checkTime(timestamp4, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canSelectLastRowFromWeatherForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select last(ts) from " + ns_timestamp_db + ".weather"); + // int nanos = rs.getTimestamp(1).getNanos(); + // Assert.assertEquals(timestamp3 % 1000_000l * 1000, nanos); + // java.lang.AssertionError: expected:<123456000> but was:<468123456> + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canSelectLastRowFromWeatherForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select last(ts2) from " + ns_timestamp_db + ".weather"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canSelectFirstRowFromWeatherForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select first(ts) from " + ns_timestamp_db + ".weather"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canSelectFirstRowFromWeatherForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select first(ts2) from " + ns_timestamp_db + ".weather"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts > '" + date2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts > '" + date2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 > '" + date2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 > '" + date2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts > '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts > '" + timestamp2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 > '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 > '" + timestamp2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanOrEqualToInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts >= '" + date2 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanOrEqualToInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 >= '" + date2 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanOrEqualToInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts >= '" + timestamp2 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLargerThanOrEqualToInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 >= '" + timestamp2 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts < '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts < '" + date3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 < '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 < '" + date3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts < '" + timestamp3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts < '" + timestamp3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 < '" + timestamp3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 < '" + timestamp3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanOrEqualToInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <= '" + date3 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanOrEqualToInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <= '" + date3 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanOrEqualToInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <= '" + timestamp3 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryLessThanOrEqualToInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <= '" + timestamp3 + "'"); + checkCount(2l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryBetweenAndInDateTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <= '" + date3 + "' AND ts > '" + date2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts <= '" + date3 + "' AND ts > '" + date2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryBetweenAndInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <= '" + date3 + "' AND ts2 > '" + date2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 <= '" + date3 + "' AND ts2 > '" + date2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryBetweenAndInNumberTypeForFirstCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <= '" + timestamp3 + "' AND ts > '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts <= '" + timestamp3 + "' AND ts > '" + timestamp2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canQueryBetweenAndInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <= '" + timestamp3 + "' AND ts2 > '" + timestamp2 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 <= '" + timestamp3 + "' AND ts2 > '" + timestamp2 + "'"); + checkTime(timestamp3, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // @Test + // public void canQueryNotEqualToInDateTypeForFirstCol() { + // try (Statement stmt = conn.createStatement()) { + // ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <> '" + date3 + "'"); + // checkCount(1l, rs); + // rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts <> '" + date3 + "'"); + // checkTime(timestamp2, rs); + // } catch (SQLException e) { + // e.printStackTrace(); + // } + // } + + @Test + public void canQueryNotEqualToInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <> '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 <> '" + date3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // @Test + // public void canQueryNotEqualToInNumberTypeForFirstCol() { + // try (Statement stmt = conn.createStatement()) { + // ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <> '" + timestamp3 + "'"); + // checkCount(1l, rs); + // rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts <> '" + timestamp3 + "'"); + // checkTime(timestamp2, rs); + // } catch (SQLException e) { + // e.printStackTrace(); + // } + // } + + @Test + public void canQueryNotEqualToInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 <> '" + timestamp3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 <> '" + timestamp3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // @Test + // public void canQueryNotEqualInDateTypeForFirstCol() { + // try (Statement stmt = conn.createStatement()) { + // ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts != '" + date3 + "'"); + // checkCount(1l, rs); + // rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts != '" + date3 + "'"); + // checkTime(timestamp2, rs); + // } catch (SQLException e) { + // e.printStackTrace(); + // } + // } + + @Test + public void canQueryNotEqualInDateTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 != '" + date3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 != '" + date3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // @Test + // public void canQueryNotEqualInNumberTypeForFirstCol() { + // try (Statement stmt = conn.createStatement()) { + // ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts != '" + timestamp3 + "'"); + // checkCount(1l, rs); + // rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts != '" + timestamp3 + "'"); + // checkTime(timestamp2, rs); + // } catch (SQLException e) { + // e.printStackTrace(); + // } + // } + + @Test + public void canQueryNotEqualInNumberTypeForSecondCol() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 != '" + timestamp3 + "'"); + checkCount(1l, rs); + rs = stmt.executeQuery("select ts2 from " + ns_timestamp_db + ".weather where ts2 != '" + timestamp3 + "'"); + checkTime(timestamp2, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canInsertTimestampWithNowAndNsOffsetInBothFirstAndSecondCol(){ + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(now + 1000b, now - 1000b, 128)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather"); + checkCount(3l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canIntervalAndSlidingAcceptNsUnitForFirstCol(){ + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select sum(f1) from " + ns_timestamp_db + ".weather where ts >= '" + date2 + "' and ts <= '" + date3 + "' interval(10000000b) sliding(10000000b)"); + rs.next(); + long sum = rs.getLong(2); + Assert.assertEquals(127l, sum); + rs.next(); + sum = rs.getLong(2); + Assert.assertEquals(128l, sum); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void canIntervalAndSlidingAcceptNsUnitForSecondCol(){ + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select sum(f1) from " + ns_timestamp_db + ".weather where ts2 >= '" + date2 + "' and ts <= '" + date3 + "' interval(10000000b) sliding(10000000b)"); + rs.next(); + long sum = rs.getLong(2); + Assert.assertEquals(127l, sum); + rs.next(); + sum = rs.getLong(2); + Assert.assertEquals(128l, sum); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testDataOutOfRangeExceptionForFirstCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(123456789012345678, 1234567890123456789, 127)"); + } catch (SQLException e) { + Assert.assertEquals("TDengine ERROR (8000060b): Timestamp data out of range", e.getMessage()); + } + } + + @Test + public void testDataOutOfRangeExceptionForSecondCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values(1234567890123456789, 123456789012345678, 127)"); + } catch (SQLException e) { + Assert.assertEquals("TDengine ERROR (8000060b): Timestamp data out of range", e.getMessage()); + } + } + + @Test + public void willAutomaticallyFillToNsUnitWithZerosForFirstCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values('" + date1 + "', '" + date1 + "', 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + date1 + "000000'"); + checkCount(1l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void willAutomaticallyFillToNsUnitWithZerosForSecondCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values('" + date1 + "', '" + date1 + "', 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + date1 + "000000'"); + checkCount(1l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void willAutomaticallyDropDigitExceedNsDigitNumberForFirstCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values('" + date1 + "999999999', '" + date1 + "999999999', 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts = '" + date1 + "999999'"); + checkCount(1l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void willAutomaticallyDropDigitExceedNsDigitNumberForSecondCol() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into " + ns_timestamp_db + ".weather(ts, ts2, f1) values('" + date1 + "999999999', '" + date1 + "999999999', 127)"); + ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts2 = '" + date1 + "999999'"); + checkCount(1l, rs); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} From 9714f60edefb75cebbff861ca28cbb2c4c4c1408 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 18 Jun 2021 14:56:30 +0800 Subject: [PATCH 011/115] [TD-4724] add case for td-4724 --- tests/pytest/fulltest.sh | 1 + tests/pytest/functions/queryTestCases.py | 364 +++++++++++++++++++++++ 2 files changed, 365 insertions(+) create mode 100644 tests/pytest/functions/queryTestCases.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 188cb6ad07..d10f76041d 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -313,6 +313,7 @@ python3 ./test.py -f query/last_row_cache.py python3 ./test.py -f account/account_create.py python3 ./test.py -f alter/alter_table.py python3 ./test.py -f query/queryGroupbySort.py +python3 ./test.py -f functions/queryTestCases.py python3 ./test.py -f insert/unsignedInt.py python3 ./test.py -f insert/unsignedBigint.py diff --git a/tests/pytest/functions/queryTestCases.py b/tests/pytest/functions/queryTestCases.py new file mode 100644 index 0000000000..b7480fdbd5 --- /dev/null +++ b/tests/pytest/functions/queryTestCases.py @@ -0,0 +1,364 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import subprocess + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self) -> str: + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/debug/build/bin")] + break + return buildPath + + def getCfgDir(self) -> str: + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + cfgDir = self.getBuildPath() + "/community/sim/dnode1/cfg" + else: + cfgDir = self.getBuildPath() + "/sim/dnode1/cfg" + return cfgDir + + def getCfgFile(self) -> str: + return self.getCfgDir()+"/taos.cfg" + + def td3690(self): + tdLog.printNoPrefix("==========TD-3690==========") + tdSql.query("show variables") + tdSql.checkData(51, 1, 864000) + + def td4082(self): + tdLog.printNoPrefix("==========TD-4082==========") + cfgfile = self.getCfgFile() + max_compressMsgSize = 100000000 + + tdSql.query("show variables") + tdSql.checkData(26, 1, -1) + + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + + tdDnodes.stop(index) + cmd = f"sed -i '$a compressMSgSize {max_compressMsgSize}' {cfgfile} " + try: + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + except Exception as e: + raise e + + tdDnodes.start(index) + tdSql.query("show variables") + tdSql.checkData(26, 1, 100000000) + + tdDnodes.stop(index) + cmd = f"sed -i '$s/{max_compressMsgSize}/{max_compressMsgSize+10}/g' {cfgfile} " + try: + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + except Exception as e: + raise e + + tdDnodes.start(index) + tdSql.query("show variables") + tdSql.checkData(26, 1, -1) + + tdDnodes.stop(index) + cmd = f"sed -i '$d' {cfgfile}" + try: + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + except Exception as e: + raise e + + tdDnodes.start(index) + + def td4097(self): + tdLog.printNoPrefix("==========TD-4097==========") + tdSql.execute("drop database if exists db") + tdSql.execute("drop database if exists db1") + tdSql.execute("create database if not exists db keep 3650") + tdSql.execute("create database if not exists db1 keep 3650") + + tdSql.execute("create stable db.stb1 (ts timestamp, c1 int) tags(t1 int)") + tdSql.execute("create stable db.stb2 (ts timestamp, c1 int) tags(t1 int)") + tdSql.execute("create stable db1.stb3 (ts timestamp, c1 int) tags(t1 int)") + + tdSql.execute("create table db.t10 using db.stb1 tags(1)") + tdSql.execute("create table db.t11 using db.stb1 tags(2)") + tdSql.execute("create table db.t20 using db.stb2 tags(3)") + tdSql.execute("create table db1.t30 using db1.stb3 tags(4)") + + tdLog.printNoPrefix("==========TD-4097==========") + # 插入数据，然后进行show create 操作 + + # p1 不进入指定数据库 + tdSql.query("show create database db") + tdSql.checkRows(1) + tdSql.error("show create database ") + tdSql.error("show create databases db ") + tdSql.error("show create database db.stb1") + tdSql.error("show create database db0") + tdSql.error("show create database db db1") + tdSql.error("show create database db, db1") + tdSql.error("show create database stb1") + tdSql.error("show create database * ") + + tdSql.query("show create stable db.stb1") + tdSql.checkRows(1) + tdSql.error("show create stable db.t10") + tdSql.error("show create stable db.stb0") + tdSql.error("show create stable stb1") + tdSql.error("show create stable ") + tdSql.error("show create stable *") + tdSql.error("show create stable db.stb1 db.stb2") + tdSql.error("show create stable db.stb1, db.stb2") + + tdSql.query("show create table db.stb1") + tdSql.checkRows(1) + tdSql.query("show create table db.t10") + tdSql.checkRows(1) + tdSql.error("show create table db.stb0") + tdSql.error("show create table stb1") + tdSql.error("show create table ") + tdSql.error("show create table *") + tdSql.error("show create table db.stb1 db.stb2") + tdSql.error("show create table db.stb1, db.stb2") + + # p2 进入指定数据库 + tdSql.execute("use db") + + tdSql.query("show create database db") + tdSql.checkRows(1) + tdSql.query("show create database db1") + tdSql.checkRows(1) + tdSql.error("show create database ") + tdSql.error("show create databases db ") + tdSql.error("show create database db.stb1") + tdSql.error("show create database db0") + tdSql.error("show create database db db1") + tdSql.error("show create database db, db1") + tdSql.error("show create database stb1") + tdSql.error("show create database * ") + + tdSql.query("show create stable db.stb1") + tdSql.checkRows(1) + tdSql.query("show create stable stb1") + tdSql.checkRows(1) + tdSql.query("show create stable db1.stb3") + tdSql.checkRows(1) + tdSql.error("show create stable db.t10") + tdSql.error("show create stable db") + tdSql.error("show create stable t10") + tdSql.error("show create stable db.stb0") + tdSql.error("show create stables stb1") + tdSql.error("show create stable ") + tdSql.error("show create stable *") + tdSql.error("show create stable db.stb1 db.stb2") + tdSql.error("show create stable stb1 stb2") + tdSql.error("show create stable db.stb1, db.stb2") + tdSql.error("show create stable stb1, stb2") + + tdSql.query("show create table db.stb1") + tdSql.checkRows(1) + tdSql.query("show create table stb1") + tdSql.checkRows(1) + tdSql.query("show create table db.t10") + tdSql.checkRows(1) + tdSql.query("show create table t10") + tdSql.checkRows(1) + tdSql.query("show create table db1.t30") + tdSql.checkRows(1) + tdSql.error("show create table t30") + tdSql.error("show create table db.stb0") + tdSql.error("show create table db.t0") + tdSql.error("show create table db") + tdSql.error("show create tables stb1") + tdSql.error("show create tables t10") + tdSql.error("show create table ") + tdSql.error("show create table *") + tdSql.error("show create table db.stb1 db.stb2") + tdSql.error("show create table db.t11 db.t10") + tdSql.error("show create table db.stb1, db.stb2") + tdSql.error("show create table db.t11, db.t10") + tdSql.error("show create table stb1 stb2") + tdSql.error("show create table t11 t10") + tdSql.error("show create table stb1, stb2") + tdSql.error("show create table t11, t10") + + # p3 删库删表后进行查询 + tdSql.execute("drop table if exists t11") + + tdSql.error("show create table t11") + tdSql.error("show create table db.t11") + tdSql.query("show create stable stb1") + tdSql.checkRows(1) + tdSql.query("show create table t10") + tdSql.checkRows(1) + + tdSql.execute("drop stable if exists stb2") + + tdSql.error("show create table stb2") + tdSql.error("show create table db.stb2") + tdSql.error("show create stable stb2") + tdSql.error("show create stable db.stb2") + tdSql.error("show create stable db.t20") + tdSql.query("show create database db") + tdSql.checkRows(1) + tdSql.query("show create stable db.stb1") + tdSql.checkRows(1) + + tdSql.execute("drop database if exists db1") + tdSql.error("show create database db1") + tdSql.error("show create stable db1.t31") + tdSql.error("show create stable db1.stb3") + tdSql.query("show create database db") + tdSql.checkRows(1) + tdSql.query("show create stable db.stb1") + tdSql.checkRows(1) + + def td4153(self): + tdLog.printNoPrefix("==========TD-4153==========") + + pass + + def td4288(self): + tdLog.printNoPrefix("==========TD-4288==========") + # keep ~ [days,365000] + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db") + tdSql.query("show variables") + tdSql.checkData(36, 1, 3650) + tdSql.query("show databases") + tdSql.checkData(0,7,"3650,3650,3650") + + days = tdSql.getData(0, 6) + tdSql.error("alter database db keep 3650001") + tdSql.error("alter database db keep 9") + tdSql.error("alter database db keep 0b") + tdSql.error("alter database db keep 3650,9,36500") + tdSql.error("alter database db keep 3650,3650,365001") + tdSql.error("alter database db keep 36500,a,36500") + tdSql.error("alter database db keep (36500,3650,3650)") + tdSql.error("alter database db keep [36500,3650,36500]") + tdSql.error("alter database db keep 36500,0xff,3650") + tdSql.error("alter database db keep 36500,0o365,3650") + tdSql.error("alter database db keep 36500,0A3Ch,3650") + tdSql.error("alter database db keep") + tdSql.error("alter database db keep0 36500") + + tdSql.execute("alter database db keep 36500") + tdSql.query("show databases") + tdSql.checkData(0, 7, "3650,3650,36500") + tdSql.execute("drop database if exists db") + + tdSql.execute("create database if not exists db1") + tdSql.query("show databases") + tdSql.checkData(0, 7, "3650,3650,3650") + tdSql.query("show variables") + tdSql.checkData(36, 1, 3650) + + tdSql.execute("alter database db1 keep 365") + tdSql.execute("drop database if exists db1") + + + pass + + def td4724(self): + tdLog.printNoPrefix("==========TD-4724==========") + cfgfile = self.getCfgFile() + minTablesPerVnode = 5 + maxTablesPerVnode = 10 + maxVgroupsPerDb = 100 + + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + + tdDnodes.stop(index) + vnode_cmd = f"sed -i '$a maxVgroupsPerDb {maxVgroupsPerDb}' {cfgfile} " + min_cmd = f"sed -i '$a minTablesPerVnode {minTablesPerVnode}' {cfgfile} " + max_cmd = f"sed -i '$a maxTablesPerVnode {maxTablesPerVnode}' {cfgfile} " + try: + _ = subprocess.check_output(vnode_cmd, shell=True).decode("utf-8") + _ = subprocess.check_output(min_cmd, shell=True).decode("utf-8") + _ = subprocess.check_output(max_cmd, shell=True).decode("utf-8") + except Exception as e: + raise e + + tdDnodes.start(index) + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 3650") + tdSql.execute("use db") + tdSql.execute("create stable db.stb1 (ts timestamp, c1 int) tags(t1 int)") + insert_sql = "insert into " + for i in range(100): + tdSql.execute(f"create table db.t1{i} using db.stb1 tags({i})") + insert_sql += f" t1{i} values({1604298064000 + i*1000}, {i})" + tdSql.query("show dnodes") + vnode_count = tdSql.getData(0, 2) + if vnode_count <= 1: + tdLog.exit("vnode is less than 2") + + tdSql.execute(insert_sql) + tdDnodes.stop(index) + cmd = f"sed -i '$d' {cfgfile}" + try: + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + _ = subprocess.check_output(cmd, shell=True).decode("utf-8") + except Exception as e: + raise e + + tdDnodes.start(index) + + pass + + def run(self): + + # master branch + # self.td3690() + # self.td4082() + # self.td4288() + self.td4724() + + # develop branch + # self.td4097() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + + + From 4bccaeeecd912ff722250700d1ea61dcb9fe0091 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 18 Jun 2021 16:44:51 +0800 Subject: [PATCH 012/115] fix quote issue --- src/kit/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 0eb1248fad..f670d05f55 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -248,7 +248,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { if (quote == c) { quote = 0; - } else if (c == '\'' || c == '"') { + } else if (quote == 0 && (c == '\'' || c == '"')) { quote = c; } From 5b2d908cb27586af7c2620d1319dcc04392d87f1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 19 Jun 2021 09:16:35 +0800 Subject: [PATCH 013/115] [TD-4734] session_window and state window support main query --- src/client/src/tscSQLParser.c | 12 +++++++++++- src/query/src/qExecutor.c | 10 ++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 9532d1e202..93d94c8711 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7774,8 +7774,15 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf return code; } } + + int32_t timeWindowQuery = + (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, false) != TSDB_CODE_SUCCESS) { + if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, timeWindowQuery) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + // parse the window_state + if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -7815,6 +7822,9 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } else { + if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } if (isTimeWindowQuery(pQueryInfo) || pQueryInfo->sessionWindow.gap > 0) { // check if the first column of the nest query result is timestamp column SColumn* pCol = taosArrayGetP(pQueryInfo->colList, 0); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7b106c178d..f4c0a394d6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1751,7 +1751,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf case OP_SessionWindow: { pRuntimeEnv->proot = createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } break; } case OP_MultiTableAggregate: { @@ -1787,7 +1790,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf } case OP_StateWindow: { pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } break; } From e5f2bab3a6b9ee035967be1b5dc6c11b9fd7b109 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 19 Jun 2021 10:41:42 +0800 Subject: [PATCH 014/115] [TD-4509]: add testcase of irate function --- tests/pytest/functions/function_irate.py | 224 +++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 tests/pytest/functions/function_irate.py diff --git a/tests/pytest/functions/function_irate.py b/tests/pytest/functions/function_irate.py new file mode 100644 index 0000000000..15adab7651 --- /dev/null +++ b/tests/pytest/functions/function_irate.py @@ -0,0 +1,224 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 100 + self.ts = 1537146000000 + self.ts1 = 1537146000000000 + + + def run(self): + # db precison ms + tdSql.prepare() + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("create table gtest1 (ts timestamp, col1 float)") + tdSql.execute("create table gtest2 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest3 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest4 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest5 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest6 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest7 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest8 (ts timestamp, col1 tinyint)") + + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + tdSql.execute("insert into gtest1 values(1537146000000,0);") + tdSql.execute("insert into gtest1 values(1537146001100,1.2);") + tdSql.execute("insert into gtest2 values(1537146001001,1);") + tdSql.execute("insert into gtest2 values(1537146001101,2);") + tdSql.execute("insert into gtest3 values(1537146001101,2);") + tdSql.execute("insert into gtest4(ts) values(1537146001101);") + tdSql.execute("insert into gtest5 values(1537146001002,4);") + tdSql.execute("insert into gtest5 values(1537146002202,4);") + tdSql.execute("insert into gtest6 values(1537146000000,5);") + tdSql.execute("insert into gtest6 values(1537146001000,2);") + tdSql.execute("insert into gtest7 values(1537146001000,1);") + tdSql.execute("insert into gtest7 values(1537146008000,2);") + tdSql.execute("insert into gtest7 values(1537146009000,6);") + tdSql.execute("insert into gtest7 values(1537146012000,3);") + tdSql.execute("insert into gtest7 values(1537146015000,3);") + tdSql.execute("insert into gtest7 values(1537146017000,1);") + tdSql.execute("insert into gtest7 values(1537146019000,3);") + tdSql.execute("insert into gtest8 values(1537146000002,4);") + tdSql.execute("insert into gtest8 values(1537146002202,4);") + + # irate verifacation + tdSql.query("select irate(col1) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col1) from test1 interval(10s);") + tdSql.checkData(0, 1, 1) + tdSql.query("select irate(col1) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col2) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col3) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col4) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col5) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col6) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col11) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col12) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col13) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col14) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col2) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col2) from test1;") + tdSql.checkData(0, 0, 1) + + tdSql.query("select irate(col1) from gtest1;") + tdSql.checkData(0, 0, 1.2/1.1) + tdSql.query("select irate(col1) from gtest2;") + tdSql.checkData(0, 0, 10) + tdSql.query("select irate(col1) from gtest3;") + tdSql.checkData(0, 0, 0) + tdSql.query("select irate(col1) from gtest4;") + tdSql.checkRows(0) + tdSql.query("select irate(col1) from gtest5;") + tdSql.checkData(0, 0, 0) + tdSql.query("select irate(col1) from gtest6;") + tdSql.checkData(0, 0, 2) + tdSql.query("select irate(col1) from gtest7;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col1) from gtest7 interval(5s) order by ts asc;") + tdSql.checkData(1, 1, 4) + tdSql.checkData(2, 1, 0) + tdSql.checkData(3, 1, 1) + tdSql.query("select irate(col1) from gtest7 interval(5s) order by ts desc ;") + tdSql.checkData(1, 1, 0) + tdSql.checkData(2, 1, 4) + tdSql.checkData(3, 1, 0) + + #error + tdSql.error("select irate(col1) from test") + tdSql.error("select irate(ts) from test1") + tdSql.error("select irate(col7) from test1") + tdSql.error("select irate(col8) from test1") + tdSql.error("select irate(col9) from test1") + tdSql.error("select irate(loc) from test1") + + # use db1 precision us + tdSql.execute("create database db1 precision 'us' keep 3650 UPDATE 1") + tdSql.execute("use db1 ") + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("create table gtest1 (ts timestamp, col1 float)") + tdSql.execute("create table gtest2 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest3 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest4 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest5 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest6 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest7 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest8 (ts timestamp, col1 tinyint)") + tdSql.execute("create table gtest9 (ts timestamp, col1 tinyint)") + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts1 + i*1000000, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + tdSql.execute("insert into gtest1 values(1537146000000000,0);") + tdSql.execute("insert into gtest1 values(1537146001100000,1.2);") + tdSql.execute("insert into gtest2 values(1537146001001000,1);") + tdSql.execute("insert into gtest2 values(1537146001101000,2);") + tdSql.execute("insert into gtest3 values(1537146001101000,2);") + tdSql.execute("insert into gtest4(ts) values(1537146001101000);") + tdSql.execute("insert into gtest5 values(1537146001002000,4);") + tdSql.execute("insert into gtest5 values(1537146002202000,4);") + tdSql.execute("insert into gtest6 values(1537146000000000,5);") + tdSql.execute("insert into gtest6 values(1537146001000000,2);") + tdSql.execute("insert into gtest7 values(1537146001000000,1);") + tdSql.execute("insert into gtest7 values(1537146008000000,2);") + tdSql.execute("insert into gtest7 values(1537146009000000,6);") + tdSql.execute("insert into gtest7 values(1537146012000000,3);") + tdSql.execute("insert into gtest7 values(1537146015000000,3);") + tdSql.execute("insert into gtest7 values(1537146017000000,1);") + tdSql.execute("insert into gtest7 values(1537146019000000,3);") + tdSql.execute("insert into gtest8 values(1537146000002000,3);") + tdSql.execute("insert into gtest8 values(1537146001003000,4);") + tdSql.execute("insert into gtest9 values(1537146000000000,4);") + tdSql.execute("insert into gtest9 values(1537146000000001,5);") + + + # irate verifacation + tdSql.query("select irate(col1) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col1) from test1 interval(10s);") + tdSql.checkData(0, 1, 1) + tdSql.query("select irate(col1) from test1;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col1) from gtest1;") + tdSql.checkData(0, 0, 1.2/1.1) + tdSql.query("select irate(col1) from gtest2;") + tdSql.checkData(0, 0, 10) + tdSql.query("select irate(col1) from gtest3;") + tdSql.checkData(0, 0, 0) + tdSql.query("select irate(col1) from gtest4;") + tdSql.checkRows(0) + tdSql.query("select irate(col1) from gtest5;") + tdSql.checkData(0, 0, 0) + tdSql.query("select irate(col1) from gtest6;") + tdSql.checkData(0, 0, 2) + tdSql.query("select irate(col1) from gtest7;") + tdSql.checkData(0, 0, 1) + tdSql.query("select irate(col1) from gtest7 interval(5s) order by ts asc;") + tdSql.checkData(1, 1, 4) + tdSql.checkData(2, 1, 0) + tdSql.checkData(3, 1, 1) + tdSql.query("select irate(col1) from gtest7 interval(5s) order by ts desc ;") + tdSql.checkData(1, 1, 0) + tdSql.checkData(2, 1, 4) + tdSql.checkData(3, 1, 0) + tdSql.query("select irate(col1) from gtest8;") + tdSql.checkData(0, 0, 1/1.001) + tdSql.query("select irate(col1) from gtest9;") + tdSql.checkData(0, 0, 1000000) + + #error + # tdSql.error("select irate(col1) from test") + tdSql.error("select irate(ts) from test1") + tdSql.error("select irate(col7) from test1") + tdSql.error("select irate(col8) from test1") + tdSql.error("select irate(col9) from test1") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 91c557d99de11efc5793d9ed6cfa2317524a1457 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 19 Jun 2021 10:58:26 +0800 Subject: [PATCH 015/115] [TD-4509]: add testcase of irate function --- tests/pytest/functions/function_irate.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/pytest/functions/function_irate.py b/tests/pytest/functions/function_irate.py index 15adab7651..2c85e1bbdd 100644 --- a/tests/pytest/functions/function_irate.py +++ b/tests/pytest/functions/function_irate.py @@ -33,8 +33,8 @@ class TDTestCase: # db precison ms tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') - tdSql.execute("create table test1 using test tags('beijing')") + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20), tag1 int)''') + tdSql.execute("create table test1 using test tags('beijing', 10)") tdSql.execute("create table gtest1 (ts timestamp, col1 float)") tdSql.execute("create table gtest2 (ts timestamp, col1 tinyint)") tdSql.execute("create table gtest3 (ts timestamp, col1 tinyint)") @@ -129,6 +129,7 @@ class TDTestCase: tdSql.error("select irate(col8) from test1") tdSql.error("select irate(col9) from test1") tdSql.error("select irate(loc) from test1") + tdSql.error("select irate(tag1) from test1") # use db1 precision us tdSql.execute("create database db1 precision 'us' keep 3650 UPDATE 1") @@ -208,11 +209,14 @@ class TDTestCase: tdSql.checkData(0, 0, 1000000) #error - # tdSql.error("select irate(col1) from test") + tdSql.error("select irate(col1) from test") tdSql.error("select irate(ts) from test1") tdSql.error("select irate(col7) from test1") tdSql.error("select irate(col8) from test1") tdSql.error("select irate(col9) from test1") + tdSql.error("select irate(loc) from test1") + tdSql.error("select irate(tag1) from test1") + From ca30708ce1e920dfc505ccef5db4dab4becb5d56 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 19 Jun 2021 11:17:31 +0800 Subject: [PATCH 016/115] [TD-4509]: add testcase of irate function --- tests/pytest/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index b9b7bbcaf6..d2d850a850 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -322,6 +322,7 @@ python3 ./test.py -f alter/alter_table.py python3 ./test.py -f query/queryGroupbySort.py python3 ./test.py -f functions/function_stateWindow.py python3 ./test.py -f functions/function_derivative.py +python3 ./test.py -f functions/function_irate.py python3 ./test.py -f insert/unsignedInt.py python3 ./test.py -f insert/unsignedBigint.py From 0c93fff0d259b698344a6baa723d1e29af2469b0 Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Sat, 19 Jun 2021 11:43:37 +0800 Subject: [PATCH 017/115] [TD-4774]Ignore some errors in valgrind (#6544) --- tests/pytest/crash_gen/valgrind_taos.supp | 226 ++++++++++++++++++++++ 1 file changed, 226 insertions(+) diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index a00b2d830c..b42015a053 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -17496,4 +17496,230 @@ obj:/usr/bin/python3.8 obj:/usr/bin/python3.8 fun:PyVectorcall_Call +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:__libc_alloc_buffer_allocate + fun:alloc_buffer_allocate + fun:__resolv_conf_allocate + fun:__resolv_conf_load + fun:__resolv_conf_get_current + fun:__res_vinit + fun:maybe_init + fun:context_get + fun:context_get + fun:__resolv_context_get + fun:gaih_inet.constprop.0 + fun:getaddrinfo + fun:taosGetFqdn + fun:taosCheckGlobalCfg + fun:taos_init_imp +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + fun:__pyx_pymod_exec_mtrand + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:PyCode_NewWithPosOnlyArgs + fun:PyCode_New + fun:__Pyx_InitCachedConstants + fun:__pyx_pymod_exec__generator + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + fun:__pyx_pymod_exec_bit_generator + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + fun:__pyx_pymod_exec__common + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + fun:__pyx_pymod_exec__bounded_integers + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + fun:__pyx_pymod_exec__mt19937 + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + fun:__pyx_pymod_exec__philox + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + fun:__pyx_pymod_exec__pcg64 + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + fun:__pyx_pymod_exec__sfc64 + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/bin/python3.8 + fun:PyTuple_Pack + fun:__Pyx_InitCachedConstants + fun:__pyx_pymod_exec__generator + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:PyCode_NewWithPosOnlyArgs + fun:PyCode_New + fun:__pyx_pymod_exec_mtrand + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:PyCode_NewWithPosOnlyArgs + fun:PyCode_New + fun:__pyx_pymod_exec_bit_generator + fun:PyModule_ExecDef + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault } \ No newline at end of file From 998c5f18421a2cb5fd31c3b7e1187c438864fb56 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 19 Jun 2021 16:09:41 +0800 Subject: [PATCH 018/115] [TD-4735]:support select last_row from subquery --- src/client/src/tscSQLParser.c | 11 +++++++++-- tests/script/general/parser/lastrow.sim | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 9532d1e202..d7fbdd3ff8 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2125,7 +2125,10 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT } static void updateLastScanOrderIfNeeded(SQueryInfo* pQueryInfo) { - if (pQueryInfo->sessionWindow.gap > 0 || tscGroupbyColumn(pQueryInfo)) { + if (pQueryInfo->sessionWindow.gap > 0 || + pQueryInfo->stateWindow || + taosArrayGetSize(pQueryInfo->pUpstream) > 0 || + tscGroupbyColumn(pQueryInfo)) { size_t numOfExpr = tscNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExpr; ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); @@ -2385,7 +2388,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query SConvertFunc cvtFunc = {.originFuncId = functionId, .execFuncId = functionId}; - if (functionId == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) { + if (functionId == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || + (hasNormalColumnFilter(pQueryInfo)) || + taosArrayGetSize(pQueryInfo->pUpstream)>0)) { cvtFunc.execFuncId = TSDB_FUNC_LAST; } @@ -7836,6 +7841,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if ((code = doFunctionsCompatibleCheck(pCmd, pQueryInfo, tscGetErrorMsgPayload(pCmd))) != TSDB_CODE_SUCCESS) { return code; } + + updateLastScanOrderIfNeeded(pQueryInfo); } else { pQueryInfo->command = TSDB_SQL_SELECT; diff --git a/tests/script/general/parser/lastrow.sim b/tests/script/general/parser/lastrow.sim index 2b8f294d5d..fea322ec16 100644 --- a/tests/script/general/parser/lastrow.sim +++ b/tests/script/general/parser/lastrow.sim @@ -70,4 +70,17 @@ sleep 100 run general/parser/lastrow_query.sim -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print =================== last_row + nested query +sql use $db +sql create table lr_nested(ts timestamp, f int) +sql insert into lr_nested values(now, 1) +sql insert into lr_nested values(now+1s, null) +sql select last_row(*) from (select * from lr_nested) +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 643807e6630242388f38d749e4630416322347d6 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 19 Jun 2021 16:25:11 +0800 Subject: [PATCH 019/115] [TD-45145]: add testcase of stat_window function --- tests/pytest/query/queryStateWindow.py | 28 ++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/pytest/query/queryStateWindow.py b/tests/pytest/query/queryStateWindow.py index 027ece89af..db9eaf8672 100644 --- a/tests/pytest/query/queryStateWindow.py +++ b/tests/pytest/query/queryStateWindow.py @@ -16,6 +16,7 @@ import taos from util.log import * from util.cases import * from util.sql import * +import numpy as np class TDTestCase: @@ -30,7 +31,7 @@ class TDTestCase: print("==============step1") tdSql.execute( - "create table if not exists st (ts timestamp, t1 int, t2 timestamp, t3 bigint, t4 float, t5 double, t6 binary(10), t7 smallint, t8 tinyint, t9 bool, t10 nchar(10), t11 int unsigned, t12 bigint unsigned, t13 smallint unsigned, t14 tinyint unsigned ) tags(dev nchar(50), tag2 binary(16))") + "create table if not exists st (ts timestamp, t1 int, t2 timestamp, t3 bigint, t4 float, t5 double, t6 binary(10), t7 smallint, t8 tinyint, t9 bool, t10 nchar(10), t11 int unsigned, t12 bigint unsigned, t13 smallint unsigned, t14 tinyint unsigned ,t15 int) tags(dev nchar(50), tag2 binary(16))") tdSql.execute( 'CREATE TABLE if not exists dev_001 using st tags("dev_01", "tag_01")') tdSql.execute( @@ -39,7 +40,7 @@ class TDTestCase: print("==============step2") tdSql.execute( - "INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.000', 1, '2020-05-13 10:00:00.000', 10, 3.1, 3.14, 'test', -10, -126, true, '测试', 15, 10, 65534, 254)('2020-05-13 10:00:01.000', 1, '2020-05-13 10:00:01.000', 10, 3.1, 3.14, 'test', -10, -126, true, '测试', 15, 10, 65534, 253)('2020-05-13 10:00:02.000', 10, '2020-05-13 10:00:00.000', 11, 3.1, 3.14, 'test', 10, -127, true, '测试', 15, 10, 65534, 253)('2020-05-13 10:00:03.000', 1, '2020-05-13 10:00:00.000', 11, 3.1, 3.14, 'test', -10, -126, true, '测试', 14, 12, 65532, 254)") + "INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.000', 1, '2020-05-13 10:00:00.000', 10, 3.1, 3.14, 'test', -10, -126, true, '测试', 15, 10, 65534, 254, 1)('2020-05-13 10:00:01.000', 1, '2020-05-13 10:00:01.000', 10, 3.1, 3.14, 'test', -10, -126, true, '测试', 15, 10, 65534, 253, 5)('2020-05-13 10:00:02.000', 10, '2020-05-13 10:00:00.000', 11, 3.1, 3.14, 'test', 10, -127, false, '测试', 15, 10, 65534, 253, 10)('2020-05-13 10:00:03.000', 1, '2020-05-13 10:00:00.000', 11, 3.1, 3.14, 'test', -10, -126, true, '测试', 14, 12, 65532, 254, 15)") for i in range(self.rowNum): tdSql.execute("insert into dev_002 (ts,t1) values(%d, %d,)" % (self.ts + i, i + 1)) @@ -55,7 +56,7 @@ class TDTestCase: tdSql.checkData(1, 0, 1) tdSql.query("select count(ts) from dev_001 state_window(t8)") tdSql.checkRows(3) - tdSql.checkData(2, 0, 1) + tdSql.checkData(2, 0, 1) tdSql.query("select count(ts) from dev_001 state_window(t11)") tdSql.checkRows(2) tdSql.checkData(0, 0, 3) @@ -69,16 +70,31 @@ class TDTestCase: tdSql.checkRows(3) tdSql.checkData(1, 0, 2) tdSql.query("select count(ts) from dev_002 state_window(t1)") - tdSql.checkRows(100000) + tdSql.checkRows(100000) + + # with all aggregate function + tdSql.query("select count(*),sum(t1),avg(t1),twa(t1),stddev(t15),leastsquares(t15,1,1),first(t15),last(t15),spread(t15),percentile(t15,90),t9 from dev_001 state_window(t9);") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 2) + tdSql.checkData(1, 1, 10) + tdSql.checkData(0, 2, 1) + # tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, np.std([1,5])) + # tdSql.checkData(0, 5, 1) + tdSql.checkData(0, 6, 1) + tdSql.checkData(0, 7, 5) + tdSql.checkData(0, 8, 4) + tdSql.checkData(0, 9, 4.6) + tdSql.checkData(0, 10, 'True') + + tdSql.query("select count(*),sum(t1),avg(t1),twa(t1),stddev(t15),leastsquares(t15,1,1),first(t10),spread(t15),t9 from dev_001 state_window(t9);") # error - tdSql.query("select count(ts) from dev_001 state_window(t7)") tdSql.error("select count(*) from dev_001 state_window(t2)") tdSql.error("select count(*) from st state_window(t3)") tdSql.error("select count(*) from dev_001 state_window(t4)") tdSql.error("select count(*) from dev_001 state_window(t5)") tdSql.error("select count(*) from dev_001 state_window(t6)") - tdSql.error("select count(*) from dev_001 state_window(t9)") tdSql.error("select count(*) from dev_001 state_window(t10)") tdSql.error("select count(*) from dev_001 state_window(tag2)") From 5e82ff44cb423198a1647b519f6043bcb1a588eb Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 19 Jun 2021 16:29:40 +0800 Subject: [PATCH 020/115] [TD-4783]:support timestamp filter expression on master query --- src/client/src/tscSQLParser.c | 11 ++++++----- tests/script/general/parser/where.sim | 15 ++++++++++++++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 9532d1e202..775b5f9022 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4798,6 +4798,12 @@ int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSq if ((ret = getColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) { goto PARSE_WHERE_EXIT; } + + if (taosArrayGetSize(pQueryInfo->pUpstream) > 0 ) { + if ((ret = getColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pTimewindow, TK_AND)) != TSDB_CODE_SUCCESS) { + goto PARSE_WHERE_EXIT; + } + } // 6. join condition if ((ret = getJoinCondInfo(&pSql->cmd, pQueryInfo, condExpr.pJoinExpr)) != TSDB_CODE_SUCCESS) { @@ -7804,11 +7810,6 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } - - if (pTableMeta->tableInfo.precision == TSDB_TIME_PRECISION_MILLI) { - pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; - pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; - } } // validate the interval info diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index 6dfea3d2e7..fbd59aafba 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -352,5 +352,18 @@ if $rows != 0 then return -1 endi - +print ==========================> td-4783 +sql create table where_ts(ts timestamp, f int) +sql insert into where_ts values('2021-06-19 16:22:00', 1); +sql insert into where_ts values('2021-06-19 16:23:00', 2); +sql insert into where_ts values('2021-06-19 16:24:00', 3); +sql insert into where_ts values('2021-06-19 16:25:00', 1); +sql select * from (select * from where_ts) where ts<'2021-06-19 16:25:00' and ts>'2021-06-19 16:22:00' +if $row != 2 then + return -1 +endi +print $data00, $data01 +if $data01 != 2 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From a0be6d9288cbfe7c4c194938b5c0202f8b19866c Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 19 Jun 2021 17:02:09 +0800 Subject: [PATCH 021/115] [TD-45145]: add testcase of stat_window function --- tests/pytest/query/queryStateWindow.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/pytest/query/queryStateWindow.py b/tests/pytest/query/queryStateWindow.py index db9eaf8672..251dbef658 100644 --- a/tests/pytest/query/queryStateWindow.py +++ b/tests/pytest/query/queryStateWindow.py @@ -87,7 +87,10 @@ class TDTestCase: tdSql.checkData(0, 9, 4.6) tdSql.checkData(0, 10, 'True') - tdSql.query("select count(*),sum(t1),avg(t1),twa(t1),stddev(t15),leastsquares(t15,1,1),first(t10),spread(t15),t9 from dev_001 state_window(t9);") + # with where + tdSql.query("select avg(t15),t9 from dev_001 where t9='true' state_window(t9);") + tdSql.checkData(0, 0, 7) + tdSql.checkData(0, 1, 'True') # error tdSql.error("select count(*) from dev_001 state_window(t2)") From c271e5f2a10dad81b178169c8e6de4df8c12b4fc Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Sat, 19 Jun 2021 11:32:54 +0000 Subject: [PATCH 022/115] [TD-4773]add test for binary insert --- tests/pytest/insert/binary.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 0d583aa2cc..0cbb7876c6 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -4,6 +4,8 @@ import sys from util.log import * from util.cases import * from util.sql import * +import subprocess +import os class TDTestCase: @@ -50,6 +52,10 @@ class TDTestCase: tdLog.info('==> $data00') tdLog.info("tdSql.checkData(0, 0, '34567')") tdSql.checkData(0, 0, '34567') + tdLog.info("insert into tb values (now+4a, \"'';\")") + config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '') + result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines()) + if "Query OK" not in result: tdLog.exit("err:insert '';") tdLog.info('drop database db') tdSql.execute('drop database db') tdLog.info('show databases') From 65eeca1964fda463446eef892cbe1efd81206255 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 20 Jun 2021 21:51:15 +0800 Subject: [PATCH 023/115] [TD-4734] session_window and state window support main query --- src/client/src/tscSQLParser.c | 11 +++++-- tests/pytest/query/querySession.py | 49 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 93d94c8711..3a8e2407c0 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7792,6 +7792,12 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf int32_t f = pExpr->base.functionId; if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); + } + + if ((timeWindowQuery || pQueryInfo->stateWindow) && f == TSDB_FUNC_LAST) { + pExpr->base.numOfParams = 1; + pExpr->base.param[0].i64 = TSDB_ORDER_ASC; + pExpr->base.param[0].nType = TSDB_DATA_TYPE_INT; } } @@ -7825,7 +7831,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateSessionNode(pCmd, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } - if (isTimeWindowQuery(pQueryInfo) || pQueryInfo->sessionWindow.gap > 0) { + if (isTimeWindowQuery(pQueryInfo)) { // check if the first column of the nest query result is timestamp column SColumn* pCol = taosArrayGetP(pQueryInfo->colList, 0); if (pCol->info.type != TSDB_DATA_TYPE_TIMESTAMP) { @@ -7934,8 +7940,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf return TSDB_CODE_TSC_INVALID_OPERATION; } - if ((isTimeWindowQuery(pQueryInfo) || pQueryInfo->sessionWindow.gap > 0) && - (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { + if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { return TSDB_CODE_TSC_INVALID_OPERATION; } diff --git a/tests/pytest/query/querySession.py b/tests/pytest/query/querySession.py index 620f755bcb..216ff68b71 100644 --- a/tests/pytest/query/querySession.py +++ b/tests/pytest/query/querySession.py @@ -51,38 +51,73 @@ class TDTestCase: tdSql.checkRows(15) tdSql.checkData(0, 1, 2) + # session(ts,5a) main query + tdSql.query("select count(*) from (select * from dev_001) session(ts,5a)") + tdSql.checkRows(15) + tdSql.checkData(0, 1, 2) + # session(ts,1s) tdSql.query("select count(*) from dev_001 session(ts,1s)") tdSql.checkRows(12) tdSql.checkData(0, 1, 5) + # session(ts,1s) main query + tdSql.query("select count(*) from (select * from dev_001) session(ts,1s)") + tdSql.checkRows(12) + tdSql.checkData(0, 1, 5) + tdSql.query("select count(*) from dev_001 session(ts,1000a)") tdSql.checkRows(12) tdSql.checkData(0, 1, 5) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1000a)") + tdSql.checkRows(12) + tdSql.checkData(0, 1, 5) + # session(ts,1m) tdSql.query("select count(*) from dev_001 session(ts,1m)") tdSql.checkRows(9) tdSql.checkData(0, 1, 8) + # session(ts,1m) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1m)") + tdSql.checkRows(9) + tdSql.checkData(0, 1, 8) + # session(ts,1h) tdSql.query("select count(*) from dev_001 session(ts,1h)") tdSql.checkRows(6) tdSql.checkData(0, 1, 11) + # session(ts,1h) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1h)") + tdSql.checkRows(6) + tdSql.checkData(0, 1, 11) + # session(ts,1d) tdSql.query("select count(*) from dev_001 session(ts,1d)") tdSql.checkRows(4) tdSql.checkData(0, 1, 13) + # session(ts,1d) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1d)") + tdSql.checkRows(4) + tdSql.checkData(0, 1, 13) + # session(ts,1w) tdSql.query("select count(*) from dev_001 session(ts,1w)") tdSql.checkRows(2) tdSql.checkData(0, 1, 15) + # session(ts,1w) + tdSql.query("select count(*) from (select * from dev_001) session(ts,1w)") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 15) + # session with where tdSql.query("select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)") + tdSql.checkRows(2) tdSql.checkData(0, 1, 13) tdSql.checkData(0, 2, 1) @@ -97,6 +132,20 @@ class TDTestCase: tdSql.checkData(0, 11, 1) tdSql.checkData(1, 11, 14) + # session with where main + + tdSql.query("select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)") + + tdSql.checkRows(2) + tdSql.checkData(0, 1, 13) + tdSql.checkData(0, 2, 1) + tdSql.checkData(0, 3, 13) + tdSql.checkData(0, 4, 7) + tdSql.checkData(0, 5, 91) + tdSql.checkData(0, 6, 1) + tdSql.checkData(0, 7, 13) + tdSql.checkData(0, 8, '{slop:1.000000, intercept:0.000000}') + # tdsql err tdSql.error("select * from dev_001 session(ts,1w)") tdSql.error("select count(*) from st session(ts,1w)") From c1cd4079db810ccb700eddcaeb2ed31d27b1b826 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 21 Jun 2021 00:24:46 +0800 Subject: [PATCH 024/115] [TD-4734] session_window and state window support main query --- src/client/src/tscSQLParser.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3a8e2407c0..b4115e5842 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7785,7 +7785,6 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } - // todo NOT support yet for(int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); From e73ddd1e6f9db6e7e36c617f39ce7fbdf170afd1 Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 21 Jun 2021 09:23:13 +0800 Subject: [PATCH 025/115] fix bug for disk full --- src/query/src/qExecutor.c | 6 +++++- src/query/src/qTsbuf.c | 30 +++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f97a0c4a74..69242adc2a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6720,7 +6720,11 @@ int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *p if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset; pTsBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); - + if (pTsBuf == NULL) { + code = TSDB_CODE_QRY_NO_DISKSPACE; + goto _error; + } + tsBufResetPos(pTsBuf); bool ret = tsBufNextPos(pTsBuf); diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 1f43c5b33c..f4c6e24290 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -2,6 +2,7 @@ #include "taoserror.h" #include "tscompression.h" #include "tutil.h" +#include "queryLog.h" static int32_t getDataStartOffset(); static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); @@ -633,10 +634,14 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); if (r != 0) { + qError("fseek failed, errno:%d", errno); return -1; } - fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); + if (fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f) != sizeof(STSBufFileHeader)) { + qError("fwrite failed, errno:%d", errno); + return -1; + } return 0; } @@ -856,9 +861,17 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); - UNUSED(ret); + if (ret == -1) { + qError("fseek failed, errno:%d", errno); + tsBufDestroy(pTSBuf); + return NULL; + } size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); - UNUSED(sz); + if (sz != len) { + qError("fwrite failed, errno:%d", errno); + tsBufDestroy(pTSBuf); + return NULL; + } pTSBuf->fileSize += len; pTSBuf->tsOrder = order; @@ -866,9 +879,16 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - STSBufUpdateHeader(pTSBuf, &header); + if (STSBufUpdateHeader(pTSBuf, &header) < 0) { + tsBufDestroy(pTSBuf); + return NULL; + } - fsync(fileno(pTSBuf->f)); + if (fsync(fileno(pTSBuf->f)) == -1) { + qError("fsync failed, errno:%d", errno); + tsBufDestroy(pTSBuf); + return NULL; + } return pTSBuf; } From 7b2a1265309f677ccc76052b67a99e1435abf4f4 Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 21 Jun 2021 09:41:22 +0800 Subject: [PATCH 026/115] fix bug --- src/query/src/qTsbuf.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index f4c6e24290..81ad0af736 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -637,9 +637,10 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { qError("fseek failed, errno:%d", errno); return -1; } - - if (fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f) != sizeof(STSBufFileHeader)) { - qError("fwrite failed, errno:%d", errno); + + size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); + if (ws != sizeof(STSBufFileHeader)) { + qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); return -1; } return 0; @@ -868,7 +869,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ } size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); if (sz != len) { - qError("fwrite failed, errno:%d", errno); + qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); tsBufDestroy(pTSBuf); return NULL; } From 1a458d66db33dd3cdaee074fa18bda07d1a9a19d Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 21 Jun 2021 09:45:26 +0800 Subject: [PATCH 027/115] fix bug --- src/query/src/qTsbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 81ad0af736..9c04c7c929 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -639,7 +639,7 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { } size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); - if (ws != sizeof(STSBufFileHeader)) { + if (ws != 1) { qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); return -1; } From 40cb2f55bb9e53353e24ac1e12855e95d8eb6ced Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Mon, 21 Jun 2021 10:13:27 +0800 Subject: [PATCH 028/115] change performance test tool to optimize write test --- tests/comparisonTest/tdengine/tdengineTest.c | 84 ++++++++++++++++++-- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/tests/comparisonTest/tdengine/tdengineTest.c b/tests/comparisonTest/tdengine/tdengineTest.c index d1cf3a1f98..0de419e036 100644 --- a/tests/comparisonTest/tdengine/tdengineTest.c +++ b/tests/comparisonTest/tdengine/tdengineTest.c @@ -181,8 +181,8 @@ void writeDataImp(void *param) { if (lastMachineid != machineid) { lastMachineid = machineid; - sqlLen += sprintf(sql + sqlLen, " dev%d using devices tags(%d,'%s',%d) values", - machineid, machineid, machinename, machinegroup); + sqlLen += sprintf(sql + sqlLen, " dev%d values", + machineid); } sqlLen += sprintf(sql + sqlLen, "(%" PRId64 ",%d,%f)", timestamp, temperature, humidity); @@ -192,7 +192,8 @@ void writeDataImp(void *param) { result = taos_query(taos, sql); code = taos_errno(result); if (code != 0) { - printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos)); + printf("insert into dev%d values (%" PRId64 ",%d,%f)\n",machineid, timestamp, temperature, humidity); + printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(result)); } taos_free_result(result); @@ -210,6 +211,7 @@ void writeDataImp(void *param) { result = taos_query(taos, sql); code = taos_errno(result); if (code != 0) { + // printf("insert into dev%d using devices tags(%d,'%s',%d) values (%" PRId64 ",%d,%f)",machineid, machineid, machinename, machinegroup, timestamp, temperature, humidity); printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos)); } taos_free_result(result); @@ -246,7 +248,7 @@ void writeData() { taos_free_result(result); result = taos_query(taos, - "create table if not exists db.devices(ts timestamp, temperature int, humidity float) " + "create stable if not exists db.devices(ts timestamp, temperature int, humidity float) " "tags(devid int, devname binary(16), devgroup int)"); code = taos_errno(result); if (code != 0) { @@ -254,6 +256,77 @@ void writeData() { } taos_free_result(result); + //create tables before insert the data + result = taos_query(taos, "use db"); + code = taos_errno(result); + if (code != 0) { + taos_error(result, taos); + } + taos_free_result(result); + + char *sql = calloc(1, 8*1024*1024); + int sqlLen = 0; + int lastMachineid = 0; + int counter = 0; + int totalRecords = 0; + for (int i = 0; i < arguments.filesNum; i++) { + char fileName[300]; + sprintf(fileName, "%s/testdata%d.csv", arguments.dataDir, i); + + FILE *fp = fopen(fileName, "r"); + if (fp == NULL) { + printf("failed to open file %s\n", fileName); + exit(1); + } + printf("open file %s success\n", fileName); + + char *line = NULL; + size_t len = 0; + while (!feof(fp)) { + free(line); + line = NULL; + len = 0; + + getline(&line, &len, fp); + if (line == NULL) break; + + if (strlen(line) < 10) continue; + + int machineid; + char machinename[16]; + int machinegroup; + int64_t timestamp; + int temperature; + float humidity; + sscanf(line, "%d%s%d%" PRId64 "%d%f", &machineid, machinename, &machinegroup, ×tamp, &temperature, &humidity); + + if (counter == 0) { + sqlLen = sprintf(sql, "create table if not exists"); + } + + if (lastMachineid != machineid) { + lastMachineid = machineid; + sqlLen += sprintf(sql + sqlLen, " dev%d using devices tags(%d,'%s',%d)", machineid, machineid, machinename, machinegroup); + } + counter++; + + if (counter >= arguments.rowsPerRequest) { + result = taos_query(taos, sql); + code = taos_errno(result); + if (code != 0) { + printf("create table error:%d reason:%s\n", code, taos_errstr(result)); + } + taos_free_result(result); + + totalRecords += counter; + counter = 0; + lastMachineid = -1; + sqlLen = 0; + } + } + fclose(fp); + } + int64_t st = getTimeStampMs(); int a = arguments.filesNum / arguments.clients; @@ -379,5 +452,4 @@ void readData() { } free(threads); -} - +} \ No newline at end of file From 3dc8eca0148bf1783a3783121d6a54bf6f60d053 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 21 Jun 2021 10:18:05 +0800 Subject: [PATCH 029/115] [td-4754] --- src/client/inc/tscUtil.h | 3 +- src/client/src/tscSQLParser.c | 38 +- src/client/src/tscServer.c | 1 + src/client/src/tscSubquery.c | 17 +- src/client/src/tscUtil.c | 34 +- src/query/inc/qAggMain.h | 2 +- src/query/inc/qExecutor.h | 2 +- src/query/inc/sql.y | 2 +- src/query/src/qAggMain.c | 995 +--------------------- src/query/src/qExecutor.c | 180 ++-- src/query/src/sql.c | 50 +- tests/script/general/parser/groupby.sim | 134 +-- tests/script/general/parser/having.sim | 2 +- tests/script/general/parser/testSuite.sim | 1 - 14 files changed, 311 insertions(+), 1150 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 35f3b42811..950d81fb61 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -210,7 +210,8 @@ int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaI void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta); SColumn* tscColumnClone(const SColumn* src); -bool tscColumnExists(SArray* pColumnList, int32_t columnIndex, uint64_t uid); +void tscColumnCopy(SColumn* pDest, const SColumn* pSrc); +int32_t tscColumnExists(SArray* pColumnList, int32_t columnId, uint64_t uid); SColumn* tscColumnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema); void tscColumnListDestroy(SArray* pColList); void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 0ab7c38186..cf94aaae35 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1668,7 +1668,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 // arithmetic expression always return result in the format of double float pExprInfo->base.resBytes = sizeof(double); - pExprInfo->base.interBytes = sizeof(double); + pExprInfo->base.interBytes = 0; pExprInfo->base.resType = TSDB_DATA_TYPE_DOUBLE; pExprInfo->base.functionId = TSDB_FUNC_ARITHM; @@ -1903,14 +1903,14 @@ SExprInfo* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tab index.columnIndex = colIndex; } - return tscExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, colId, pSchema->bytes, + return tscExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, colId, 0, (functionId == TSDB_FUNC_TAGPRJ)); } SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag, int16_t colId) { SExprInfo* pExpr = tscExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, - pColSchema->bytes, colId, pColSchema->bytes, TSDB_COL_IS_TAG(flag)); + pColSchema->bytes, colId, 0, TSDB_COL_IS_TAG(flag)); tstrncpy(pExpr->base.aliasName, pColSchema->name, sizeof(pExpr->base.aliasName)); tstrncpy(pExpr->base.token, pColSchema->name, sizeof(pExpr->base.token)); @@ -2065,7 +2065,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS type = pSchema->type; bytes = pSchema->bytes; } - + SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, getNewResColId(pCmd), bytes, false); tstrncpy(pExpr->base.aliasName, name, tListLen(pExpr->base.aliasName)); @@ -2134,6 +2134,17 @@ static void updateLastScanOrderIfNeeded(SQueryInfo* pQueryInfo) { } } +static UNUSED_FUNC void updateFunctionInterBuf(SQueryInfo* pQueryInfo, bool superTable) { + size_t numOfExpr = tscNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExpr; ++i) { + SExprInfo* pExpr = tscExprGet(pQueryInfo, i); + + int32_t param = (int32_t)pExpr->base.param[0].i64; + getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, pExpr->base.functionId, param, &pExpr->base.resType, &pExpr->base.resBytes, + &pExpr->base.interBytes, 0, superTable); + } +} + int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; int32_t functionId = pItem->pNode->functionId; @@ -2562,7 +2573,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // set the first column ts for top/bottom query SColumnIndex index1 = {index.tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pCmd), - TSDB_KEYSIZE, false); + 0, false); tstrncpy(pExpr->base.aliasName, aAggs[TSDB_FUNC_TS].name, sizeof(pExpr->base.aliasName)); const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; @@ -3077,15 +3088,10 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return true; } - if (pQueryInfo->groupbyExpr.numOfGroupCols != 1) { + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); + if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); return true; - } else { - SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); - if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return true; - } } } else if (tscIsSessionWindowQuery(pQueryInfo)) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -3636,7 +3642,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (!tscColumnExists(pTableMetaInfo->tagColList, index.columnIndex, pTableMetaInfo->pTableMeta->id.uid)) { + if (tscColumnExists(pTableMetaInfo->tagColList, pTagSchema1->colId, pTableMetaInfo->pTableMeta->id.uid) < 0) { tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema1); if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { @@ -3668,7 +3674,7 @@ static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tS if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMeta); - if (!tscColumnExists(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid)) { + if (tscColumnExists(pTableMetaInfo->tagColList, pTagSchema2->colId, pTableMeta->id.uid) < 0) { tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pTagSchema2); if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { @@ -7830,6 +7836,9 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if ((code = doFunctionsCompatibleCheck(pCmd, pQueryInfo, tscGetErrorMsgPayload(pCmd))) != TSDB_CODE_SUCCESS) { return code; } + +// updateFunctionInterBuf(pQueryInfo, false); + } else { pQueryInfo->command = TSDB_SQL_SELECT; @@ -7958,6 +7967,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf updateLastScanOrderIfNeeded(pQueryInfo); tscFieldInfoUpdateOffset(pQueryInfo); +// updateFunctionInterBuf(pQueryInfo, isSTable); if ((code = validateFillNode(pCmd, pQueryInfo, pSqlNode)) != TSDB_CODE_SUCCESS) { return code; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8c5e99474d..c3cf63bd26 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -795,6 +795,7 @@ static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, pSqlExpr->colBytes = htons(pExpr->colBytes); pSqlExpr->resType = htons(pExpr->resType); pSqlExpr->resBytes = htons(pExpr->resBytes); + pSqlExpr->interBytes = htonl(pExpr->interBytes); pSqlExpr->functionId = htons(pExpr->functionId); pSqlExpr->numOfParams = htons(pExpr->numOfParams); pSqlExpr->resColId = htons(pExpr->resColId); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 22a603b71e..c3df4773e1 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -103,13 +103,6 @@ bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx) { pthread_mutex_lock(&subState->mutex); -// bool done = allSubqueryDone(pParentSql); -// if (done) { -// tscDebug("0x%"PRIx64" subquery:0x%"PRIx64",%d all subs already done", pParentSql->self, pSql->self, idx); -// pthread_mutex_unlock(&subState->mutex); -// return false; -// } - tscDebug("0x%"PRIx64" subquery:0x%"PRIx64", index:%d state set to 1", pParentSql->self, pSql->self, idx); subState->states[idx] = 1; @@ -2389,8 +2382,14 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { SColumn *pCol = taosArrayGetP(pColList, i); if (pCol->info.flist.numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - SColumn *p = tscColumnClone(pCol); - taosArrayPush(pNewQueryInfo->colList, &p); + int32_t index1 = tscColumnExists(pNewQueryInfo->colList, pCol->columnIndex, pCol->tableUid); + if (index1 >= 0) { + SColumn* x = taosArrayGetP(pNewQueryInfo->colList, index1); + tscColumnCopy(x, pCol); + } else { + SColumn *p = tscColumnClone(pCol); + taosArrayPush(pNewQueryInfo->colList, &p); + } } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9d2c500a92..83ec886e19 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2270,18 +2270,14 @@ int32_t tscExprCopyAll(SArray* dst, const SArray* src, bool deepcopy) { return 0; } -bool tscColumnExists(SArray* pColumnList, int32_t columnIndex, uint64_t uid) { - // ignore the tbname columnIndex to be inserted into source list - if (columnIndex < 0) { - return false; - } - +// ignore the tbname columnIndex to be inserted into source list +int32_t tscColumnExists(SArray* pColumnList, int32_t columnId, uint64_t uid) { size_t numOfCols = taosArrayGetSize(pColumnList); int32_t i = 0; while (i < numOfCols) { SColumn* pCol = taosArrayGetP(pColumnList, i); - if ((pCol->columnIndex != columnIndex) || (pCol->tableUid != uid)) { + if ((pCol->info.colId != columnId) || (pCol->tableUid != uid)) { ++i; continue; } else { @@ -2290,10 +2286,10 @@ bool tscColumnExists(SArray* pColumnList, int32_t columnIndex, uint64_t uid) { } if (i >= numOfCols || numOfCols == 0) { - return false; + return -1; } - return true; + return i; } void tscExprAssign(SExprInfo* dst, const SExprInfo* src) { @@ -2379,13 +2375,7 @@ SColumn* tscColumnClone(const SColumn* src) { return NULL; } - dst->columnIndex = src->columnIndex; - dst->tableUid = src->tableUid; - dst->info.flist.numOfFilters = src->info.flist.numOfFilters; - dst->info.flist.filterInfo = tFilterInfoDup(src->info.flist.filterInfo, src->info.flist.numOfFilters); - dst->info.type = src->info.type; - dst->info.colId = src->info.colId; - dst->info.bytes = src->info.bytes; + tscColumnCopy(dst, src); return dst; } @@ -2394,6 +2384,18 @@ static void tscColumnDestroy(SColumn* pCol) { free(pCol); } +void tscColumnCopy(SColumn* pDest, const SColumn* pSrc) { + destroyFilterInfo(&pDest->info.flist); + + pDest->columnIndex = pSrc->columnIndex; + pDest->tableUid = pSrc->tableUid; + pDest->info.flist.numOfFilters = pSrc->info.flist.numOfFilters; + pDest->info.flist.filterInfo = tFilterInfoDup(pSrc->info.flist.filterInfo, pSrc->info.flist.numOfFilters); + pDest->info.type = pSrc->info.type; + pDest->info.colId = pSrc->info.colId; + pDest->info.bytes = pSrc->info.bytes; +} + void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid) { assert(src != NULL && dst != NULL); diff --git a/src/query/inc/qAggMain.h b/src/query/inc/qAggMain.h index 57e7d2982f..044c538f47 100644 --- a/src/query/inc/qAggMain.h +++ b/src/query/inc/qAggMain.h @@ -204,7 +204,7 @@ typedef struct SAggFunctionInfo { bool (*init)(SQLFunctionCtx *pCtx); // setup the execute environment void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function - void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version, todo merge with blockwise function +// void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version, todo merge with blockwise function // finalizer must be called after all xFunction has been executed to generated final result. void (*xFinalize)(SQLFunctionCtx *pCtx); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 955dd734cf..9cd1c5b033 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -295,7 +295,7 @@ enum OPERATOR_TYPE_E { OP_MultiTableAggregate = 14, OP_MultiTableTimeInterval = 15, OP_DummyInput = 16, //TODO remove it after fully refactor. - OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. + OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. OP_GlobalAggregate = 18, // global merge for the multi-way data sources. OP_Filter = 19, OP_Distinct = 20, diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 63bfd85976..8f9b3a8e61 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -469,7 +469,7 @@ tagitem(A) ::= PLUS(X) FLOAT(Y). { //////////////////////// The SELECT statement ///////////////////////////////// %type select {SSqlNode*} %destructor select {destroySqlNode($$);} -select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) session_option(H) windowstate_option(D) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). { +select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) session_option(H) windowstate_option(D) fill_opt(F) sliding_opt(S) groupby_opt(P) having_opt(N) orderby_opt(Z) slimit_opt(G) limit_opt(L). { A = tSetQuerySqlNode(&T, W, X, Y, P, Z, &K, &H, &D, &S, F, &L, &G, N); } diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index bc14c75af5..3dff0f5d50 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -456,20 +456,6 @@ static void count_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); } -static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SET_VAL(pCtx, 1, 1); - *((int64_t *)pCtx->pOutput) += pCtx->size; - - // do not need it actually - SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); - pInfo->hasResult = DATA_SET_FLAG; -} - static void count_func_merge(SQLFunctionCtx *pCtx) { int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { @@ -609,46 +595,6 @@ static void do_sum(SQLFunctionCtx *pCtx) { } } -static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SET_VAL(pCtx, 1, 1); - int64_t *res = (int64_t*) pCtx->pOutput; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - *res += GET_INT8_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - *res += GET_INT16_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - *res += GET_INT32_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - *res += GET_INT64_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - uint64_t *r = (uint64_t *)pCtx->pOutput; - *r += GET_UINT8_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - uint64_t *r = (uint64_t *)pCtx->pOutput; - *r += GET_UINT16_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - uint64_t *r = (uint64_t *)pCtx->pOutput; - *r += GET_UINT32_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - uint64_t *r = (uint64_t *)pCtx->pOutput; - *r += GET_UINT64_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = (double*) pCtx->pOutput; - *retVal += GET_DOUBLE_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double*) pCtx->pOutput; - *retVal += GET_FLOAT_VAL(pData); - } - - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - static void sum_function(SQLFunctionCtx *pCtx) { do_sum(pCtx); @@ -661,17 +607,6 @@ static void sum_function(SQLFunctionCtx *pCtx) { } } -static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { - do_sum_f(pCtx, index); - - // keep the result data in output buffer, not in the intermediate buffer - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; - pSum->hasResult = DATA_SET_FLAG; - } -} - static void sum_func_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; @@ -847,53 +782,6 @@ static void avg_function(SQLFunctionCtx *pCtx) { } } -static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SET_VAL(pCtx, 1, 1); - - // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - pAvgInfo->sum += GET_INT8_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - pAvgInfo->sum += GET_INT16_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - pAvgInfo->sum += GET_INT32_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - pAvgInfo->sum += GET_INT64_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - pAvgInfo->sum += GET_DOUBLE_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - pAvgInfo->sum += GET_FLOAT_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - pAvgInfo->sum += GET_UINT8_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - pAvgInfo->sum += GET_UINT16_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - pAvgInfo->sum += GET_UINT32_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - pAvgInfo->sum += GET_UINT64_VAL(pData); - } - - // restore sum and count of elements - pAvgInfo->num += 1; - - // set has result flag - pResInfo->hasResult = DATA_SET_FLAG; - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - static void avg_func_merge(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -1307,78 +1195,6 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { } } -static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { - char *pData = GET_INPUT_DATA(pCtx, index); - TSKEY key = GET_TS_DATA(pCtx, index); - - int32_t num = 0; - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int8_t *output = (int8_t *)pCtx->pOutput; - int8_t i = GET_INT8_VAL(pData); - - UPDATE_DATA(pCtx, *output, i, num, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *output = (int16_t*) pCtx->pOutput; - int16_t i = GET_INT16_VAL(pData); - - UPDATE_DATA(pCtx, *output, i, num, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *output = (int32_t*) pCtx->pOutput; - int32_t i = GET_INT32_VAL(pData); - - UPDATE_DATA(pCtx, *output, i, num, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *output = (int64_t*) pCtx->pOutput; - int64_t i = GET_INT64_VAL(pData); - - UPDATE_DATA(pCtx, *output, i, num, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *output = (float*) pCtx->pOutput; - float i = GET_FLOAT_VAL(pData); - - UPDATE_DATA(pCtx, *output, i, num, isMin, key); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *output = (double*) pCtx->pOutput; - double i = GET_DOUBLE_VAL(pData); - - UPDATE_DATA(pCtx, *output, i, num, isMin, key); - } - - GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SET_VAL(pCtx, 1, 1); - minMax_function_f(pCtx, index, 0); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - char *flag = pCtx->pOutput + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - -static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SET_VAL(pCtx, 1, 1); - minMax_function_f(pCtx, index, 1); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - char *flag = pCtx->pOutput + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - #define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, _type, num) \ for (int32_t i = 0; i < (ctx)->size; ++i) { \ if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], (_type))) { \ @@ -1472,114 +1288,6 @@ static void stddev_function(SQLFunctionCtx *pCtx) { } } -static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { - // the second stage to calculate standard deviation - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) { - pStd->stage++; - avg_finalizer(pCtx); - - pResInfo->initialized = true; // set it initialized to avoid re-initialization - - // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); - - pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); - assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); - } - - /* the first stage is to calculate average value */ - if (pStd->stage == 0) { - avg_function_f(pCtx, index); - } else if (pStd->num > 0) { - double avg = pStd->avg; - void * pData = GET_INPUT_DATA(pCtx, index); - - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - pStd->res += POW2(GET_INT32_VAL(pData) - avg); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - pStd->res += POW2(GET_FLOAT_VAL(pData) - avg); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - pStd->res += POW2(GET_DOUBLE_VAL(pData) - avg); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - pStd->res += POW2(GET_INT64_VAL(pData) - avg); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - pStd->res += POW2(GET_INT16_VAL(pData) - avg); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - pStd->res += POW2(GET_INT8_VAL(pData) - avg); - break; - } - case TSDB_DATA_TYPE_UINT: { - pStd->res += POW2(GET_UINT32_VAL(pData) - avg); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - pStd->res += POW2(GET_UINT64_VAL(pData) - avg); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - pStd->res += POW2(GET_UINT16_VAL(pData) - avg); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - pStd->res += POW2(GET_UINT8_VAL(pData) - avg); - break; - } - default: - qError("stddev function not support data type:%d", pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } -} - -static UNUSED_FUNC void stddev_next_step(SQLFunctionCtx *pCtx) { - /* - * the stddevInfo and the average info struct share the same buffer area - * And the position of each element in their struct is exactly the same matched - */ - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); - - if (pStd->stage == 0) { - /* - * stddev is calculated in two stage: - * 1. get the average value of all data; - * 2. get final result, based on the average values; - * so, if this routine is in second stage, no further step is required - */ - pStd->stage++; - avg_finalizer(pCtx); - - pResInfo->initialized = true; // set it initialized to avoid re-initialization - - // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); - - pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); - assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); - } else { - pResInfo->complete = true; - } -} - static void stddev_finalizer(SQLFunctionCtx *pCtx) { SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -1696,97 +1404,6 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); } -static void stddev_dst_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - // the second stage to calculate standard deviation - SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - double *retVal = &pStd->res; - - // all data are null, no need to proceed - SArray* resList = (SArray*) pCtx->param[0].pz; - if (resList == NULL) { - return; - } - - // find the correct group average results according to the tag value - int32_t len = (int32_t) taosArrayGetSize(resList); - assert(len > 0); - - double avg = 0; - if (len == 1) { - SResPair* p = taosArrayGet(resList, 0); - avg = p->avg; - } else { // todo opt performance by using iterator since the timestamp lsit is matched with the output result - SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare); - assert(p != NULL); - - avg = p->avg; - } - - int32_t num = 0; - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += POW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - qError("stddev function not support data type:%d", pCtx->inputType); - } - - pStd->num += num; - SET_VAL(pCtx, num, 1); - - // copy to the final output buffer for super table - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); -} - - static void stddev_dst_merge(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -1833,7 +1450,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx) { // todo opt for null block static void first_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC /*|| pCtx->preAggVals.dataBlockLoaded == false*/) { + if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1862,27 +1479,6 @@ static void first_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SET_VAL(pCtx, 1, 1); - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - - TSKEY ts = GET_TS_DATA(pCtx, index); - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - - SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->complete = true; // get the first not-null data, completed -} - static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = GET_TS_LIST(pCtx); @@ -1932,21 +1528,6 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - first_data_assign_impl(pCtx, pData, index); - - SET_VAL(pCtx, 1, 1); -} - static void first_dist_func_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); @@ -1978,70 +1559,55 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SQLFunctionCtx *pCtx) { - if (pCtx->order != pCtx->param[0].i64/* || pCtx->preAggVals.dataBlockLoaded == false*/) { - return; - } - - int32_t notNullElems = 0; - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - if (!pCtx->requireNull) { - continue; - } - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - - TSKEY ts = GET_TS_DATA(pCtx, i); - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - - SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); - pInfo->hasResult = DATA_SET_FLAG; - - pInfo->complete = true; // set query completed on this column - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - // the scan order is not the required order, ignore it if (pCtx->order != pCtx->param[0].i64) { return; } + SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); + + int32_t notNullElems = 0; if (pCtx->order == TSDB_ORDER_DESC) { - SET_VAL(pCtx, 1, 1); - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - TSKEY ts = GET_TS_DATA(pCtx, index); - DO_UPDATE_TAG_COLUMNS(pCtx, ts); + for (int32_t i = pCtx->size - 1; i >= 0; --i) { + char *data = GET_INPUT_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { + continue; + } - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - pResInfo->complete = true; // set query completed - } else { // in case of ascending order check, all data needs to be checked - SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); - TSKEY ts = GET_TS_DATA(pCtx, index); + memcpy(pCtx->pOutput, data, pCtx->inputBytes); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); - if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { - pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - - *(TSKEY*)buf = ts; + TSKEY ts = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, ts); + + pResInfo->hasResult = DATA_SET_FLAG; + pResInfo->complete = true; // set query completed on this column + notNullElems++; + break; + } + } else { // ascending order + for (int32_t i = pCtx->size - 1; i >= 0; --i) { + char *data = GET_INPUT_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { + continue; + } + + TSKEY ts = GET_TS_DATA(pCtx, i); + + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { + pResInfo->hasResult = DATA_SET_FLAG; + memcpy(pCtx->pOutput, data, pCtx->inputBytes); + + *(TSKEY*)buf = ts; + DO_UPDATE_TAG_COLUMNS(pCtx, ts); + } + + notNullElems++; + break; } } + + SET_VAL(pCtx, notNullElems, 1); } static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { @@ -2092,29 +1658,6 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->size == 0) { - return; - } - - char *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - /* - * 1. for scan data in asc order, no need to check data - * 2. for data blocks that are not loaded, no need to check data - */ - if (pCtx->order != pCtx->param[0].i64) { - return; - } - - last_data_assign_impl(pCtx, pData, index); - - SET_VAL(pCtx, 1, 1); -} - /* * in the secondary merge(local reduce), the output is limited by the * final output size, so the main difference between last_dist_func_merge and second_merge @@ -2616,28 +2159,6 @@ static void top_function(SQLFunctionCtx *pCtx) { } } -static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - assert(pRes->num >= 0); - - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { - buildTopBotStruct(pRes, pCtx); - } - - SET_VAL(pCtx, 1, 1); - TSKEY ts = GET_TS_DATA(pCtx, index); - - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; -} - static void top_func_merge(SQLFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); @@ -2695,27 +2216,6 @@ static void bottom_function(SQLFunctionCtx *pCtx) { } } -static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_DATA(pCtx, index); - TSKEY ts = GET_TS_DATA(pCtx, index); - - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { - buildTopBotStruct(pRes, pCtx); - } - - SET_VAL(pCtx, 1, 1); - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; -} - static void bottom_func_merge(SQLFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); @@ -2868,50 +2368,6 @@ static void percentile_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { - pInfo->stage += 1; - - // all data are null, set it completed - if (pInfo->numOfElems == 0) { - pResInfo->complete = true; - - return; - } else { - pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); - } - } - - if (pInfo->stage == 0) { - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if (v < GET_DOUBLE_VAL(&pInfo->minval)) { - SET_DOUBLE_VAL(&pInfo->minval, v); - } - - if (v > GET_DOUBLE_VAL(&pInfo->maxval)) { - SET_DOUBLE_VAL(&pInfo->maxval, v); - } - - pInfo->numOfElems += 1; - return; - } - - tMemBucketPut(pInfo->pMemBucket, pData, 1); - - SET_VAL(pCtx, 1, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].dKey; @@ -2930,24 +2386,6 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -static UNUSED_FUNC void percentile_next_step(SQLFunctionCtx *pCtx) { - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pInfo->stage == 0) { - // all data are null, set it completed - if (pInfo->numOfElems == 0) { - pResInfo->complete = true; - } else { - pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); - } - - pInfo->stage += 1; - } else { - pResInfo->complete = true; - } -} - ////////////////////////////////////////////////////////////////////////////////// static void buildHistogramInfo(SAPercentileInfo* pInfo) { pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); @@ -3012,24 +2450,6 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { } } -static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - tHistogramAdd(&pInfo->pHisto, v); - - SET_VAL(pCtx, 1, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - static void apercentile_func_merge(SQLFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); @@ -3213,60 +2633,6 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); } -static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - double(*param)[3] = pInfo->mat; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *p = pData; - LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - int8_t *p = pData; - LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *p = pData; - LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *p = pData; - LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *p = pData; - LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *p = pData; - LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); - break; - } - default: - qError("error data type in leastsquare function:%d", pCtx->inputType); - }; - - SET_VAL(pCtx, 1, 1); - pInfo->num += 1; - - if (pInfo->num > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } -} - static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { // no data in query SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); @@ -3319,7 +2685,6 @@ static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); } - char *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { memcpy(pCtx->pOutput, pData, (size_t) pCtx->size * pCtx->inputBytes); @@ -3331,22 +2696,6 @@ static void col_project_function(SQLFunctionCtx *pCtx) { } } -static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 - return; - } - - // only one output - if (pCtx->param[0].i64 == 1 && pResInfo->numOfRes >= 1) { - return; - } - - INC_INIT_VAL(pCtx, 1); - char *pData = GET_INPUT_DATA(pCtx, index); - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); -} - /** * only used for tag projection query in select clause * @param pCtx @@ -3368,13 +2717,6 @@ static void tag_project_function(SQLFunctionCtx *pCtx) { } } -static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { - INC_INIT_VAL(pCtx, 1); - - tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->tag.nType, true); - pCtx->pOutput += pCtx->outputBytes; -} - /** * used in group by clause. when applying group by tags, the tags value is * assign by using tag function. @@ -3393,11 +2735,6 @@ static void tag_function(SQLFunctionCtx *pCtx) { } } -static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { - SET_VAL(pCtx, 1, 1); - tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); -} - static void copy_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); @@ -3793,61 +3130,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { } } -static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - // the output start from the second source element - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is set - GET_RES_INFO(pCtx)->numOfRes += 1; - } - - int32_t step = 1/*GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - pCtx->param[1].nType = pCtx->inputType; - pCtx->param[1].i64 = *(int32_t *)pData; - } else { - *(int32_t *)pCtx->pOutput = *(int32_t *)pData - (int32_t)pCtx->param[1].i64; - pCtx->param[1].i64 = *(int32_t *)pData; - *(int64_t *)pCtx->ptsOutputBuf = GET_TS_DATA(pCtx, index); - } - break; - }; - case TSDB_DATA_TYPE_BIGINT: { - DIFF_IMPL(pCtx, pData, int64_t); - break; - }; - case TSDB_DATA_TYPE_DOUBLE: { - DIFF_IMPL(pCtx, pData, double); - break; - }; - case TSDB_DATA_TYPE_FLOAT: { - DIFF_IMPL(pCtx, pData, float); - break; - }; - case TSDB_DATA_TYPE_SMALLINT: { - DIFF_IMPL(pCtx, pData, int16_t); - break; - }; - case TSDB_DATA_TYPE_TINYINT: { - DIFF_IMPL(pCtx, pData, int8_t); - break; - }; - default: - qError("error input type"); - } - - if (GET_RES_INFO(pCtx)->numOfRes > 0) { - pCtx->pOutput += pCtx->outputBytes * step; - pCtx->ptsOutputBuf = (char *)pCtx->ptsOutputBuf + TSDB_KEYSIZE * step; - } -} - char *getArithColumnData(void *param, const char* name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; @@ -3870,16 +3152,6 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { arithmeticTreeTraverse(sas->pExprInfo->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); } -static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { - INC_INIT_VAL(pCtx, 1); - SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - - sas->offset = index; - arithmeticTreeTraverse(sas->pExprInfo->pExpr, 1, pCtx->pOutput, sas, pCtx->order, getArithColumnData); - - pCtx->pOutput += pCtx->outputBytes; -} - #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ { \ type *inputData = (type *)data; \ @@ -3998,49 +3270,6 @@ static void spread_function(SQLFunctionCtx *pCtx) { } } -static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SET_VAL(pCtx, 1, 1); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - double val = 0.0; - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - val = GET_INT8_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - val = GET_INT16_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - val = GET_INT32_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - val = (double)(GET_INT64_VAL(pData)); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - val = GET_DOUBLE_VAL(pData); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - val = GET_FLOAT_VAL(pData); - } - - // keep the result data in output buffer, not in the intermediate buffer - if (val > pInfo->max) { - pInfo->max = val; - } - - if (val < pInfo->min) { - pInfo->min = val; - } - - pResInfo->hasResult = DATA_SET_FLAG; - pInfo->hasResult = DATA_SET_FLAG; - - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - } -} - /* * here we set the result value back to the intermediate buffer, to apply the finalize the function * the final result is generated in spread_function_finalizer @@ -4393,26 +3622,6 @@ static void twa_function(SQLFunctionCtx *pCtx) { } } -static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - int32_t notNullElems = twa_function_impl(pCtx, index, 1); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - } - - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); - } -} - /* * To copy the input to interResBuf to avoid the input buffer space be over writen * by next input data. The TWA function only applies to each table, so no merge procedure @@ -4590,23 +3799,6 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - STSBuf *pTSbuf = pInfo->pTSBuf; - - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, pData, TSDB_KEYSIZE); - SET_VAL(pCtx, pCtx->size, 1); - - pResInfo->hasResult = DATA_SET_FLAG; -} - static void ts_comp_finalize(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -4736,46 +3928,6 @@ static void rate_function(SQLFunctionCtx *pCtx) { } } -static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[index]; - } - - if (INT64_MIN == pRateInfo->lastValue) { - pRateInfo->lastValue = v; - } else if (v < pRateInfo->lastValue) { - pRateInfo->correctionValue += pRateInfo->lastValue; - } - - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[index]; - - SET_VAL(pCtx, 1, 1); - - // set has result flag - pRateInfo->hasResult = DATA_SET_FLAG; - pResInfo->hasResult = DATA_SET_FLAG; - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -4846,39 +3998,6 @@ static void irate_function(SQLFunctionCtx *pCtx) { } } -static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_DATA(pCtx, index); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return; - } - - // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - pRateInfo->firstKey = pRateInfo->lastKey; - pRateInfo->firstValue = pRateInfo->lastValue; - - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[index]; - -// qDebug("====%p irate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->firstValue , pRateInfo->firstKey); - SET_VAL(pCtx, 1, 1); - - // set has result flag - pRateInfo->hasResult = DATA_SET_FLAG; - pResInfo->hasResult = DATA_SET_FLAG; - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - void blockInfo_func(SQLFunctionCtx* pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); @@ -5062,7 +4181,7 @@ int32_t functionCompatList[] = { // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, stddev_dst, interp rate irate 1, 1, 1, 1, -1, 1, 1, 1, 5, 1, 1, // tid_tag, blk_info - 6, 7 + 6, 7 }; SAggFunctionInfo aAggs[] = {{ @@ -5073,7 +4192,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO, function_setup, count_function, - count_function_f, doFinalizer, count_func_merge, countRequired, @@ -5086,7 +4204,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO, function_setup, sum_function, - sum_function_f, function_finalizer, sum_func_merge, statisRequired, @@ -5099,7 +4216,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO, function_setup, avg_function, - avg_function_f, avg_finalizer, avg_func_merge, statisRequired, @@ -5112,7 +4228,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, min_func_setup, min_function, - min_function_f, function_finalizer, min_func_merge, statisRequired, @@ -5125,7 +4240,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, max_func_setup, max_function, - max_function_f, function_finalizer, max_func_merge, statisRequired, @@ -5138,7 +4252,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, function_setup, stddev_function, - stddev_function_f, stddev_finalizer, noop1, dataBlockRequired, @@ -5151,7 +4264,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, percentile_function_setup, percentile_function, - percentile_function_f, percentile_finalizer, noop1, dataBlockRequired, @@ -5164,7 +4276,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE, apercentile_function_setup, apercentile_function, - apercentile_function_f, apercentile_finalizer, apercentile_func_merge, dataBlockRequired, @@ -5177,7 +4288,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, function_setup, first_function, - first_function_f, function_finalizer, noop1, firstFuncRequired, @@ -5190,7 +4300,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_SELECTIVITY, function_setup, last_function, - last_function_f, function_finalizer, noop1, lastFuncRequired, @@ -5204,7 +4313,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, last_row_function, - noop2, last_row_finalizer, last_dist_func_merge, dataBlockRequired, @@ -5218,7 +4326,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SELECTIVITY, top_bottom_function_setup, top_function, - top_function_f, top_bottom_func_finalizer, top_func_merge, dataBlockRequired, @@ -5232,7 +4339,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SELECTIVITY, top_bottom_function_setup, bottom_function, - bottom_function_f, top_bottom_func_finalizer, bottom_func_merge, dataBlockRequired, @@ -5245,7 +4351,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO, spread_function_setup, spread_function, - spread_function_f, spread_function_finalizer, spread_func_merge, countRequired, @@ -5258,7 +4363,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, twa_function_setup, twa_function, - twa_function_f, twa_function_finalizer, twa_function_copy, dataBlockRequired, @@ -5271,7 +4375,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, leastsquares_function_setup, leastsquares_function, - leastsquares_function_f, leastsquares_finalizer, noop1, dataBlockRequired, @@ -5284,7 +4387,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, date_col_output_function, - date_col_output_function_f, doFinalizer, copy_function, noDataRequired, @@ -5297,7 +4399,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, noop1, - noop2, doFinalizer, copy_function, dataBlockRequired, @@ -5310,7 +4411,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO, function_setup, tag_function, - noop2, doFinalizer, copy_function, noDataRequired, @@ -5323,7 +4423,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_NEED_TS, ts_comp_function_setup, ts_comp_function, - ts_comp_function_f, ts_comp_finalize, copy_function, dataBlockRequired, @@ -5336,7 +4435,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO, function_setup, tag_function, - tag_function_f, doFinalizer, copy_function, noDataRequired, @@ -5349,7 +4447,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_MO | TSDB_FUNCSTATE_NEED_TS, function_setup, col_project_function, - col_project_function_f, doFinalizer, copy_function, dataBlockRequired, @@ -5362,7 +4459,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_MO, function_setup, tag_project_function, - tag_project_function_f, doFinalizer, copy_function, noDataRequired, @@ -5375,7 +4471,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS, function_setup, arithmetic_function, - arithmetic_function_f, doFinalizer, copy_function, dataBlockRequired, @@ -5388,7 +4483,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, diff_function_setup, diff_function, - diff_function_f, doFinalizer, noop1, dataBlockRequired, @@ -5402,7 +4496,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, first_dist_function, - first_dist_function_f, function_finalizer, first_dist_func_merge, firstDistFuncRequired, @@ -5415,7 +4508,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, last_dist_function, - last_dist_function_f, function_finalizer, last_dist_func_merge, lastDistFuncRequired, @@ -5428,7 +4520,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, function_setup, stddev_dst_function, - stddev_dst_function_f, stddev_dst_finalizer, stddev_dst_merge, dataBlockRequired, @@ -5441,7 +4532,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_OF | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS , function_setup, interp_function, - do_sum_f, // todo filter handle doFinalizer, copy_function, dataBlockRequired, @@ -5454,7 +4544,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, rate_function_setup, rate_function, - rate_function_f, rate_finalizer, rate_func_copy, dataBlockRequired, @@ -5467,7 +4556,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, rate_function_setup, irate_function, - irate_function_f, rate_finalizer, rate_func_copy, dataBlockRequired, @@ -5480,7 +4568,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE, function_setup, noop1, - noop2, noop1, noop1, dataBlockRequired, @@ -5492,7 +4579,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_NEED_TS | TSDB_FUNCSTATE_SELECTIVITY, deriv_function_setup, deriv_function, - noop2, doFinalizer, noop1, dataBlockRequired, @@ -5505,7 +4591,6 @@ SAggFunctionInfo aAggs[] = {{ TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, function_setup, blockInfo_func, - noop2, blockinfo_func_finalizer, block_func_merge, dataBlockRequired, diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7b106c178d..689ab08415 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -161,7 +161,7 @@ static void setResultOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResul int32_t numOfCols, int32_t* rowCellInfoOffset); void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); @@ -309,7 +309,7 @@ static bool isProjQuery(SQueryAttr *pQueryAttr) { return true; } -static bool hasNullRv(SColIndex* pColIndex, SDataStatis *pStatis) { +static bool hasNull(SColIndex* pColIndex, SDataStatis *pStatis) { if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } @@ -708,12 +708,13 @@ static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBloc static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - bool hasPrev = pCtx[0].preAggVals.isSet; + bool hasAggregates = pCtx[0].preAggVals.isSet; for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].size = forwardStep; + pCtx[k].size = forwardStep; pCtx[k].startTs = pWin->skey; + // keep it temprarily char* start = pCtx[k].pInput; int32_t pos = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? offset : offset - (forwardStep - 1); @@ -725,20 +726,18 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx pCtx[k].ptsList = &tsCol[pos]; } - int32_t functionId = pCtx[k].functionId; - // not a whole block involved in query processing, statistics data can not be used // NOTE: the original value of isSet have been changed here if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { pCtx[k].preAggVals.isSet = false; } - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunction(&pCtx[k]); + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { + aAggs[pCtx[k].functionId].xFunction(&pCtx[k]); } // restore it - pCtx[k].preAggVals.isSet = hasPrev; + pCtx[k].preAggVals.isSet = hasAggregates; pCtx[k].pInput = start; } } @@ -847,9 +846,6 @@ static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, in } } -// window start key interpolation - - static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, int32_t rowIndex) { if (pDataBlock == NULL) { @@ -975,10 +971,9 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - int32_t functionId = pCtx[k].functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { pCtx[k].startTs = startTs;// this can be set during create the struct - aAggs[functionId].xFunction(&pCtx[k]); + aAggs[pCtx[k].functionId].xFunction(&pCtx[k]); } } } @@ -1287,6 +1282,15 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn return; } + int64_t* tsList = NULL; + SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); + if (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + tsList = (int64_t*) pFirstColData->pData; + } + + STimeWindow w = TSWINDOW_INITIALIZER; + + int32_t num = 0; for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { char* val = ((char*)pColInfoData->pData) + bytes * j; if (isNull(val, type)) { @@ -1294,33 +1298,51 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } // Compare with the previous row of this column, and do not set the output buffer again if they are identical. - if (pInfo->prevData == NULL || (memcmp(pInfo->prevData, val, bytes) != 0)) { - if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); - } - + if (pInfo->prevData == NULL) { + pInfo->prevData = malloc(bytes); memcpy(pInfo->prevData, val, bytes); - - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); - } - - int32_t ret = - setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } + num++; + continue; } - - // todo opt perf - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - pInfo->binfo.pCtx[k].size = 1; - int32_t functionId = pInfo->binfo.pCtx[k].functionId; - if (functionNeedToExecute(pRuntimeEnv, &pInfo->binfo.pCtx[k], functionId)) { - aAggs[functionId].xFunctionF(&pInfo->binfo.pCtx[k], j); - } + if (memcmp(pInfo->prevData, val, bytes) == 0) { + num++; + continue; } + + if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, + bytes); + } + + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, + item->groupIndex); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + + num = 1; + memcpy(pInfo->prevData, val, bytes); + } + + if (num > 0) { + char* val = ((char*)pColInfoData->pData) + bytes * (pSDataBlock->info.rows - num); + memcpy(pInfo->prevData, val, bytes); + + if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, + bytes); + } + + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, + item->groupIndex); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); } } @@ -1419,7 +1441,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasic // not assign result buffer yet, add new result buffer, TODO remove it char* d = pData; int16_t len = bytes; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(type)) { d = varDataVal(pData); len = varDataLen(pData); } @@ -1461,11 +1483,12 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD return -1; } -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // in case of timestamp column, always generated results. + int32_t functionId = pCtx->functionId; if (functionId == TSDB_FUNC_TS) { return true; } @@ -1505,7 +1528,7 @@ void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColInde pCtx->preAggVals.isSet = false; } - pCtx->hasNull = hasNullRv(pColIndex, pStatis); + pCtx->hasNull = hasNull(pColIndex, pStatis); // set the statistics data for primary time stamp column if (pCtx->functionId == TSDB_FUNC_SPREAD && pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { @@ -3470,6 +3493,7 @@ int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, return 0; } +// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -4675,8 +4699,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); pInfo->resultRowFactor = - (int32_t)(GET_ROW_PARAM_FOR_MULTIOUTPUT(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, - false)); + (int32_t)(GET_ROW_PARAM_FOR_MULTIOUTPUT(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, false)); pRuntimeEnv->scanFlag = MERGE_STAGE; // TODO init when creating pCtx @@ -5248,6 +5271,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); } + static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { @@ -6260,7 +6284,7 @@ static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { return true; } -static UNUSED_FUNC bool validateQueryTableCols(SQueriedTableInfo* pTableInfo, SSqlExpr** pExpr, int32_t numOfOutput, +static bool validateQueryTableCols(SQueriedTableInfo* pTableInfo, SSqlExpr** pExpr, int32_t numOfOutput, SColumnInfo* pTagCols, void* pMsg) { int32_t numOfTotal = pTableInfo->numOfCols + pTableInfo->numOfTags; if (pTableInfo->numOfCols < 0 || pTableInfo->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { @@ -6445,6 +6469,7 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { pExprMsg->resType = htons(pExprMsg->resType); pExprMsg->resBytes = htons(pExprMsg->resBytes); + pExprMsg->interBytes = htonl(pExprMsg->interBytes); pExprMsg->functionId = htons(pExprMsg->functionId); pExprMsg->numOfParams = htons(pExprMsg->numOfParams); @@ -6652,41 +6677,41 @@ _cleanup: return code; } - int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int32_t filterNum) { - if (filterNum <= 0) { - return TSDB_CODE_SUCCESS; - } - - *dst = calloc(filterNum, sizeof(*src)); - if (*dst == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy(*dst, src, sizeof(*src) * filterNum); - - for (int32_t i = 0; i < filterNum; i++) { - if ((*dst)[i].filterstr && dst[i]->len > 0) { - void *pz = calloc(1, (size_t)(*dst)[i].len + 1); - - if (pz == NULL) { - if (i == 0) { - free(*dst); - } else { - freeColumnFilterInfo(*dst, i); - } - - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy(pz, (void *)src->pz, (size_t)src->len + 1); - - (*dst)[i].pz = (int64_t)pz; - } - } - +int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int32_t filterNum) { + if (filterNum <= 0) { return TSDB_CODE_SUCCESS; } + *dst = calloc(filterNum, sizeof(*src)); + if (*dst == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy(*dst, src, sizeof(*src) * filterNum); + + for (int32_t i = 0; i < filterNum; i++) { + if ((*dst)[i].filterstr && dst[i]->len > 0) { + void *pz = calloc(1, (size_t)(*dst)[i].len + 1); + + if (pz == NULL) { + if (i == 0) { + free(*dst); + } else { + freeColumnFilterInfo(*dst, i); + } + + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy(pz, (void *)src->pz, (size_t)src->len + 1); + + (*dst)[i].pz = (int64_t)pz; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t buildArithmeticExprFromMsg(SExprInfo *pExprInfo, void *pQueryMsg) { qDebug("qmsg:%p create arithmetic expr from binary", pQueryMsg); @@ -6745,8 +6770,8 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp for (int32_t i = 0; i < numOfOutput; ++i) { pExprs[i].base = *pExprMsg[i]; - memset(pExprs[i].base.param, 0, sizeof(tVariant) * tListLen(pExprs[i].base.param)); + memset(pExprs[i].base.param, 0, sizeof(tVariant) * tListLen(pExprs[i].base.param)); for (int32_t j = 0; j < pExprMsg[i]->numOfParams; ++j) { tVariantAssign(&pExprs[i].base.param[j], &pExprMsg[i]->param[j]); } @@ -6821,6 +6846,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp return TSDB_CODE_QRY_INVALID_MSG; } + // todo remove it if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].base.resType, &pExprs[i].base.resBytes, &pExprs[i].base.interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { tfree(pExprs); diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 8764c6d0d3..8abeb23b0e 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -259,15 +259,15 @@ static const YYACTIONTYPE yy_action[] = { /* 500 */ 1019, 999, 265, 1036, 146, 150, 981, 274, 1035, 163, /* 510 */ 142, 269, 164, 157, 979, 733, 165, 224, 789, 263, /* 520 */ 166, 153, 894, 283, 290, 43, 191, 271, 39, 299, - /* 530 */ 890, 306, 73, 278, 1113, 996, 70, 47, 104, 154, + /* 530 */ 890, 306, 73, 278, 1113, 996, 70, 154, 104, 47, /* 540 */ 155, 1112, 281, 1109, 170, 313, 1105, 110, 279, 156, /* 550 */ 1104, 277, 158, 275, 273, 1101, 159, 171, 270, 912, /* 560 */ 40, 37, 44, 192, 878, 120, 876, 122, 123, 874, /* 570 */ 873, 250, 182, 871, 870, 869, 868, 867, 866, 185, /* 580 */ 187, 863, 861, 859, 857, 189, 854, 190, 46, 79, /* 590 */ 84, 272, 331, 1037, 115, 323, 324, 325, 326, 327, - /* 600 */ 328, 329, 214, 341, 234, 289, 831, 252, 251, 830, - /* 610 */ 211, 212, 254, 98, 99, 255, 829, 812, 811, 259, + /* 600 */ 328, 329, 214, 234, 289, 341, 831, 252, 251, 830, + /* 610 */ 211, 212, 254, 98, 99, 255, 829, 259, 812, 811, /* 620 */ 10, 264, 872, 284, 133, 710, 174, 134, 173, 913, /* 630 */ 172, 175, 177, 176, 135, 178, 865, 914, 864, 2, /* 640 */ 136, 950, 856, 855, 82, 30, 4, 266, 160, 161, @@ -335,15 +335,15 @@ static const YYCODETYPE yy_lookahead[] = { /* 500 */ 194, 240, 240, 267, 194, 194, 240, 194, 267, 244, /* 510 */ 60, 263, 194, 250, 194, 118, 194, 263, 112, 195, /* 520 */ 194, 254, 194, 124, 194, 194, 194, 263, 194, 194, - /* 530 */ 194, 194, 130, 263, 194, 255, 132, 129, 194, 253, + /* 530 */ 194, 194, 130, 263, 194, 255, 132, 253, 194, 129, /* 540 */ 252, 194, 128, 194, 194, 194, 194, 194, 123, 251, /* 550 */ 194, 122, 249, 121, 120, 194, 248, 194, 119, 194, /* 560 */ 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, /* 570 */ 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, /* 580 */ 194, 194, 194, 194, 194, 194, 194, 194, 134, 195, /* 590 */ 195, 195, 107, 195, 90, 89, 50, 86, 88, 54, - /* 600 */ 87, 85, 195, 82, 195, 195, 5, 5, 148, 5, - /* 610 */ 195, 195, 148, 201, 201, 5, 5, 94, 93, 137, + /* 600 */ 87, 85, 195, 195, 195, 82, 5, 5, 148, 5, + /* 610 */ 195, 195, 148, 201, 201, 5, 5, 137, 94, 93, /* 620 */ 80, 114, 195, 109, 196, 81, 206, 196, 210, 212, /* 630 */ 211, 209, 208, 207, 196, 205, 195, 214, 195, 202, /* 640 */ 196, 230, 195, 195, 115, 80, 197, 91, 247, 246, @@ -395,18 +395,18 @@ static const unsigned short int yy_shift_ofst[] = { /* 120 */ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, /* 130 */ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, /* 140 */ 9, 9, 9, 9, 450, 450, 450, 397, 397, 397, - /* 150 */ 450, 397, 450, 402, 404, 408, 399, 414, 425, 429, + /* 150 */ 450, 397, 450, 402, 404, 399, 410, 414, 425, 429, /* 160 */ 432, 434, 439, 454, 450, 450, 450, 485, 47, 47, /* 170 */ 450, 450, 504, 506, 546, 511, 510, 545, 513, 516, - /* 180 */ 485, 143, 450, 521, 521, 450, 521, 450, 521, 450, + /* 180 */ 485, 143, 450, 523, 523, 450, 523, 450, 523, 450, /* 190 */ 450, 734, 734, 27, 99, 99, 126, 99, 53, 180, /* 200 */ 283, 283, 283, 283, 193, 269, 273, 327, 327, 327, /* 210 */ 327, 230, 251, 250, 238, 172, 172, 234, 307, 342, /* 220 */ 348, 210, 317, 323, 338, 339, 341, 309, 297, 343, /* 230 */ 354, 355, 357, 358, 346, 359, 360, 70, 171, 406, /* 240 */ 362, 312, 316, 319, 455, 459, 325, 329, 364, 333, - /* 250 */ 400, 601, 460, 602, 604, 464, 610, 611, 523, 525, - /* 260 */ 482, 507, 514, 540, 529, 544, 565, 556, 561, 572, + /* 250 */ 400, 601, 460, 602, 604, 464, 610, 611, 524, 526, + /* 260 */ 480, 507, 514, 540, 529, 544, 565, 556, 561, 572, /* 270 */ 574, 575, 564, 577, 578, 580, 657, 581, 582, 584, /* 280 */ 571, 541, 576, 543, 585, 514, 589, 562, 590, 563, /* 290 */ 596, 665, 670, 671, 672, 673, 674, 597, 666, 606, @@ -435,8 +435,8 @@ static const short yy_reduce_ofst[] = { /* 120 */ 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, /* 130 */ 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, /* 140 */ 390, 391, 392, 393, 324, 394, 395, 248, 254, 264, - /* 150 */ 396, 270, 398, 280, 267, 286, 288, 298, 263, 303, - /* 160 */ 308, 401, 403, 405, 407, 409, 410, 411, 412, 413, + /* 150 */ 396, 270, 398, 280, 267, 284, 288, 298, 263, 303, + /* 160 */ 308, 401, 403, 405, 407, 408, 409, 411, 412, 413, /* 170 */ 415, 416, 417, 419, 418, 420, 422, 426, 424, 430, /* 180 */ 421, 423, 427, 428, 431, 441, 438, 443, 444, 447, /* 190 */ 448, 437, 449, @@ -457,7 +457,7 @@ static const YYACTIONTYPE yy_default[] = { /* 120 */ 877, 832, 875, 832, 832, 832, 832, 832, 832, 832, /* 130 */ 832, 832, 832, 832, 832, 832, 832, 862, 832, 832, /* 140 */ 832, 832, 832, 832, 853, 853, 853, 832, 832, 832, - /* 150 */ 853, 832, 853, 1047, 1051, 1045, 1033, 1041, 1032, 1028, + /* 150 */ 853, 832, 853, 1047, 1051, 1033, 1045, 1041, 1032, 1028, /* 160 */ 1026, 1024, 1023, 1055, 853, 853, 853, 893, 889, 889, /* 170 */ 853, 853, 911, 909, 907, 899, 905, 901, 903, 897, /* 180 */ 880, 832, 853, 887, 887, 853, 887, 853, 887, 853, @@ -1024,8 +1024,8 @@ static const char *const yyTokenName[] = { /* 249 */ "fill_opt", /* 250 */ "sliding_opt", /* 251 */ "groupby_opt", - /* 252 */ "orderby_opt", - /* 253 */ "having_opt", + /* 252 */ "having_opt", + /* 253 */ "orderby_opt", /* 254 */ "slimit_opt", /* 255 */ "limit_opt", /* 256 */ "union", @@ -1210,7 +1210,7 @@ static const char *const yyRuleName[] = { /* 158 */ "tagitem ::= MINUS FLOAT", /* 159 */ "tagitem ::= PLUS INTEGER", /* 160 */ "tagitem ::= PLUS FLOAT", - /* 161 */ "select ::= SELECT selcollist from where_opt interval_opt session_option windowstate_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 161 */ "select ::= SELECT selcollist from where_opt interval_opt session_option windowstate_option fill_opt sliding_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt", /* 162 */ "select ::= LP select RP", /* 163 */ "union ::= select", /* 164 */ "union ::= union UNION ALL select", @@ -1467,7 +1467,7 @@ tSqlExprListDestroy((yypminor->yy441)); case 239: /* tagNamelist */ case 249: /* fill_opt */ case 251: /* groupby_opt */ - case 252: /* orderby_opt */ + case 253: /* orderby_opt */ case 264: /* sortlist */ case 268: /* grouplist */ { @@ -1492,7 +1492,7 @@ destroyRelationInfo((yypminor->yy244)); } break; case 245: /* where_opt */ - case 253: /* having_opt */ + case 252: /* having_opt */ case 259: /* expr */ case 269: /* expritem */ { @@ -1961,7 +1961,7 @@ static const struct { { 242, -2 }, /* (158) tagitem ::= MINUS FLOAT */ { 242, -2 }, /* (159) tagitem ::= PLUS INTEGER */ { 242, -2 }, /* (160) tagitem ::= PLUS FLOAT */ - { 240, -14 }, /* (161) select ::= SELECT selcollist from where_opt interval_opt session_option windowstate_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + { 240, -14 }, /* (161) select ::= SELECT selcollist from where_opt interval_opt session_option windowstate_option fill_opt sliding_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ { 240, -3 }, /* (162) select ::= LP select RP */ { 256, -1 }, /* (163) union ::= select */ { 256, -4 }, /* (164) union ::= union UNION ALL select */ @@ -1998,8 +1998,8 @@ static const struct { { 249, -4 }, /* (195) fill_opt ::= FILL LP ID RP */ { 250, -4 }, /* (196) sliding_opt ::= SLIDING LP tmvar RP */ { 250, 0 }, /* (197) sliding_opt ::= */ - { 252, 0 }, /* (198) orderby_opt ::= */ - { 252, -3 }, /* (199) orderby_opt ::= ORDER BY sortlist */ + { 253, 0 }, /* (198) orderby_opt ::= */ + { 253, -3 }, /* (199) orderby_opt ::= ORDER BY sortlist */ { 264, -4 }, /* (200) sortlist ::= sortlist COMMA item sortorder */ { 264, -2 }, /* (201) sortlist ::= item sortorder */ { 266, -2 }, /* (202) item ::= ids cpxName */ @@ -2010,8 +2010,8 @@ static const struct { { 251, -3 }, /* (207) groupby_opt ::= GROUP BY grouplist */ { 268, -3 }, /* (208) grouplist ::= grouplist COMMA item */ { 268, -1 }, /* (209) grouplist ::= item */ - { 253, 0 }, /* (210) having_opt ::= */ - { 253, -2 }, /* (211) having_opt ::= HAVING expr */ + { 252, 0 }, /* (210) having_opt ::= */ + { 252, -2 }, /* (211) having_opt ::= HAVING expr */ { 255, 0 }, /* (212) limit_opt ::= */ { 255, -2 }, /* (213) limit_opt ::= LIMIT signed */ { 255, -4 }, /* (214) limit_opt ::= LIMIT signed OFFSET signed */ @@ -2677,9 +2677,9 @@ static void yy_reduce( } yymsp[-1].minor.yy506 = yylhsminor.yy506; break; - case 161: /* select ::= SELECT selcollist from where_opt interval_opt session_option windowstate_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 161: /* select ::= SELECT selcollist from where_opt interval_opt session_option windowstate_option fill_opt sliding_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ { - yylhsminor.yy236 = tSetQuerySqlNode(&yymsp[-13].minor.yy0, yymsp[-12].minor.yy441, yymsp[-11].minor.yy244, yymsp[-10].minor.yy166, yymsp[-4].minor.yy441, yymsp[-3].minor.yy441, &yymsp[-9].minor.yy340, &yymsp[-8].minor.yy259, &yymsp[-7].minor.yy348, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy441, &yymsp[0].minor.yy414, &yymsp[-1].minor.yy414, yymsp[-2].minor.yy166); + yylhsminor.yy236 = tSetQuerySqlNode(&yymsp[-13].minor.yy0, yymsp[-12].minor.yy441, yymsp[-11].minor.yy244, yymsp[-10].minor.yy166, yymsp[-4].minor.yy441, yymsp[-2].minor.yy441, &yymsp[-9].minor.yy340, &yymsp[-8].minor.yy259, &yymsp[-7].minor.yy348, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy441, &yymsp[0].minor.yy414, &yymsp[-1].minor.yy414, yymsp[-3].minor.yy166); } yymsp[-13].minor.yy236 = yylhsminor.yy236; break; diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 124e76e85c..507431f536 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -654,53 +654,91 @@ if $data31 != @20-03-27 05:10:19.000@ then return -1 endi -#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; -#if $rows != 40 then -# return -1 -#endi -# -#if $data01 != 1.000000000 then -# return -1 -#endi -#if $data02 != t1 then -# return -1 -#endi -#if $data03 != 1 then -# return -1 -#endi -#if $data04 != 1 then -# return -1 -#endi -# -#if $data11 != 1.000000000 then -# return -1 -#endi -#if $data12 != t1 then -# return -1 -#endi -#if $data13 != 1 then -# return -1 -#endi -#if $data14 != 1 then -# return -1 -#endi -# -#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1; -#if $rows != 2 then -# return -1 -#endi -# -#if $data11 != 1.000000000 then -# return -1 -#endi -#if $data12 != t2 then -# return -1 -#endi -#if $data13 != 1 then -# return -1 -#endi -#if $data14 != 2 then -# return -1 -#endi +print ===============> +sql select stddev(c),c from st where t2=1 or t2=2 group by c; +if $rows != 4 then + return -1 +endi + +if $data00 != 0.000000000 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data10 != 0.000000000 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + +if $data20 != 0.000000000 then + return -1 +endi + +if $data21 != 3 then + return -1 +endi + +if $data30 != 0.000000000 then + return -1 +endi + +if $data31 != 4 then + return -1 +endi + +sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; +if $rows != 40 then + return -1 +endi + +if $data01 != 1.000000000 then + return -1 +endi +if $data02 != t1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + +if $data11 != 1.000000000 then + return -1 +endi +if $data12 != t1 then + return -1 +endi +if $data13 != 1 then + return -1 +endi +if $data14 != 1 then + return -1 +endi + +sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1; +if $rows != 2 then + return -1 +endi + +if $data11 != 1.000000000 then + return -1 +endi +if $data12 != t2 then + return -1 +endi +if $data13 != 1 then + return -1 +endi +if $data14 != 2 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/having.sim b/tests/script/general/parser/having.sim index ddafdd7329..a8d2102bef 100644 --- a/tests/script/general/parser/having.sim +++ b/tests/script/general/parser/having.sim @@ -1,6 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 545e19edec..5f71138966 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -63,4 +63,3 @@ run general/parser/between_and.sim run general/parser/last_cache.sim run general/parser/nestquery.sim run general/parser/precision_ns.sim - From 9b4f437d49d581b02335ffb62afa24aa7f414e94 Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Mon, 21 Jun 2021 10:27:38 +0800 Subject: [PATCH 030/115] [fix] add jdbc nano second test cases --- .../TimestampPrecisionInNanoInJniTest.java | 54 ++----------------- 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java index d733209f25..68541dbe9d 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java @@ -70,6 +70,9 @@ public class TimestampPrecisionInNanoInJniTest { } private void checkCount(long count, ResultSet rs) throws SQLException { + if (count == 0) { + Assert(0); + } rs.next(); long test_count = rs.getLong(1); Assert.assertEquals(count, test_count); @@ -154,9 +157,6 @@ public class TimestampPrecisionInNanoInJniTest { public void canSelectLastRowFromWeatherForFirstCol() { try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("select last(ts) from " + ns_timestamp_db + ".weather"); - // int nanos = rs.getTimestamp(1).getNanos(); - // Assert.assertEquals(timestamp3 % 1000_000l * 1000, nanos); - // java.lang.AssertionError: expected:<123456000> but was:<468123456> checkTime(timestamp3, rs); } catch (SQLException e) { e.printStackTrace(); @@ -417,18 +417,6 @@ public class TimestampPrecisionInNanoInJniTest { } } - // @Test - // public void canQueryNotEqualToInDateTypeForFirstCol() { - // try (Statement stmt = conn.createStatement()) { - // ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <> '" + date3 + "'"); - // checkCount(1l, rs); - // rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts <> '" + date3 + "'"); - // checkTime(timestamp2, rs); - // } catch (SQLException e) { - // e.printStackTrace(); - // } - // } - @Test public void canQueryNotEqualToInDateTypeForSecondCol() { try (Statement stmt = conn.createStatement()) { @@ -441,18 +429,6 @@ public class TimestampPrecisionInNanoInJniTest { } } - // @Test - // public void canQueryNotEqualToInNumberTypeForFirstCol() { - // try (Statement stmt = conn.createStatement()) { - // ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts <> '" + timestamp3 + "'"); - // checkCount(1l, rs); - // rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts <> '" + timestamp3 + "'"); - // checkTime(timestamp2, rs); - // } catch (SQLException e) { - // e.printStackTrace(); - // } - // } - @Test public void canQueryNotEqualToInNumberTypeForSecondCol() { try (Statement stmt = conn.createStatement()) { @@ -465,18 +441,6 @@ public class TimestampPrecisionInNanoInJniTest { } } - // @Test - // public void canQueryNotEqualInDateTypeForFirstCol() { - // try (Statement stmt = conn.createStatement()) { - // ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts != '" + date3 + "'"); - // checkCount(1l, rs); - // rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts != '" + date3 + "'"); - // checkTime(timestamp2, rs); - // } catch (SQLException e) { - // e.printStackTrace(); - // } - // } - @Test public void canQueryNotEqualInDateTypeForSecondCol() { try (Statement stmt = conn.createStatement()) { @@ -489,18 +453,6 @@ public class TimestampPrecisionInNanoInJniTest { } } - // @Test - // public void canQueryNotEqualInNumberTypeForFirstCol() { - // try (Statement stmt = conn.createStatement()) { - // ResultSet rs = stmt.executeQuery("select count(*) from " + ns_timestamp_db + ".weather where ts != '" + timestamp3 + "'"); - // checkCount(1l, rs); - // rs = stmt.executeQuery("select ts from " + ns_timestamp_db + ".weather where ts != '" + timestamp3 + "'"); - // checkTime(timestamp2, rs); - // } catch (SQLException e) { - // e.printStackTrace(); - // } - // } - @Test public void canQueryNotEqualInNumberTypeForSecondCol() { try (Statement stmt = conn.createStatement()) { From a37da4e731a1dc0bf77ddb5abf53a905b7288747 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 21 Jun 2021 10:37:59 +0800 Subject: [PATCH 031/115] [td-4776]:update the error message. --- src/client/src/tscSQLParser.c | 21 +++++++++------------ src/query/src/qAggMain.c | 3 +-- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index cf94aaae35..d5496584e9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2254,10 +2254,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col case TSDB_FUNC_LEASTSQR: { // 1. valid the number of parameters int32_t numOfParams = (pItem->pNode->pParam == NULL)? 0: (int32_t) taosArrayGetSize(pItem->pNode->pParam); + + // no parameters or more than one parameter for function if (pItem->pNode->pParam == NULL || (functionId != TSDB_FUNC_LEASTSQR && functionId != TSDB_FUNC_DERIVATIVE && numOfParams != 1) || ((functionId == TSDB_FUNC_LEASTSQR || functionId == TSDB_FUNC_DERIVATIVE) && numOfParams != 3)) { - /* no parameters or more than one parameter for function */ return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -2271,14 +2272,15 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + STableComInfo info = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + // functions can not be applied to tags + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta))) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } // 2. check if sql function can be applied on this column data type - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - STableComInfo info = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); if (!IS_NUMERIC_TYPE(pSchema->type)) { @@ -2307,11 +2309,6 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); } - // functions can not be applied to tags - if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), intermediateResSize, false); if (functionId == TSDB_FUNC_LEASTSQR) { // set the leastsquares parameters @@ -2340,9 +2337,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } if (info.precision == TSDB_TIME_PRECISION_MILLI) { - tickPerSec /= 1000000; + tickPerSec /= TSDB_TICK_PER_SECOND(TSDB_TIME_PRECISION_MICRO); } else if (info.precision == TSDB_TIME_PRECISION_MICRO) { - tickPerSec /= 1000; + tickPerSec /= TSDB_TICK_PER_SECOND(TSDB_TIME_PRECISION_MILLI); } if (tickPerSec <= 0 || tickPerSec < TSDB_TICK_PER_SECOND(info.precision)) { diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 3dff0f5d50..150c58ed4d 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4166,8 +4166,7 @@ void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { * function compatible list. * tag and ts are not involved in the compatibility check * - * 1. functions that are not simultaneously present with any other functions. e.g., - * diff/ts_z/top/bottom + * 1. functions that are not simultaneously present with any other functions. e.g., diff/ts_z/top/bottom * 2. functions that are only allowed to be present only with same functions. e.g., last_row, interp * 3. functions that are allowed to be present with other functions. * e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last... From 349e14a73fef986e4a872f5ef3da6594b33e8897 Mon Sep 17 00:00:00 2001 From: Yang Zhao Date: Mon, 21 Jun 2021 10:59:45 +0800 Subject: [PATCH 032/115] Update TimestampPrecisionInNanoInJniTest.java --- .../taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java index 68541dbe9d..042d76d576 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TimestampPrecisionInNanoInJniTest.java @@ -71,7 +71,7 @@ public class TimestampPrecisionInNanoInJniTest { private void checkCount(long count, ResultSet rs) throws SQLException { if (count == 0) { - Assert(0); + Assert.fail(); } rs.next(); long test_count = rs.getLong(1); From 5afd77489b504799748fb04061fada9e8fab0beb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 21 Jun 2021 11:28:48 +0800 Subject: [PATCH 033/115] [td-225]fix compiler error. --- src/inc/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index dac2dc84b6..365f24e126 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -100,7 +100,7 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_TIME_PRECISION_MICRO_STR "us" #define TSDB_TIME_PRECISION_NANO_STR "ns" -#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L)) +#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) #define T_APPEND_MEMBER(dst, ptr, type, member) \ From 4239fd33c6e5ee5423fca48f16ae365055b2ae00 Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Mon, 21 Jun 2021 15:41:45 +0800 Subject: [PATCH 034/115] [TD-4746] add test case for irregular locale setting for java --- .../jdbc/cases/BadLocaleSettingTest.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java new file mode 100644 index 0000000000..b351ee94bb --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BadLocaleSettingTest.java @@ -0,0 +1,58 @@ +package com.taosdata.jdbc.cases; + + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import java.io.IOException; + +import java.sql.*; +import java.util.Properties; + +public class BadLocaleSettingTest { + + private static final String host = "127.0.0.1"; + private static final String dbName = "bad_locale_test"; + private static Connection conn; + + @Test + public void canSetLocale() { + try { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url, properties); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + dbName); + stmt.execute("create database if not exists " + dbName); + stmt.execute("use " + dbName); + stmt.execute("drop table if exists weather"); + stmt.execute("create table weather(ts timestamp, temperature float, humidity int)"); + stmt.executeUpdate("insert into weather values(1624071506435, 12.3, 4)"); + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() { + System.setProperty("sun.jnu.encoding", "ANSI_X3.4-1968"); + System.setProperty("file.encoding", "ANSI_X3.4-1968"); + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file From 5a9dbf41d46d140e1c62daa107848021d2c7ebaa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 21 Jun 2021 15:46:15 +0800 Subject: [PATCH 035/115] [td-225]fix compiler error. --- src/query/src/qAggMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 150c58ed4d..5044320cc7 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -3945,7 +3945,7 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { return; } - *(double*) pCtx->pOutput = do_calc_rate(pRateInfo, TSDB_TICK_PER_SECOND(pCtx->param[0].i64)); + *(double*) pCtx->pOutput = do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].i64)); // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; From a2f84ae93cdb908457779e1d4c29fb1db2897874 Mon Sep 17 00:00:00 2001 From: zyyang-taosdata Date: Mon, 21 Jun 2021 16:34:42 +0800 Subject: [PATCH 036/115] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 2 +- src/connector/go | 2 +- src/connector/hivemq-tdengine-extension | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 79eacb2d05..7e556ec225 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.20.5") + SET(TD_VER_NUMBER "2.0.20.8") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 49eadd26d7..39538e39ab 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.20.5' +version: '2.0.20.8' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | diff --git a/src/connector/go b/src/connector/go index 7a26c432f8..b8f76da4a7 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5 +Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4 diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension index b62a26ecc1..ce52010141 160000 --- a/src/connector/hivemq-tdengine-extension +++ b/src/connector/hivemq-tdengine-extension @@ -1 +1 @@ -Subproject commit b62a26ecc164a310104df57691691b237e091c89 +Subproject commit ce5201014136503d34fecbd56494b67b4961056c From 25686cc55b9df51b4606fcd2d94f674f80b1b25a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 21 Jun 2021 17:03:46 +0800 Subject: [PATCH 037/115] [TD-4787]: reconnect arbitrator if link broken --- src/sync/src/syncMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index d6b81cdf36..821d526d5e 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -869,7 +869,7 @@ static void syncRestartPeer(SSyncPeer *pPeer) { sDebug("%s, peer conn is restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { + if (pPeer->nodeId == 0 || ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { sDebug("%s, check peer connection in 1000 ms", pPeer->id); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } From e53055b358aeb441df3adb7369cdd4b72d699929 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 21 Jun 2021 17:43:34 +0800 Subject: [PATCH 038/115] [TD-4807] : fix typos. --- documentation20/cn/08.connector/docs.md | 2 +- documentation20/en/02.getting-started/docs.md | 8 ++++---- documentation20/en/08.connector/docs.md | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index f26928eec7..2d76c866d1 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -805,7 +805,7 @@ C#连接器支持的系统有：Linux 64/Windows x64/Windows x86 * 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 * .NET接口文件TDengineDrivercs.cs和参考程序示例TDengineTest.cs均位于Windows客户端install_directory/examples/C#目录下。 -* 在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。 +* 在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（Dapper）框架驱动。 ### 安装验证 diff --git a/documentation20/en/02.getting-started/docs.md b/documentation20/en/02.getting-started/docs.md index fcfb88a6fe..19af3b5f31 100644 --- a/documentation20/en/02.getting-started/docs.md +++ b/documentation20/en/02.getting-started/docs.md @@ -2,7 +2,7 @@ ## Quick Install -TDegnine software consists of 3 parts: server, client, and alarm module. At the moment, TDengine server only runs on Linux (Windows, mac OS and more OS supports will come soon), but client can run on either Windows or Linux. TDengine client can be installed and run on Windows or Linux. Applications based-on any OSes can all connect to server taosd via a RESTful interface. About CPU, TDegnine supports X64/ARM64/MIPS64/Alpha64, and ARM32、RISC-V, other more CPU architectures will be supported soon. You can set up and install TDengine server either from the [source code](https://www.taosdata.com/en/getting-started/#Install-from-Source) or the [packages](https://www.taosdata.com/en/getting-started/#Install-from-Package). +TDengine software consists of 3 parts: server, client, and alarm module. At the moment, TDengine server only runs on Linux (Windows, mac OS and more OS supports will come soon), but client can run on either Windows or Linux. TDengine client can be installed and run on Windows or Linux. Applications based-on any OSes can all connect to server taosd via a RESTful interface. About CPU, TDengine supports X64/ARM64/MIPS64/Alpha64, and ARM32、RISC-V, other more CPU architectures will be supported soon. You can set up and install TDengine server either from the [source code](https://www.taosdata.com/en/getting-started/#Install-from-Source) or the [packages](https://www.taosdata.com/en/getting-started/#Install-from-Package). ### Install from Source @@ -14,7 +14,7 @@ Please visit our [TDengine Official Docker Image: Distribution, Downloading, and ### Install from Package -It’s extremely easy to install for TDegnine, which takes only a few seconds from downloaded to successful installed. The server installation package includes clients and connectors. We provide 3 installation packages, which you can choose according to actual needs: +It’s extremely easy to install for TDengine, which takes only a few seconds from downloaded to successful installed. The server installation package includes clients and connectors. We provide 3 installation packages, which you can choose according to actual needs: Click [here](https://www.taosdata.com/cn/getting-started/#%E9%80%9A%E8%BF%87%E5%AE%89%E8%A3%85%E5%8C%85%E5%AE%89%E8%A3%85) to download the install package. @@ -39,8 +39,8 @@ If the service is running successfully, you can play around through TDengine she **Note:** - The `systemctl` command needs the **root** privilege. Use **sudo** if you are not the **root** user. -- To get better product feedback and improve our solution, TDegnine will collect basic usage information, but you can modify the configuration parameter **telemetryReporting** in the system configuration file taos.cfg, and set it to 0 to turn it off. -- TDegnine uses FQDN (usually hostname) as the node ID. In order to ensure normal operation, you need to set hostname for the server running taosd, and configure DNS service or hosts file for the machine running client application, to ensure the FQDN can be resolved. +- To get better product feedback and improve our solution, TDengine will collect basic usage information, but you can modify the configuration parameter **telemetryReporting** in the system configuration file taos.cfg, and set it to 0 to turn it off. +- TDengine uses FQDN (usually hostname) as the node ID. In order to ensure normal operation, you need to set hostname for the server running taosd, and configure DNS service or hosts file for the machine running client application, to ensure the FQDN can be resolved. - TDengine supports installation on Linux systems with[ systemd ](https://en.wikipedia.org/wiki/Systemd)as the process service management, and uses `which systemctl` command to detect whether `systemd` packages exist in the system: ```bash diff --git a/documentation20/en/08.connector/docs.md b/documentation20/en/08.connector/docs.md index 36dc06a36e..3da9937f98 100644 --- a/documentation20/en/08.connector/docs.md +++ b/documentation20/en/08.connector/docs.md @@ -6,7 +6,7 @@ TDengine provides many connectors for development, including C/C++, JAVA, Python At present, TDengine connectors support a wide range of platforms, including hardware platforms such as X64/X86/ARM64/ARM32/MIPS/Alpha, and development environments such as Linux/Win64/Win32. The comparison matrix is as follows: -| **CPU** | **X64 64bit** | **X64 64bit** | **X64 64bit** | **X86 32bit** | **ARM64** | **ARM32** | **MIPS Godson** | **Alpha Whenwei** | **X64 TimecomTech** | +| **CPU** | **X64 64bit** | **X64 64bit** | **X64 64bit** | **X86 32bit** | **ARM64** | **ARM32** | **MIPS Godson** | **Alpha Sunway** | **X64 TimecomTech** | | ----------- | ------------- | ------------- | ------------- | ------------- | --------- | --------- | --------------- | ----------------- | ------------------- | | **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** | | **C/C++** | ● | ● | ● | ○ | ● | ● | ○ | ○ | ○ | @@ -75,9 +75,9 @@ Edit the taos.cfg file (default path/etc/taos/taos.cfg) and change firstEP to En * X64 hardware environment: TDengine-client-2.X.X.X-Windows-x64.exe * X86 hardware environment: TDengine-client-2.X.X.X-Windows-x86.exe -**2. Execute installation, select default vales as prompted to complete** +**2. Execute installation, select default values as prompted to complete** -**3. Installatino path** +**3. Installation path** Default installation path is: C:\TDengine, with following files(directories): @@ -327,7 +327,7 @@ typedef struct TAOS_BIND { } TAOS_BIND; ``` -Add the curren bound parameters to the batch. After calling this function, you can call `taos_stmt_bind_param` again to bind the new parameters. It should be noted that this function only supports insert/import statements, and if it is other SQL statements such as select, it will return errors. +Add the current bound parameters to the batch. After calling this function, you can call `taos_stmt_bind_param` again to bind the new parameters. It should be noted that this function only supports insert/import statements, and if it is other SQL statements such as select, it will return errors. - `int taos_stmt_execute(TAOS_STMT *stmt)` @@ -523,7 +523,7 @@ Users can directly view the usage information of the module through Python's hel Refer to help (taos.TDEngineConnection) in python. This class corresponds to a connection between the client and TDengine. In the scenario of client multithreading, it is recommended that each thread apply for an independent connection instance, but not recommended that multiple threads share a connection. -- *TDegnineCursor* class +- *TDengineCursor* class Refer to help (taos.TDengineCursor) in python. This class corresponds to the write and query operations performed by the client. In the scenario of client multithreading, this cursor instance must be kept exclusive to threads and cannot be used by threads, otherwise errors will occur in the returned results. @@ -685,7 +685,7 @@ Return value: } ``` -- Craete a database demo: +- Create a database demo: ```bash curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6041/rest/sql @@ -771,7 +771,7 @@ The C # connector supports: Linux 64/Windows x64/Windows x86. - For application driver installation, please refer to the[ steps of installing connector driver](https://www.taosdata.com/en/documentation/connector#driver). - . NET interface file TDengineDrivercs.cs and reference sample TDengineTest.cs are both located in the Windows client install_directory/examples/C# directory. -- On Windows, C # applications can use the native C interface of TDengine to perform all database operations, and future versions will provide the ORM (dapper) framework driver. +- On Windows, C # applications can use the native C interface of TDengine to perform all database operations, and future versions will provide the ORM (Dapper) framework driver. ### Installation verification @@ -908,7 +908,7 @@ Use Microsoft [windows-build-tools](https://github.com/felixrieseberg/windows-bu #### Solution 2 -Mannually install the following tools: +Manually install the following tools: - Install Visual Studio related tools: [Visual Studio Build Tools](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) or [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community) - Install [Python](https://www.python.org/downloads/) 2.7 (not supported in v3.x.x) and execute npm config set python python2.7 From 1dd0c56d876f3c7928c8ec95cfac3e379b0ad2a6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 21 Jun 2021 17:44:09 +0800 Subject: [PATCH 039/115] [td-225]fix compiler error. --- src/query/src/qAggMain.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 5044320cc7..85b4cb5e8f 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2670,10 +2670,6 @@ static void date_col_output_function(SQLFunctionCtx *pCtx) { *(int64_t *)(pCtx->pOutput) = pCtx->startTs; } -static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_t index) { - date_col_output_function(pCtx); -} - static void col_project_function(SQLFunctionCtx *pCtx) { // the number of output rows should not affect the final number of rows, so set it to be 0 if (pCtx->numOfParams == 2) { From 207481330e0255b2d2a3ae40e0a8197320aed5f5 Mon Sep 17 00:00:00 2001 From: zyyang-taosdata Date: Mon, 21 Jun 2021 17:55:32 +0800 Subject: [PATCH 040/115] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 2 +- src/connector/go | 2 +- src/connector/hivemq-tdengine-extension | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index ed8e7a156c..134f09f179 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.1.1.0") + SET(TD_VER_NUMBER "2.1.3.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index a63225ab32..c7c7846062 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.1.1.0' +version: '2.1.3.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | diff --git a/src/connector/go b/src/connector/go index 7a26c432f8..b8f76da4a7 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5 +Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4 diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension index b62a26ecc1..ce52010141 160000 --- a/src/connector/hivemq-tdengine-extension +++ b/src/connector/hivemq-tdengine-extension @@ -1 +1 @@ -Subproject commit b62a26ecc164a310104df57691691b237e091c89 +Subproject commit ce5201014136503d34fecbd56494b67b4961056c From 1b6c658a2596671872cf87796613f8797d2c9f7d Mon Sep 17 00:00:00 2001 From: zyyang-taosdata Date: Mon, 21 Jun 2021 18:07:55 +0800 Subject: [PATCH 041/115] change snap version --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index c7c7846062..65a8e6b684 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.1.1.0 + - usr/lib/libtaos.so.2.1.3.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From 6a95d859e9dbae356bb4ae18b2cf8b4ab87b6673 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 21 Jun 2021 21:04:55 +0800 Subject: [PATCH 042/115] [TD-4827] --- src/client/src/tscSQLParser.c | 2 +- .../general/parser/select_with_tags.sim | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d78eb65f2e..08c840c1de 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8151,7 +8151,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS int32_t colType = -1; STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; if (pCols != NULL && taosArrayGetSize(pCols) > 0) { - SColIndex* idx = taosArrayGet(pCols, 0); + SColIndex* idx = taosArrayGet(pCols, taosArrayGetSize(pCols) - 1); SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex); if (pSchema != NULL) { colType = pSchema->type; diff --git a/tests/script/general/parser/select_with_tags.sim b/tests/script/general/parser/select_with_tags.sim index 45f6f5c49f..f5c94d2ae6 100644 --- a/tests/script/general/parser/select_with_tags.sim +++ b/tests/script/general/parser/select_with_tags.sim @@ -68,6 +68,27 @@ endw sleep 100 + +#======================= only check first table tag, TD-4827 +sql select count(*) from $mt where t1 in (0) +if $rows != 1 then + return -1 +endi +if $data00 != $rowNum then + return -1; +endi + +$secTag = ' . abc +$secTag = $secTag . 0 +$secTag = $secTag . ' +sql select count(*) from $mt where t2 =$secTag and t1 in (0) +if $rows != 1 then + return -1 +endi +if $data00 != $rowNum then + return -1; +endi + #================================ sql select ts from select_tags_mt0 print $rows From 41c7b062bb43e38e5cb73caca154bc96f07de65d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Jun 2021 08:29:58 +0800 Subject: [PATCH 043/115] [TD-4098] refactor IN filter --- src/client/src/tscSQLParser.c | 101 ++++++++++++---------------------- src/tsdb/src/tsdbRead.c | 2 +- 2 files changed, 36 insertions(+), 67 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d78eb65f2e..4d189d0245 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -65,7 +65,6 @@ static char* getAccountId(SSqlObj* pSql); static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision); static bool serializeExprListToVariant(SArray* pList, tVariant **dest, int16_t colType, uint8_t precision); -static int32_t validateParamOfRelationIn(tVariant *pVar, int32_t colType); static bool has(SArray* pFieldList, int32_t startIdx, const char* name); static char* cloneCurrentDBName(SSqlObj* pSql); @@ -156,78 +155,60 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, return ret; } - tSqlExprItem* item = (tSqlExprItem *)taosArrayGet(pList, 0); - int32_t firstTokenType = item->pNode->token.type; - int32_t type = firstTokenType; + tSqlExpr* item = ((tSqlExprItem*)(taosArrayGet(pList, 0)))->pNode; + int32_t firstVarType = item->value.nType; - //nchar to binary and other xxint to bigint - toTSDBType(type); - if (colType != TSDB_DATA_TYPE_TIMESTAMP && !IS_UNSIGNED_NUMERIC_TYPE(colType)) { - if (type != colType && (type != TSDB_DATA_TYPE_BINARY || colType != TSDB_DATA_TYPE_NCHAR)) { - return false; - } - } - type = colType; - SBufferWriter bw = tbufInitWriter( NULL, false); - tbufEnsureCapacity(&bw, 512); + if (colType == TSDB_DATA_TYPE_TIMESTAMP) { + tbufWriteUint32(&bw, TSDB_DATA_TYPE_BIGINT); + } else { + tbufWriteUint32(&bw, colType); + } + tbufWriteInt32(&bw, (int32_t)(pList->size)); - int32_t size = (int32_t)(pList->size); - tbufWriteUint32(&bw, type); - tbufWriteInt32(&bw, size); - - for (int32_t i = 0; i < size; i++) { + for (int32_t i = 0; i < (int32_t)pList->size; i++) { tSqlExpr* pSub = ((tSqlExprItem*)(taosArrayGet(pList, i)))->pNode; + tVariant* var = &pSub->value; // check all the token type in expr list same or not - if (firstTokenType != pSub->token.type) { + if (firstVarType != var->nType) { break; } - toTSDBType(pSub->token.type); - - tVariant var; - tVariantCreate(&var, &pSub->token); - if (type == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(type)) { - tbufWriteInt64(&bw, var.i64); - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - // ugly code, refactor later - if (IS_UNSIGNED_NUMERIC_TYPE(pSub->token.type) || IS_SIGNED_NUMERIC_TYPE(pSub->token.type)) { - tbufWriteUint64(&bw, var.i64); + if ((colType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(colType))) { + tbufWriteInt64(&bw, var->i64); + } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) { + tbufWriteUint64(&bw, var->u64); + } else if (colType == TSDB_DATA_TYPE_DOUBLE || colType == TSDB_DATA_TYPE_FLOAT) { + if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { + tbufWriteDouble(&bw, (double)(var->i64)); } else { - tVariantDestroy(&var); - break; + tbufWriteDouble(&bw, var->dKey); } - } - else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) { - tbufWriteDouble(&bw, var.dKey); - } else if (type == TSDB_DATA_TYPE_BINARY){ - tbufWriteBinary(&bw, var.pz, var.nLen); - } else if (type == TSDB_DATA_TYPE_NCHAR) { - char *buf = (char *)calloc(1, (var.nLen + 1)*TSDB_NCHAR_SIZE); - if (tVariantDump(&var, buf, type, false) != TSDB_CODE_SUCCESS) { + } else if (colType == TSDB_DATA_TYPE_BINARY) { + tbufWriteBinary(&bw, var->pz, var->nLen); + } else if (colType == TSDB_DATA_TYPE_NCHAR) { + char *buf = (char *)calloc(1, (var->nLen + 1)*TSDB_NCHAR_SIZE); + if (tVariantDump(var, buf, colType, false) != TSDB_CODE_SUCCESS) { free(buf); - tVariantDestroy(&var); break; } tbufWriteBinary(&bw, buf, twcslen((wchar_t *)buf) * TSDB_NCHAR_SIZE); free(buf); - } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { - if (var.nType == TSDB_DATA_TYPE_BINARY) { - if (convertTimestampStrToInt64(&var, precision) < 0) { - tVariantDestroy(&var); + } else if (colType == TSDB_DATA_TYPE_TIMESTAMP) { + if (var->nType == TSDB_DATA_TYPE_BINARY) { + if (convertTimestampStrToInt64(var, precision) < 0) { break; } - tbufWriteInt64(&bw, var.i64); - } else if (var.nType == TSDB_DATA_TYPE_BIGINT) { - tbufWriteInt64(&bw, var.i64); + tbufWriteInt64(&bw, var->i64); + } else if (var->nType == TSDB_DATA_TYPE_BIGINT) { + tbufWriteInt64(&bw, var->i64); } + } else { + break; } - tVariantDestroy(&var); - - if (i == size - 1) { ret = true;} - } - + if (i == (int32_t)(pList->size - 1)) { ret = true;} + } if (ret == true) { if ((*dst = calloc(1, sizeof(tVariant))) != NULL) { tVariantCreateFromBinary(*dst, tbufGetData(&bw, false), tbufTell(&bw), TSDB_DATA_TYPE_BINARY); @@ -239,13 +220,6 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, return ret; } -static int32_t validateParamOfRelationIn(tVariant *pVar, int32_t colType) { - if (pVar->nType != TSDB_DATA_TYPE_BINARY) { - return -1; - } - SBufferReader br = tbufInitReader(pVar->pz, pVar->nLen, false); - return colType == TSDB_DATA_TYPE_NCHAR ? 0 : (tbufReadUint32(&br) == colType ? 0: -1); -} static uint8_t convertOptr(SStrToken *pToken) { switch (pToken->type) { @@ -3366,11 +3340,6 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, if (pRight->tokenId != TK_SET || !serializeExprListToVariant(pRight->pParam, &pVal, colType, timePrecision)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } - if (validateParamOfRelationIn(pVal, colType) != TSDB_CODE_SUCCESS) { - tVariantDestroy(pVal); - free(pVal); - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - } pColumnFilter->pz = (int64_t)calloc(1, pVal->nLen + 1); pColumnFilter->len = pVal->nLen; pColumnFilter->filterstr = 1; @@ -8151,7 +8120,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS int32_t colType = -1; STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; if (pCols != NULL && taosArrayGetSize(pCols) > 0) { - SColIndex* idx = taosArrayGet(pCols, 0); + SColIndex* idx = taosArrayGet(pCols, taosArrayGetSize(pCols) - 1); SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex); if (pSchema != NULL) { colType = pSchema->type; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 92edd4d160..92a0d489b3 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3364,7 +3364,7 @@ static bool tableFilterFp(const void* pNode, void* param) { GET_TYPED_DATA(v, uint64_t, pInfo->sch.type, val); return NULL != taosHashGet((SHashObj *)pInfo->q, (char *)&v, sizeof(v)); } - else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_DOUBLE) { + else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) { double v; GET_TYPED_DATA(v, double, pInfo->sch.type, val); return NULL != taosHashGet((SHashObj *)pInfo->q, (char *)&v, sizeof(v)); From b533fb04d6ef8fcc2c7e3cef2b6bc590a86be838 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Jun 2021 08:48:44 +0800 Subject: [PATCH 044/115] [TD-4098] refactor IN filter --- tests/pytest/insert/in_function.py | 146 ++++++++++++++--------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/tests/pytest/insert/in_function.py b/tests/pytest/insert/in_function.py index d1fbfd702a..79599de7a7 100644 --- a/tests/pytest/insert/in_function.py +++ b/tests/pytest/insert/in_function.py @@ -621,65 +621,65 @@ class TDTestCase: tdLog.info(cmd1) tdSql.execute(cmd1) - cmd2 = 'select * from in_stable_4 where in_float in (\'888\');' - tdLog.info(cmd2) - tdSql.error(cmd2) - try: - tdSql.execute(cmd2) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") + #cmd2 = 'select * from in_stable_4 where in_float in (\'888\');' + #tdLog.info(cmd2) + #tdSql.error(cmd2) + #try: + # tdSql.execute(cmd2) + # tdLog.exit("invalid operation: not supported filter condition") + #except Exception as e: + # tdLog.info(repr(e)) + # tdLog.info("invalid operation: not supported filter condition") - cmd3 = 'select * from in_stable_4 where in_double in (\'66666\');' - tdLog.info(cmd3) - tdSql.error(cmd3) - try: - tdSql.execute(cmd3) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") + #cmd3 = 'select * from in_stable_4 where in_double in (\'66666\');' + #tdLog.info(cmd3) + #tdSql.error(cmd3) + #try: + # tdSql.execute(cmd3) + # tdLog.exit("invalid operation: not supported filter condition") + #except Exception as e: + # tdLog.info(repr(e)) + # tdLog.info("invalid operation: not supported filter condition") - cmd4 = 'select * from in_stable_4 where tin_float in (\'666\');' - tdLog.info(cmd4) - tdSql.error(cmd4) - try: - tdSql.execute(cmd4) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") + #cmd4 = 'select * from in_stable_4 where tin_float in (\'666\');' + #tdLog.info(cmd4) + #tdSql.error(cmd4) + #try: + # tdSql.execute(cmd4) + # tdLog.exit("invalid operation: not supported filter condition") + #except Exception as e: + # tdLog.info(repr(e)) + # tdLog.info("invalid operation: not supported filter condition") - cmd5 = 'select * from in_stable_4 where tin_double in (\'88888\');' - tdLog.info(cmd5) - tdSql.error(cmd5) - try: - tdSql.execute(cmd5) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") + #cmd5 = 'select * from in_stable_4 where tin_double in (\'88888\');' + #tdLog.info(cmd5) + #tdSql.error(cmd5) + #try: + # tdSql.execute(cmd5) + # tdLog.exit("invalid operation: not supported filter condition") + #except Exception as e: + # tdLog.info(repr(e)) + # tdLog.info("invalid operation: not supported filter condition") - cmd6 = 'select * from in_float_double_1 where in_float in (\'888\');' - tdLog.info(cmd6) - tdSql.error(cmd6) - try: - tdSql.execute(cmd6) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") - - cmd7 = 'select * from in_float_double_1 where in_double in (\'66666\');' - tdLog.info(cmd7) - tdSql.error(cmd7) - try: - tdSql.execute(cmd7) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") + #cmd6 = 'select * from in_float_double_1 where in_float in (\'888\');' + #tdLog.info(cmd6) + #tdSql.error(cmd6) + #try: + # tdSql.execute(cmd6) + # tdLog.exit("invalid operation: not supported filter condition") + #except Exception as e: + # tdLog.info(repr(e)) + # tdLog.info("invalid operation: not supported filter condition") + # + #cmd7 = 'select * from in_float_double_1 where in_double in (\'66666\');' + #tdLog.info(cmd7) + #tdSql.error(cmd7) + #try: + # tdSql.execute(cmd7) + # tdLog.exit("invalid operation: not supported filter condition") + #except Exception as e: + # tdLog.info(repr(e)) + # tdLog.info("invalid operation: not supported filter condition") @@ -698,24 +698,24 @@ class TDTestCase: tdSql.execute(cmd1) cmd2 = 'select * from normal_in_float_double_1 where in_float in (\'888\');' - tdLog.info(cmd2) - tdSql.error(cmd2) - try: - tdSql.execute(cmd2) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") - - cmd3 = 'select * from normal_in_float_double_1 where in_double in (\'66666\');' - tdLog.info(cmd3) - tdSql.error(cmd3) - try: - tdSql.execute(cmd3) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") + #tdLog.info(cmd2) + #tdSql.error(cmd2) + #try: + # tdSql.execute(cmd2) + # tdLog.exit("invalid operation: not supported filter condition") + #except Exception as e: + # tdLog.info(repr(e)) + # tdLog.info("invalid operation: not supported filter condition") + # + #cmd3 = 'select * from normal_in_float_double_1 where in_double in (\'66666\');' + #tdLog.info(cmd3) + #tdSql.error(cmd3) + #try: + # tdSql.execute(cmd3) + # tdLog.exit("invalid operation: not supported filter condition") + #except Exception as e: + # tdLog.info(repr(e)) + # tdLog.info("invalid operation: not supported filter condition") def stop(self): tdSql.close() From ad651eff54a4d06ef6d3741e45240c8e7c03438a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Jun 2021 09:29:03 +0800 Subject: [PATCH 045/115] [TD-4098] refactor IN filter --- tests/script/general/parser/where.sim | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index 00a22eede6..2fe63b9892 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -139,18 +139,18 @@ sql_error select * from $mt where c1 like 1 sql create table wh_mt1 (ts timestamp, c1 smallint, c2 int, c3 bigint, c4 float, c5 double, c6 tinyint, c7 binary(10), c8 nchar(10), c9 bool, c10 timestamp) tags (t1 binary(10), t2 smallint, t3 int, t4 bigint, t5 float, t6 double) sql create table wh_mt1_tb1 using wh_mt1 tags ('tb11', 1, 1, 1, 1, 1) sql insert into wh_mt1_tb1 values (now, 1, 1, 1, 1, 1, 1, 'binary', 'nchar', true, '2019-01-01 00:00:00.000') -sql_error select last(*) from wh_mt1 where c1 in ('1') -sql_error select last(*) from wh_mt1_tb1 where c1 in ('1') -sql_error select last(*) from wh_mt1 where c2 in ('1') -sql_error select last(*) from wh_mt1_tb1 where c2 in ('1') -sql_error select last(*) from wh_mt1 where c3 in ('1') -sql_error select last(*) from wh_mt1_tb1 where c3 in ('1') -sql_error select last(*) from wh_mt1 where c4 in ('1') -sql_error select last(*) from wh_mt1_tb1 where c4 in ('1') -sql_error select last(*) from wh_mt1 where c5 in ('1') -sql_error select last(*) from wh_mt1_tb1 where c5 in ('1') -sql_error select last(*) from wh_mt1 where c6 in ('1') -sql_error select last(*) from wh_mt1_tb1 where c6 in ('1') +#sql_error select last(*) from wh_mt1 where c1 in ('1') +#sql_error select last(*) from wh_mt1_tb1 where c1 in ('1') +#sql_error select last(*) from wh_mt1 where c2 in ('1') +#sql_error select last(*) from wh_mt1_tb1 where c2 in ('1') +#sql_error select last(*) from wh_mt1 where c3 in ('1') +#sql_error select last(*) from wh_mt1_tb1 where c3 in ('1') +#sql_error select last(*) from wh_mt1 where c4 in ('1') +#sql_error select last(*) from wh_mt1_tb1 where c4 in ('1') +#sql_error select last(*) from wh_mt1 where c5 in ('1') +#sql_error select last(*) from wh_mt1_tb1 where c5 in ('1') +#sql_error select last(*) from wh_mt1 where c6 in ('1') +#sql_error select last(*) from wh_mt1_tb1 where c6 in ('1') #sql_error select last(*) from wh_mt1 where c7 in ('binary') #sql_error select last(*) from wh_mt1_tb1 where c7 in ('binary') #sql_error select last(*) from wh_mt1 where c8 in ('nchar') From ef8cffc1b835d42b7405c1f46ea444652bcd87bf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Jun 2021 14:26:12 +0800 Subject: [PATCH 046/115] logfile name for tmq --- src/client/src/tscSystem.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index bd79f81846..355f8a403c 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -45,6 +45,7 @@ int32_t tscNumOfObj = 0; // number of sqlObj in current process. static void *tscCheckDiskUsageTmr; void *tscRpcCache; // cache to keep rpc obj int32_t tscNumOfThreads = 1; // num of rpc threads +char tscLogFileName[12] = "taoslog"; static pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently static pthread_once_t tscinit = PTHREAD_ONCE_INIT; static volatile int tscInitRes = 0; @@ -132,7 +133,7 @@ void taos_init_imp(void) { printf("failed to create log dir:%s\n", tsLogDir); } - sprintf(temp, "%s/taoslog", tsLogDir); + sprintf(temp, "%s/%s", tsLogDir, tscLogFileName); if (taosInitLog(temp, tsNumOfLogLines, 10) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } From 2f4da22b11b0cf1e673c6c784d1c6b6ac38c11c9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Jun 2021 14:31:15 +0800 Subject: [PATCH 047/115] minor changes --- src/os/inc/osDef.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index bbe0f98ec0..07fb5c283b 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -40,10 +40,9 @@ extern "C" { #define ASSERT(x) #endif -#ifdef UNUSED -#undefine UNUSED -#endif +#ifndef UNUSED #define UNUSED(x) ((void)(x)) +#endif #ifdef UNUSED_FUNC #undefine UNUSED_FUNC From c6f5a6c4cfb0f13903ce9be29857864766df3ab8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Jun 2021 14:42:00 +0800 Subject: [PATCH 048/115] minor changes --- cmake/env.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/env.inc b/cmake/env.inc index 15cd5e1880..356bd61442 100755 --- a/cmake/env.inc +++ b/cmake/env.inc @@ -35,8 +35,8 @@ ENDIF () # Set compiler options SET(COMMON_C_FLAGS "${COMMON_FLAGS} -std=gnu99") -SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}") -SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}") +SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_C_FLAGS} ${DEBUG_FLAGS}") +SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_C_FLAGS} ${RELEASE_FLAGS}") # Set c++ compiler options SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11") From 25ec36ddd664b76a194153eae5cffab8080313b9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Jun 2021 14:47:52 +0800 Subject: [PATCH 049/115] [TD-4835] change c++ std option --- cmake/define.inc | 14 +++++++------- cmake/env.inc | 11 ++++++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index da80baabbb..86f5ebaf75 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -57,7 +57,7 @@ IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) MESSAGE(STATUS "linux64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () @@ -65,7 +65,7 @@ IF (TD_LINUX_32) ADD_DEFINITIONS(-D_TD_LINUX_32) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "linux32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_64) @@ -73,7 +73,7 @@ IF (TD_ARM_64) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_32) @@ -81,7 +81,7 @@ IF (TD_ARM_32) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") ENDIF () IF (TD_MIPS_64) @@ -89,7 +89,7 @@ IF (TD_MIPS_64) ADD_DEFINITIONS(-D_TD_MIPS_64) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_MIPS_32) @@ -97,7 +97,7 @@ IF (TD_MIPS_32) ADD_DEFINITIONS(-D_TD_MIPS_32) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -fPIC -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_APLHINE) @@ -142,7 +142,7 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "darwin64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") IF (TD_MEMORY_SANITIZER) SET(DEBUG_FLAGS "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -O0 -g3 -DDEBUG") ELSE () diff --git a/cmake/env.inc b/cmake/env.inc index 17ec6a535e..356bd61442 100755 --- a/cmake/env.inc +++ b/cmake/env.inc @@ -34,13 +34,14 @@ ENDIF () # # Set compiler options -SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_FLAGS} ${DEBUG_FLAGS}") -SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FLAGS}") +SET(COMMON_C_FLAGS "${COMMON_FLAGS} -std=gnu99") +SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_C_FLAGS} ${DEBUG_FLAGS}") +SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_C_FLAGS} ${RELEASE_FLAGS}") # Set c++ compiler options -# SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11") -# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}") -# SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") +SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11") +SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}") +SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") IF (${CMAKE_BUILD_TYPE} MATCHES "Debug") SET(CMAKE_BUILD_TYPE "Debug") From 86f935d4b09da6614e03f1cf1f67dd6b2aef2818 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Jun 2021 14:48:21 +0800 Subject: [PATCH 050/115] [td-225] --- src/query/src/qAggMain.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 85b4cb5e8f..8efc4aad4c 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2675,6 +2675,8 @@ static void col_project_function(SQLFunctionCtx *pCtx) { if (pCtx->numOfParams == 2) { return; } + + // only one row is required. if (pCtx->param[0].i64 == 1) { SET_VAL(pCtx, pCtx->size, 1); } else { @@ -2683,7 +2685,8 @@ static void col_project_function(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->pOutput, pData, (size_t) pCtx->size * pCtx->inputBytes); + int32_t numOfRows = (pCtx->param[0].i64 == 1)? 1:pCtx->size; + memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, From 8ae2c04c36739473dbd71a5b41f7ce68ddb35bb3 Mon Sep 17 00:00:00 2001 From: wpan Date: Tue, 22 Jun 2021 14:57:58 +0800 Subject: [PATCH 051/115] fix bug --- src/client/src/tscSub.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index e4f67ba0de..421a6ce93a 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -509,6 +509,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { } pSub->pSql = pSql; pSql->pSubscription = pSub; + pSub->lastSyncTime = 0; // no table list now, force to update it tscDebug("begin table synchronization"); From 09650a42308ae30fa02927fb93661a6eca6f00c3 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 22 Jun 2021 15:48:43 +0800 Subject: [PATCH 052/115] [TD-4836] : emphasize setting firstEp in all dnode in cluster. --- documentation20/cn/10.cluster/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/10.cluster/docs.md b/documentation20/cn/10.cluster/docs.md index 62d709c279..db20ca4edb 100644 --- a/documentation20/cn/10.cluster/docs.md +++ b/documentation20/cn/10.cluster/docs.md @@ -85,7 +85,7 @@ taos> 将后续的数据节点添加到现有集群，具体有以下几步： -1. 按照[《立即开始》](https://www.taosdata.com/cn/documentation/getting-started/)一章的方法在每个物理节点启动taosd； +1. 按照[《立即开始》](https://www.taosdata.com/cn/documentation/getting-started/)一章的方法在每个物理节点启动taosd；（注意：每个物理节点都需要在 taos.cfg 文件中将 firstEP 参数配置为新集群首个节点的 End Point——在本例中是 h1.taos.com:6030） 2. 在第一个数据节点，使用CLI程序taos, 登录进TDengine系统, 执行命令: From 1acccacb242c4941e188619704f9c89b05249d7c Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 22 Jun 2021 15:54:25 +0800 Subject: [PATCH 053/115] [TD-4837] : "update" parameter can not be changed by "ALTER DATABASE". --- documentation20/cn/11.administrator/docs.md | 2 +- documentation20/cn/12.taos-sql/docs.md | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index ae13a36f76..10951ed1fb 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -131,7 +131,7 @@ taosd -C - quorum：多副本环境下指令执行的确认数要求。取值范围：1、2，单位为个，默认值：1。（可通过 alter database 修改） - precision：时间戳精度标识。ms表示毫秒，us表示微秒，默认值：ms。（2.1.2.0 版本之前、2.0.20.7 版本之前在 taos.cfg 文件中不支持此参数。） - cacheLast：是否在内存中缓存子表的最近数据。0：关闭；1：缓存子表最近一行数据；2：缓存子表每一列的最近的非NULL值；3：同时打开缓存最近行和列功能。默认值：0。（可通过 alter database 修改）（从 2.1.2.0 版本开始此参数支持 0～3 的取值范围，在此之前取值只能是 [0, 1]；而 2.0.11.0 之前的版本在 SQL 指令中不支持此参数。）（2.1.2.0 版本之前、2.0.20.7 版本之前在 taos.cfg 文件中不支持此参数。） -- update：是否允许更新。0：不允许；1：允许。默认值：0。（可通过 alter database 修改） +- update：是否允许更新。0：不允许；1：允许。默认值：0。 对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL： diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index b0268a9ed4..278757b81b 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -139,11 +139,6 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传 ``` FSYNC 参数控制执行 fsync 操作的周期。缺省值为 3000，单位是毫秒，取值范围为 [0, 180000]。如果设置为 0，表示每次写入，立即执行 fsync。该设置项主要用于调节 WAL 参数设为 2 时的系统行为。 - ```mysql - ALTER DATABASE db_name UPDATE 0; - ``` - UPDATE 参数控制是否允许更新数据。缺省值为 0，取值范围为 [0, 1]。0 表示会直接丢弃后写入的相同时间戳的数据；1 表示会使用后写入的数据覆盖已有的相同时间戳的数据。 - **Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。另外，从 2.1.3.0 版本开始，修改这些参数后无需重启服务器即可生效。 - **显示系统所有数据库** From 565cc797d25361d04471b68dd4bae260e12c0151 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Jun 2021 16:12:38 +0800 Subject: [PATCH 054/115] [td-225]upate the log. --- src/client/src/tscPrepare.c | 27 +++------------------------ src/client/src/tscUtil.c | 4 ++-- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index bac8920d8f..8bb776ffee 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1156,27 +1156,6 @@ static void insertBatchClean(STscStmt* pStmt) { tfree(pCmd->insertParam.pTableNameList); -/* - STableDataBlocks** p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, NULL); - - STableDataBlocks* pOneTableBlock = *p; - - while (1) { - SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - - pOneTableBlock->size = sizeof(SSubmitBlk); - - pBlocks->numOfRows = 0; - - p = taosHashIterate(pCmd->insertParam.pTableBlockHashList, p); - if (p == NULL) { - break; - } - - pOneTableBlock = *p; - } -*/ - pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); pCmd->insertParam.numOfTables = 0; @@ -1499,7 +1478,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pRes->numOfRows = 1; strtolower(pSql->sqlstr, sql); - tscDebugL("%p SQL: %s", pSql, pSql->sqlstr); + tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); if (tscIsInsertData(pSql->sqlstr)) { pStmt->isInsert = true; @@ -1604,7 +1583,7 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) > 0) { SHashObj* hashList = pCmd->insertParam.pTableBlockHashList; pCmd->insertParam.pTableBlockHashList = NULL; - tscResetSqlCmd(pCmd, true); + tscResetSqlCmd(pCmd, false); pCmd->insertParam.pTableBlockHashList = hashList; } @@ -1663,7 +1642,7 @@ int taos_stmt_close(TAOS_STMT* stmt) { } else { if (pStmt->multiTbInsert) { taosHashCleanup(pStmt->mtb.pTableHash); - pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, true); + pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, false); taosHashCleanup(pStmt->pSql->cmd.insertParam.pTableBlockHashList); pStmt->pSql->cmd.insertParam.pTableBlockHashList = NULL; taosArrayDestroy(pStmt->mtb.tags); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 83ec886e19..74dbe42eeb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1332,7 +1332,7 @@ void tscFreeSubobj(SSqlObj* pSql) { tscDebug("0x%"PRIx64" start to free sub SqlObj, numOfSub:%d", pSql->self, pSql->subState.numOfSub); for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - tscDebug("0x%"PRIx64" free sub SqlObj:%p, index:%d", pSql->self, pSql->pSubs[i], i); + tscDebug("0x%"PRIx64" free sub SqlObj:0x%"PRIx64", index:%d", pSql->self, pSql->pSubs[i]->self, i); taos_free_result(pSql->pSubs[i]); pSql->pSubs[i] = NULL; } @@ -1784,7 +1784,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl tscSortRemoveDataBlockDupRows(pOneTableBlock); char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); - tscDebug("0x%"PRIx64" name:%s, name:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), + tscDebug("0x%"PRIx64" name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); From 612b6c84521e3fc0c762d7446657e1e3cfed7496 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 22 Jun 2021 16:33:00 +0800 Subject: [PATCH 055/115] [TD-4805]: fix startup os config display --- src/dnode/src/dnodeMain.c | 2 ++ src/os/inc/osSysinfo.h | 1 + src/os/src/darwin/darwinSysInfo.c | 11 ++++++++--- src/os/src/detail/osSysinfo.c | 11 ++++++++--- src/os/src/windows/wSysinfo.c | 7 ++++++- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 410e6bb188..22ce6c995a 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -253,6 +253,8 @@ static int32_t dnodeInitStorage() { dnodeCheckDataDirOpenned(tsDnodeDir); + taosGetDisk(); + taosPrintDiskInfo(); dInfo("dnode storage is initialized at %s", tsDnodeDir); return 0; } diff --git a/src/os/inc/osSysinfo.h b/src/os/inc/osSysinfo.h index 895b5dd499..1be2e94dc5 100644 --- a/src/os/inc/osSysinfo.h +++ b/src/os/inc/osSysinfo.h @@ -36,6 +36,7 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ; bool taosGetProcMemory(float *memoryUsedMB) ; bool taosGetSysMemory(float *memoryUsedMB); void taosPrintOsInfo(); +void taosPrintDiskInfo(); int taosSystem(const char * cmd) ; void taosKillSystem(); bool taosGetSystemUid(char *uid); diff --git a/src/os/src/darwin/darwinSysInfo.c b/src/os/src/darwin/darwinSysInfo.c index 6e70043779..3c80ee4260 100644 --- a/src/os/src/darwin/darwinSysInfo.c +++ b/src/os/src/darwin/darwinSysInfo.c @@ -137,9 +137,6 @@ void taosPrintOsInfo() { // uInfo(" os openMax: %" PRId64, tsOpenMax); // uInfo(" os streamMax: %" PRId64, tsStreamMax); uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); - uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); - uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); struct utsname buf; @@ -155,6 +152,14 @@ void taosPrintOsInfo() { uInfo("=================================="); } +void taosPrintDiskInfo() { + uInfo("=================================="); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); + uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); + uInfo("=================================="); +} + void taosKillSystem() { uError("function taosKillSystem, exit!"); exit(0); diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index bbda08aa25..a2a560b296 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -506,9 +506,6 @@ void taosPrintOsInfo() { uInfo(" os openMax: %" PRId64, tsOpenMax); uInfo(" os streamMax: %" PRId64, tsStreamMax); uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); - uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); - uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); struct utsname buf; @@ -523,6 +520,14 @@ void taosPrintOsInfo() { uInfo(" os machine: %s", buf.machine); } +void taosPrintDiskInfo() { + uInfo("=================================="); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); + uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); + uInfo("=================================="); +} + void taosKillSystem() { // SIGINT uInfo("taosd will shut down soon"); diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 8a81e3079a..72793a1049 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -205,10 +205,15 @@ void taosGetSystemInfo() { void taosPrintOsInfo() { uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + uInfo("=================================="); +} + +void taosPrintDiskInfo() { + uInfo("=================================="); uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); - uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); uInfo("=================================="); } From c5c3ee4c3769511c9d99090f8119172529690582 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Jun 2021 16:56:30 +0800 Subject: [PATCH 056/115] [td-4802]: fix crash caused by too many vgroups; --- src/mnode/src/mnodeTable.c | 52 ++++++++++++++++++++++++++++++-------- src/query/src/qExecutor.c | 20 ++++++++++----- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 032c6ee94b..ae37f74fa3 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1740,16 +1740,22 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { return TSDB_CODE_SUCCESS; } -static int32_t calculateVgroupMsgLength(SSTableVgroupMsg* pInfo, int32_t numOfTable) { +static int32_t doGetVgroupInfoLength(char* name) { + SSTableObj *pTable = mnodeGetSuperTable(name); + int32_t len = 0; + if (pTable != NULL && pTable->vgHash != NULL) { + len = (taosHashGetSize(pTable->vgHash) * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg)); + } + + mnodeDecTableRef(pTable); + return len; +} + +static int32_t getVgroupInfoLength(SSTableVgroupMsg* pInfo, int32_t numOfTable) { int32_t contLen = sizeof(SSTableVgroupRspMsg) + 32 * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg); for (int32_t i = 0; i < numOfTable; ++i) { char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; - SSTableObj *pTable = mnodeGetSuperTable(stableName); - if (pTable != NULL && pTable->vgHash != NULL) { - contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg)); - } - - mnodeDecTableRef(pTable); + contLen += doGetVgroupInfoLength(stableName); } return contLen; @@ -1820,7 +1826,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { int32_t numOfTable = htonl(pInfo->numOfTables); // calculate the required space. - int32_t contLen = calculateVgroupMsgLength(pInfo, numOfTable); + int32_t contLen = getVgroupInfoLength(pInfo, numOfTable); SSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -2860,6 +2866,27 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { } } +static SMultiTableMeta* ensureMsgBufferSpace(SMultiTableMeta *pMultiMeta, SArray* pList, int32_t* totalMallocLen, int32_t numOfVgroupList) { + int32_t len = 0; + for (int32_t i = 0; i < numOfVgroupList; ++i) { + char *name = taosArrayGetP(pList, i); + len += doGetVgroupInfoLength(name); + } + + if (len + pMultiMeta->contLen > (*totalMallocLen)) { + while (len + pMultiMeta->contLen > (*totalMallocLen)) { + (*totalMallocLen) *= 2; + } + + pMultiMeta = rpcReallocCont(pMultiMeta, *totalMallocLen); + if (pMultiMeta == NULL) { + return NULL; + } + } + + return pMultiMeta; +} + static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { SMultiTableInfoMsg *pInfo = pMsg->rpcMsg.pCont; @@ -2950,8 +2977,6 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { } } - char* msg = (char*) pMultiMeta + pMultiMeta->contLen; - // add the additional super table names that needs the vgroup info for(;t < num; ++t) { taosArrayPush(pList, &nameList[t]); @@ -2961,6 +2986,13 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pList); pMultiMeta->numOfVgroup = htonl(numOfVgroupList); + pMultiMeta = ensureMsgBufferSpace(pMultiMeta, pList, &totalMallocLen, numOfVgroupList); + if (pMultiMeta == NULL) { + code = TSDB_CODE_MND_OUT_OF_MEMORY; + goto _end; + } + + char* msg = (char*) pMultiMeta + pMultiMeta->contLen; for(int32_t i = 0; i < numOfVgroupList; ++i) { char* name = taosArrayGetP(pList, i); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3145a7fc3d..719c1de252 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1305,9 +1305,16 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn continue; } - if (memcmp(pInfo->prevData, val, bytes) == 0) { - num++; - continue; + if (IS_VAR_DATA_TYPE(type)) { + if(varDataLen(val) == varDataLen(pInfo->prevData) && memcmp(pInfo->prevData, val, varDataLen(val)) == 0) { + num++; + continue; + } + } else { + if (memcmp(pInfo->prevData, val, bytes)) { + num++; + continue; + } } if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { @@ -1416,9 +1423,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInf } static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { - int64_t v = -1; - GET_TYPED_DATA(v, int64_t, type, pData); - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(type)) { if (pResultRow->key == NULL) { pResultRow->key = malloc(varDataTLen(pData)); varDataCopy(pResultRow->key, pData); @@ -1426,6 +1431,9 @@ static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); } } else { + int64_t v = -1; + GET_TYPED_DATA(v, int64_t, type, pData); + pResultRow->win.skey = v; pResultRow->win.ekey = v; } From 837c44ebfb557260d99cc7b667eefb858a70bfe6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Jun 2021 17:28:26 +0800 Subject: [PATCH 057/115] [td-225]fix typo --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 719c1de252..b5960b9e58 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1311,7 +1311,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn continue; } } else { - if (memcmp(pInfo->prevData, val, bytes)) { + if (memcmp(pInfo->prevData, val, bytes) == 0) { num++; continue; } From 106bbc7e63cc047ca81e0699c7bb482312f45615 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 22 Jun 2021 18:42:31 +0800 Subject: [PATCH 058/115] [TD-4314]: add testcase of nested query with top/bottom function --- tests/pytest/fulltest.sh | 2 ++ tests/pytest/query/nestedQuery/queryInterval.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index c9f68972f9..9373553cba 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -235,6 +235,8 @@ python3 ./test.py -f query/queryTscomputWithNow.py python3 ./test.py -f query/computeErrorinWhere.py python3 ./test.py -f query/queryTsisNull.py python3 ./test.py -f query/subqueryFilter.py +python3 ./test.py -f query/nestedQuery/queryInterval.py +python3 ./test.py -f query/queryStateWindow.py #stream diff --git a/tests/pytest/query/nestedQuery/queryInterval.py b/tests/pytest/query/nestedQuery/queryInterval.py index 3ddf751b7f..11c42c463e 100644 --- a/tests/pytest/query/nestedQuery/queryInterval.py +++ b/tests/pytest/query/nestedQuery/queryInterval.py @@ -83,14 +83,16 @@ class TDTestCase: tdSql.checkData(29, 0, "2020-07-01 05:10:00.000") # subquery and parent query with top and bottom - tdSql.query("select top(avg_val,2) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num);") - tdSql.checkData(0, 1, 115) - tdSql.query("select bottom(avg_val,3) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num);") - tdSql.checkData(0, 1, 125) + tdSql.query("select top(avg_val,2) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num) order by avg_val desc;") + tdSql.checkData(0, 1, 117) + tdSql.query("select bottom(avg_val,3) from(select avg(value) as avg_val,num from st where loc!='beijing0' group by num) order by avg_val asc;") + tdSql.checkData(0, 1, 111) # tdSql.query("select top(avg_val,2) from(select avg(value) as avg_val from st where loc='beijing1' interval(8m) sliding(3m));") + tdSql.checkData(0, 1, 120) + # clear env testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") os.system("rm -rf wal/%s.sql" % testcaseFilename ) From f1de963dc54cf9f961c57fb20e49914e325f497b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 22 Jun 2021 18:49:20 +0800 Subject: [PATCH 059/115] [TD-4823]: taosdemo getTableName return empty. (#6570) * [TD-4823]: taosdemo getTableName return empty. * fix typo. --- src/kit/taosdemo/taosdemo.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4202a833ea..6b12e66cb9 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6000,6 +6000,12 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n", __func__, __LINE__, pThreadInfo->threadID, tableSeq, tableName); + if (0 == strlen(tableName)) { + errorPrint("[%d] %s() LN%d, getTableName return null\n", + pThreadInfo->threadID, __func__, __LINE__); + free(pThreadInfo->buffer); + return NULL; + } int64_t remainderBufLen = maxSqlLen; char *pstr = pThreadInfo->buffer; From 20abb07e3f3da32fe9ad6954f4ff50c68e8d694b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Jun 2021 20:40:09 +0800 Subject: [PATCH 060/115] fix compile errors --- src/client/tests/timeParseTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp index ba06a6b9aa..3cc6d541e0 100644 --- a/src/client/tests/timeParseTest.cpp +++ b/src/client/tests/timeParseTest.cpp @@ -98,7 +98,7 @@ TEST(testCase, parse_time) { taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 852048000999); - int64_t k = timezone; + // int64_t k = timezone; char t42[] = "1997-1-1T0:0:0.999999999Z"; taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND); @@ -163,7 +163,7 @@ TEST(testCase, parse_time) { taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0); EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND); - char* t = "2021-01-08T02:11:40.000+00:00"; + char t[] = "2021-01-08T02:11:40.000+00:00"; taosParseTime(t, &time, strlen(t), TSDB_TIME_PRECISION_MILLI, 0); printf("%ld\n", time); } From 2763345e778c8807879c9a368bd81f3a8e832463 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 22 Jun 2021 21:48:01 +0800 Subject: [PATCH 061/115] [TD-4838]: taosdump binary length bug. (#6580) --- src/kit/taosdump/taosdump.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index f80ac069a0..8525496b0e 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -1858,13 +1858,13 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); //pstr = stpcpy(pstr, tbuf); //*(pstr++) = '\''; - pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; } case TSDB_DATA_TYPE_NCHAR: { char tbuf[COMMAND_SIZE] = {0}; convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); - pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; } case TSDB_DATA_TYPE_TIMESTAMP: @@ -1897,7 +1897,8 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* total_sqlstr_len += curr_sqlstr_len; - if ((count >= arguments->data_batch) || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { + if ((count >= arguments->data_batch) + || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { fprintf(fp, ";\n"); count = 0; } //else { @@ -1905,6 +1906,8 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* //} } + printf("total_sqlstr_len: %d\n", total_sqlstr_len); + fprintf(fp, "\n"); atomic_add_fetch_64(&totalDumpOutRows, totalRows); From aeaee423b6fefd6e321dfb071f9b0d483946f744 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 22 Jun 2021 21:48:39 +0800 Subject: [PATCH 062/115] [TD-4823]: taosdemo getTableName return empty. (#6571) * [TD-4823]: taosdemo getTableName return empty. * fix typo. --- src/kit/taosdemo/taosdemo.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 865224d2c3..33ee2a9bc2 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6004,6 +6004,12 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n", __func__, __LINE__, pThreadInfo->threadID, tableSeq, tableName); + if (0 == strlen(tableName)) { + errorPrint("[%d] %s() LN%d, getTableName return null\n", + pThreadInfo->threadID, __func__, __LINE__); + free(pThreadInfo->buffer); + return NULL; + } int64_t remainderBufLen = maxSqlLen; char *pstr = pThreadInfo->buffer; From 6766c3944851b6a0fd519a803d9e0ca4e444c492 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 22 Jun 2021 21:58:36 +0800 Subject: [PATCH 063/115] [TD-4838]: taosdump binary length bug. (#6581) --- src/kit/taosdump/taosdump.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 15db83297c..165bbdf990 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -1858,13 +1858,13 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); //pstr = stpcpy(pstr, tbuf); //*(pstr++) = '\''; - pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; } case TSDB_DATA_TYPE_NCHAR: { char tbuf[COMMAND_SIZE] = {0}; convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); - pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; } case TSDB_DATA_TYPE_TIMESTAMP: @@ -1897,7 +1897,8 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* total_sqlstr_len += curr_sqlstr_len; - if ((count >= arguments->data_batch) || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { + if ((count >= arguments->data_batch) + || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { fprintf(fp, ";\n"); count = 0; } //else { @@ -1905,6 +1906,8 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* //} } + printf("total_sqlstr_len: %d\n", total_sqlstr_len); + fprintf(fp, "\n"); atomic_add_fetch_64(&totalDumpOutRows, totalRows); From d76117d8304d30b16772e0b1a1341a2b22f22b4d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Jun 2021 23:47:35 +0800 Subject: [PATCH 064/115] [td-225]fix bug by regression test. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b5960b9e58..9b356d57b8 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1306,7 +1306,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } if (IS_VAR_DATA_TYPE(type)) { - if(varDataLen(val) == varDataLen(pInfo->prevData) && memcmp(pInfo->prevData, val, varDataLen(val)) == 0) { + if(varDataLen(val) == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), varDataLen(val)) == 0) { num++; continue; } From 6f2f18f0aac7d818d2bb89ff472520d2b52d1a07 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Jun 2021 09:57:57 +0800 Subject: [PATCH 065/115] remove compile options in test script --- src/query/tests/astTest.cpp | 1 + src/query/tests/histogramTest.cpp | 2 ++ src/query/tests/patternMatchTest.cpp | 2 ++ src/query/tests/percentileTest.cpp | 2 ++ src/query/tests/resultBufferTest.cpp | 2 ++ src/query/tests/tsBufTest.cpp | 2 ++ src/query/tests/unitTest.cpp | 2 +- 7 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index ce7b2f94a1..7bd1c0bf8e 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -10,6 +10,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Werror=unused-function" typedef struct ResultObj { int32_t numOfResult; diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp index 3088d6f807..44a31f4241 100644 --- a/src/query/tests/histogramTest.cpp +++ b/src/query/tests/histogramTest.cpp @@ -5,6 +5,8 @@ #include "taos.h" #include "qHistogram.h" + +#pragma GCC diagnostic ignored "-Werror=unused-function" namespace { void doHistogramAddTest() { SHistogramInfo* pHisto = NULL; diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index f3e0d3e119..cd242afc84 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -6,6 +6,8 @@ #include "qAggMain.h" #include "tcompare.h" +#pragma GCC diagnostic ignored "-Werror=unused-function" + TEST(testCase, patternMatchTest) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; diff --git a/src/query/tests/percentileTest.cpp b/src/query/tests/percentileTest.cpp index 104bfb3c06..952129c8e7 100644 --- a/src/query/tests/percentileTest.cpp +++ b/src/query/tests/percentileTest.cpp @@ -7,6 +7,8 @@ #include "qPercentile.h" +#pragma GCC diagnostic ignored "-Werror=unused-function" + namespace { tMemBucket *createBigIntDataBucket(int32_t start, int32_t end) { tMemBucket *pBucket = tMemBucketCreate(sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, start, end); diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 491d75ccb9..052e654066 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -6,6 +6,8 @@ #include "taos.h" #include "tsdb.h" +#pragma GCC diagnostic ignored "-Werror=unused-function" + namespace { // simple test void simpleTest() { diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index dd7f03a494..30faed1bc2 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -9,6 +9,8 @@ #include "ttoken.h" #include "tutil.h" +#pragma GCC diagnostic ignored "-Werror=unused-function" + namespace { /** * diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index d2b058cf7c..75153a76ae 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -13,7 +13,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" - +#pragma GCC diagnostic ignored "-Werror=unused-function" namespace { int32_t testValidateName(char* name) { SStrToken token = {0}; From ed13f4e541b0a21c814dd697b20aeff2cb158837 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 23 Jun 2021 10:48:58 +0800 Subject: [PATCH 066/115] [td-225] refactor. --- src/query/src/qExecutor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9b356d57b8..97a6cf807c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1306,7 +1306,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } if (IS_VAR_DATA_TYPE(type)) { - if(varDataLen(val) == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), varDataLen(val)) == 0) { + int32_t len = varDataLen(val); + if(len == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), len) == 0) { num++; continue; } From 6bc71839b14e1d8abeb9f3606a7dcc404a916d58 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Jun 2021 11:17:16 +0800 Subject: [PATCH 067/115] [TD-4098] refactor IN filter --- src/client/src/tscSQLParser.c | 18 ++++- tests/pytest/insert/in_function.py | 110 ++++++++++++++--------------- 2 files changed, 72 insertions(+), 56 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4d189d0245..4ce2356f8b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -176,18 +176,32 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, break; } if ((colType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(colType))) { + if (var->nType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(var->nType)) { + break; + } tbufWriteInt64(&bw, var->i64); } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) { + if (IS_SIGNED_NUMERIC_TYPE(var->nType) && IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { + break; + } tbufWriteUint64(&bw, var->u64); } else if (colType == TSDB_DATA_TYPE_DOUBLE || colType == TSDB_DATA_TYPE_FLOAT) { if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { tbufWriteDouble(&bw, (double)(var->i64)); - } else { + } else if (var->nType == TSDB_DATA_TYPE_DOUBLE || var->nType == TSDB_DATA_TYPE_FLOAT){ tbufWriteDouble(&bw, var->dKey); + } else { + break; } } else if (colType == TSDB_DATA_TYPE_BINARY) { + if (var->nType != TSDB_DATA_TYPE_BINARY) { + break; + } tbufWriteBinary(&bw, var->pz, var->nLen); } else if (colType == TSDB_DATA_TYPE_NCHAR) { + if (var->nType != TSDB_DATA_TYPE_BINARY) { + break; + } char *buf = (char *)calloc(1, (var->nLen + 1)*TSDB_NCHAR_SIZE); if (tVariantDump(var, buf, colType, false) != TSDB_CODE_SUCCESS) { free(buf); @@ -203,6 +217,8 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, tbufWriteInt64(&bw, var->i64); } else if (var->nType == TSDB_DATA_TYPE_BIGINT) { tbufWriteInt64(&bw, var->i64); + } else { + break; } } else { break; diff --git a/tests/pytest/insert/in_function.py b/tests/pytest/insert/in_function.py index 79599de7a7..263c8a78aa 100644 --- a/tests/pytest/insert/in_function.py +++ b/tests/pytest/insert/in_function.py @@ -621,65 +621,65 @@ class TDTestCase: tdLog.info(cmd1) tdSql.execute(cmd1) - #cmd2 = 'select * from in_stable_4 where in_float in (\'888\');' - #tdLog.info(cmd2) - #tdSql.error(cmd2) - #try: - # tdSql.execute(cmd2) - # tdLog.exit("invalid operation: not supported filter condition") - #except Exception as e: - # tdLog.info(repr(e)) - # tdLog.info("invalid operation: not supported filter condition") + cmd2 = 'select * from in_stable_4 where in_float in (\'888\');' + tdLog.info(cmd2) + tdSql.error(cmd2) + try: + tdSql.execute(cmd2) + tdLog.exit("invalid operation: not supported filter condition") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("invalid operation: not supported filter condition") - #cmd3 = 'select * from in_stable_4 where in_double in (\'66666\');' - #tdLog.info(cmd3) - #tdSql.error(cmd3) - #try: - # tdSql.execute(cmd3) - # tdLog.exit("invalid operation: not supported filter condition") - #except Exception as e: - # tdLog.info(repr(e)) - # tdLog.info("invalid operation: not supported filter condition") + cmd3 = 'select * from in_stable_4 where in_double in (\'66666\');' + tdLog.info(cmd3) + tdSql.error(cmd3) + try: + tdSql.execute(cmd3) + tdLog.exit("invalid operation: not supported filter condition") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("invalid operation: not supported filter condition") - #cmd4 = 'select * from in_stable_4 where tin_float in (\'666\');' - #tdLog.info(cmd4) - #tdSql.error(cmd4) - #try: - # tdSql.execute(cmd4) - # tdLog.exit("invalid operation: not supported filter condition") - #except Exception as e: - # tdLog.info(repr(e)) - # tdLog.info("invalid operation: not supported filter condition") + cmd4 = 'select * from in_stable_4 where tin_float in (\'666\');' + tdLog.info(cmd4) + tdSql.error(cmd4) + try: + tdSql.execute(cmd4) + tdLog.exit("invalid operation: not supported filter condition") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("invalid operation: not supported filter condition") - #cmd5 = 'select * from in_stable_4 where tin_double in (\'88888\');' - #tdLog.info(cmd5) - #tdSql.error(cmd5) - #try: - # tdSql.execute(cmd5) - # tdLog.exit("invalid operation: not supported filter condition") - #except Exception as e: - # tdLog.info(repr(e)) - # tdLog.info("invalid operation: not supported filter condition") + cmd5 = 'select * from in_stable_4 where tin_double in (\'88888\');' + tdLog.info(cmd5) + tdSql.error(cmd5) + try: + tdSql.execute(cmd5) + tdLog.exit("invalid operation: not supported filter condition") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("invalid operation: not supported filter condition") - #cmd6 = 'select * from in_float_double_1 where in_float in (\'888\');' - #tdLog.info(cmd6) - #tdSql.error(cmd6) - #try: - # tdSql.execute(cmd6) - # tdLog.exit("invalid operation: not supported filter condition") - #except Exception as e: - # tdLog.info(repr(e)) - # tdLog.info("invalid operation: not supported filter condition") - # - #cmd7 = 'select * from in_float_double_1 where in_double in (\'66666\');' - #tdLog.info(cmd7) - #tdSql.error(cmd7) - #try: - # tdSql.execute(cmd7) - # tdLog.exit("invalid operation: not supported filter condition") - #except Exception as e: - # tdLog.info(repr(e)) - # tdLog.info("invalid operation: not supported filter condition") + cmd6 = 'select * from in_float_double_1 where in_float in (\'888\');' + tdLog.info(cmd6) + tdSql.error(cmd6) + try: + tdSql.execute(cmd6) + tdLog.exit("invalid operation: not supported filter condition") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("invalid operation: not supported filter condition") + + cmd7 = 'select * from in_float_double_1 where in_double in (\'66666\');' + tdLog.info(cmd7) + tdSql.error(cmd7) + try: + tdSql.execute(cmd7) + tdLog.exit("invalid operation: not supported filter condition") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("invalid operation: not supported filter condition") From 48a5f68b71ef2571d590fb98aa2bd92936d41930 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 23 Jun 2021 14:36:12 +0800 Subject: [PATCH 068/115] [TD-2569] : describe SESSION & STATE window. --- documentation20/cn/00.index/docs.md | 2 +- documentation20/cn/12.taos-sql/docs.md | 53 +++++++++++++++----------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/documentation20/cn/00.index/docs.md b/documentation20/cn/00.index/docs.md index 4c37ce598c..c900cd373d 100644 --- a/documentation20/cn/00.index/docs.md +++ b/documentation20/cn/00.index/docs.md @@ -42,7 +42,7 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 * [数据写入](/taos-sql#insert)：支持单表单条、多条、多表多条写入，支持历史数据写入 * [数据查询](/taos-sql#select)：支持时间段、值过滤、排序、查询结果手动分页等 * [SQL函数](/taos-sql#functions)：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等 -* [时间维度聚合](/taos-sql#aggregation)：将表中数据按照时间段进行切割后聚合，降维处理 +* [窗口切分聚合](/taos-sql#aggregation)：将表中数据按照时间段等方式进行切割后聚合，降维处理 * [边界限制](/taos-sql#limitation)：库、表、SQL等边界限制条件 * [错误码](/taos-sql/error-code)：TDengine 2.0 错误码以及对应的十进制码 diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 278757b81b..5904abbbaa 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -476,9 +476,10 @@ Query OK, 1 row(s) in set (0.001091s) SELECT select_expr [, select_expr ...] FROM {tb_name_list} [WHERE where_condition] - [INTERVAL (interval_val [, interval_offset])] - [SLIDING sliding_val] - [FILL fill_val] + [SESSION(ts_col, tol_val)] + [STATE_WINDOW(col)] + [INTERVAL(interval_val [, interval_offset]) [SLIDING sliding_val]] + [FILL(fill_mod_and_val)] [GROUP BY col_list] [ORDER BY col_list { DESC | ASC }] [SLIMIT limit_val [SOFFSET offset_val]] @@ -1284,39 +1285,45 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 3 row(s) in set (0.001046s) ``` -## 时间维度聚合 +## 按窗口切分聚合 -TDengine支持按时间段进行聚合，可以将表中数据按照时间段进行切割后聚合生成结果，比如温度传感器每秒采集一次数据，但需查询每隔10分钟的温度平均值。这个聚合适合于降维(down sample)操作, 语法如下： +TDengine 支持按时间段等窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这类聚合适合于降维（down sample）操作，语法如下： ```mysql SELECT function_list FROM tb_name [WHERE where_condition] - INTERVAL (interval [, offset]) - [SLIDING sliding] - [FILL ({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] + [SESSION(ts_col, tol_val)] + [STATE_WINDOW(col)] + [INTERVAL(interval [, offset]) [SLIDING sliding]] + [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] SELECT function_list FROM stb_name [WHERE where_condition] - INTERVAL (interval [, offset]) - [SLIDING sliding] - [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})] + [SESSION(ts_col, tol_val)] + [STATE_WINDOW(col)] + [INTERVAL(interval [, offset]) [SLIDING sliding]] + [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] [GROUP BY tags] ``` -- 聚合时间段的长度由关键词INTERVAL指定，最短时间间隔10毫秒（10a），并且支持偏移（偏移必须小于间隔）。聚合查询中，能够同时执行的聚合和选择函数仅限于单个输出的函数：count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last，不能使用具有多行输出结果的函数（例如：top、bottom、diff以及四则运算）。 -- WHERE语句可以指定查询的起止时间和其他过滤条件。 -- SLIDING语句用于指定聚合时间段的前向增量。 -- FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种： - 1. 不进行填充：NONE(默认填充模式)。 - 2. VALUE填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。 - 3. NULL填充：使用NULL填充数据。例如：FILL(NULL)。 - 4. PREV填充：使用前一个非NULL值填充数据。例如：FILL(PREV)。 - 5. NEXT填充：使用下一个非NULL值填充数据。例如：FILL(NEXT)。 +- 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：TOP、BOTTOM、DIFF 以及四则运算）。 +- 查询过滤、聚合等操作按照每个切分窗口为独立的单位执行。聚合查询目前支持三种窗口的划分方式： + 1. 时间窗口：聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。当 SLIDING 与 INTERVAL 取值相等的时候，滑动窗口即为翻转窗口。 + 2. 状态窗口：使用整数（布尔值）或字符串来标识产生记录时设备的状态量，产生的记录如果具有相同的状态量取值则归属于同一个状态窗口，数值改变后该窗口关闭。状态量所对应的列作为 STAT_WINDOW 语句的参数来指定。 + 3. 会话窗口：时间戳所在的列由 SESSION 语句的 ts_col 参数指定，会话窗口根据相邻两条记录的时间戳差值来确定是否属于同一个会话——如果时间戳差异在 tol_val 以内，则认为记录仍属于同一个窗口；如果时间变化超过 tol_val，则自动开启下一个窗口。 +- WHERE 语句可以指定查询的起止时间和其他过滤条件。 +- FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： + 1. 不进行填充：NONE（默认填充模式）。 + 2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。 + 3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 + 4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 + 5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 + 6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 说明： - 1. 使用FILL语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过1千万条具有插值的结果。 + 1. 使用 FILL 语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过 1 千万条具有插值的结果。 2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 - 3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用GROUP BY语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了GROUP BY语句分组，则返回结果中每个GROUP内不按照时间序列严格单调递增。 + 3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 GROUP BY 语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了 GROUP BY 语句分组，则返回结果中每个 GROUP 内不按照时间序列严格单调递增。 时间聚合也常被用于连续查询场景，可以参考文档 [连续查询(Continuous Query)](https://www.taosdata.com/cn/documentation/advanced-features#continuous-query)。 @@ -1326,7 +1333,7 @@ SELECT function_list FROM stb_name CREATE TABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT); ``` -针对智能电表采集的数据，以10分钟为一个阶段，计算过去24小时的电流数据的平均值、最大值、电流的中位数、以及随着时间变化的电流走势拟合直线。如果没有计算值，用前一个非NULL值填充。使用的查询语句如下： +针对智能电表采集的数据，以 10 分钟为一个阶段，计算过去 24 小时的电流数据的平均值、最大值、电流的中位数、以及随着时间变化的电流走势拟合直线。如果没有计算值，用前一个非 NULL 值填充。使用的查询语句如下： ```mysql SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), PERCENTILE(current, 50) FROM meters From 1583f180d4dc570f07a33c07da744651249e0eb9 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Wed, 23 Jun 2021 15:09:59 +0800 Subject: [PATCH 069/115] pytest utils add taosdemoCfg.py --- tests/pytest/util/taosdemoCfg.py | 450 +++++++++++++++++++++++++++++++ 1 file changed, 450 insertions(+) create mode 100644 tests/pytest/util/taosdemoCfg.py diff --git a/tests/pytest/util/taosdemoCfg.py b/tests/pytest/util/taosdemoCfg.py new file mode 100644 index 0000000000..5071e915a5 --- /dev/null +++ b/tests/pytest/util/taosdemoCfg.py @@ -0,0 +1,450 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime +import inspect +import psutil +import shutil +import json +from util.log import * +from multiprocessing import cpu_count + + +# TODO: fully test the function. Handle exceptions. +# Handle json format not accepted by taosdemo +class TDTaosdemoCfg: + def __init__(self): + self.insert_cfg = { + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": cpu_count(), + "thread_count_create_tbl": cpu_count(), + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 32766, + "max_sql_len": 32766, + "databases": None + } + + self.db = { + "name": 'db', + "drop": 'yes', + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 6, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "cachelast": 0, + "quorum": 1, + "fsync": 3000, + "update": 0 + } + + self.query_cfg = { + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "db", + "query_times": 2, + "query_mode": "taosc", + "specified_table_query": None, + "super_table_query": None + } + + self.table_query = { + "query_interval": 1, + "concurrent": 3, + "sqls": None + } + + self.stable_query = { + "stblname": "stb", + "query_interval": 1, + "threads": 3, + "sqls": None + } + + self.sub_cfg = { + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "db", + "confirm_parameter_prompt": "no", + "specified_table_query": None, + "super_table_query": None + } + + self.table_sub = { + "concurrent": 1, + "mode": "sync", + "interval": 10000, + "restart": "yes", + "keepProgress": "yes", + "sqls": None + } + + self.stable_sub = { + "stblname": "stb", + "threads": 1, + "mode": "sync", + "interval": 10000, + "restart": "yes", + "keepProgress": "yes", + "sqls": None + } + + self.stbs = [] + self.stb_template = { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 10, + "childtable_offset": 0, + "interlace_rows": 0, + "insert_interval": 0, + "max_sql_len": 32766, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT", "count": 1}], + "tags": [{"type": "BIGINT", "count": 1}] + } + + self.tb_query_sql = [] + self.tb_query_sql_template = { + "sql": "select last_row(*) from stb_0 ", + "result": "temp/query_res0.txt" + } + + self.stb_query_sql = [] + self.stb_query_sql_template = { + "sql": "select last_row(ts) from xxxx", + "result": "temp/query_res2.txt" + } + + self.tb_sub_sql = [] + self.tb_sub_sql_template = { + "sql": "select * from stb_0 ;", + "result": "temp/subscribe_res0.txt" + } + + self.stb_sub_sql = [] + self.stb_sub_sql_template = { + "sql": "select * from xxxx where ts > '2021-02-25 11:35:00.000' ;", + "result": "temp/subscribe_res1.txt" + } + + # The following functions are import functions for different dicts and lists + # except import_sql, all other import functions will a dict and overwrite the origional dict + # dict_in: the dict used to overwrite the target + def import_insert_cfg(self, dict_in): + self.insert_cfg = dict_in + + def import_db(self, dict_in): + self.db = dict_in + + def import_stbs(self, dict_in): + self.stbs = dict_in + + def import_query_cfg(self, dict_in): + self.query_cfg = dict_in + + def import_table_query(self, dict_in): + self.table_query = dict_in + + def import_stable_query(self, dict_in): + self.stable_query = dict_in + + def import_sub_cfg(self, dict_in): + self.sub_cfg = dict_in + + def import_table_sub(self, dict_in): + self.table_sub = dict_in + + def import_stable_sub(self, dict_in): + self.stable_sub = dict_in + + def import_sql(self, Sql_in, mode): + """used for importing the sql later used + + Args: + Sql_in (dict): the imported sql dict + mode (str): the sql storing location within TDTaosdemoCfg + format: 'fileType_tableType' + fileType: query, sub + tableType: table, stable + """ + if mode == 'query_table': + self.tb_query_sql = Sql_in + elif mode == 'query_stable': + self.stb_query_sql = Sql_in + elif mode == 'sub_table': + self.tb_sub_sql = Sql_in + elif mode == 'sub_stable': + self.stb_sub_sql = Sql_in + # import functions end + + # The following functions are alter functions for different dicts + # Args: + # key: the key that is going to be modified + # value: the value of the key that is going to be modified + # if key = 'databases' | "specified_table_query" | "super_table_query"|"sqls" + # value will not be used + + def alter_insert_cfg(self, key, value): + + if key == 'databases': + self.insert_cfg[key] = [ + { + 'dbinfo': self.db, + 'super_tables': self.stbs + } + ] + else: + self.insert_cfg[key] = value + + def alter_db(self, key, value): + self.db[key] = value + + def alter_query_tb(self, key, value): + if key == "sqls": + self.table_query[key] = self.tb_query_sql + else: + self.table_query[key] = value + + def alter_query_stb(self, key, value): + if key == "sqls": + self.stable_query[key] = self.stb_query_sql + else: + self.stable_query[key] = value + + def alter_query_cfg(self, key, value): + if key == "specified_table_query": + self.query_cfg["specified_table_query"] = self.table_query + elif key == "super_table_query": + self.query_cfg["super_table_query"] = self.stable_query + else: + self.table_query[key] = value + + def alter_sub_cfg(self, key, value): + if key == "specified_table_query": + self.sub_cfg["specified_table_query"] = self.table_sub + elif key == "super_table_query": + self.sub_cfg["super_table_query"] = self.stable_sub + else: + self.table_query[key] = value + + def alter_sub_stb(self, key, value): + if key == "sqls": + self.stable_sub[key] = self.stb_sub_sql + else: + self.stable_sub[key] = value + + def alter_sub_tb(self, key, value): + if key == "sqls": + self.table_sub[key] = self.tb_sub_sql + else: + self.table_sub[key] = value + # alter function ends + + # the following functions are for handling the sql lists + def append_sql_stb(self, target, value): + """for appending sql dict into specific sql list + + Args: + target (str): the target append list + format: 'fileType_tableType' + fileType: query, sub + tableType: table, stable + unique: 'insert_stbs' + value (dict): the sql dict going to be appended + """ + if target == 'insert_stbs': + self.stbs.append(value) + elif target == 'query_table': + self.tb_query_sql.append(value) + elif target == 'query_stable': + self.stb_query_sql.append(value) + elif target == 'sub_table': + self.tb_sub_sql.append(value) + elif target == 'sub_stable': + self.stb_sub_sql.append(value) + + def pop_sql_stb(self, target, index): + """for poping a sql dict from specific sql list + + Args: + target (str): the target append list + format: 'fileType_tableType' + fileType: query, sub + tableType: table, stable + unique: 'insert_stbs' + index (int): the sql dict that is going to be popped + """ + if target == 'insert_stbs': + self.stbs.pop(index) + elif target == 'query_table': + self.tb_query_sql.pop(index) + elif target == 'query_stable': + self.stb_query_sql.pop(index) + elif target == 'sub_table': + self.tb_sub_sql.pop(index) + elif target == 'sub_stable': + self.stb_sub_sql.pop(index) + # sql list modification function end + + # The following functions are get functions for different dicts + def get_db(self): + return self.db + + def get_stb(self): + return self.stbs + + def get_insert_cfg(self): + return self.insert_cfg + + def get_query_cfg(self): + return self.query_cfg + + def get_tb_query(self): + return self.table_query + + def get_stb_query(self): + return self.stable_query + + def get_sub_cfg(self): + return self.sub_cfg + + def get_tb_sub(self): + return self.table_sub + + def get_stb_sub(self): + return self.stable_sub + + def get_sql(self, target): + """general get function for all sql lists + + Args: + target (str): the sql list want to get + format: 'fileType_tableType' + fileType: query, sub + tableType: table, stable + unique: 'insert_stbs' + """ + if target == 'query_table': + return self.tb_query_sql + elif target == 'query_stable': + return self.stb_query_sql + elif target == 'sub_table': + return self.tb_sub_sql + elif target == 'sub_stable': + return self.stb_sub_sql + + def get_template(self, target): + """general get function for the default sql template + + Args: + target (str): the sql list want to get + format: 'fileType_tableType' + fileType: query, sub + tableType: table, stable + unique: 'insert_stbs' + """ + if target == 'insert_stbs': + return self.stb_template + elif target == 'query_table': + return self.tb_query_sql_template + elif target == 'query_stable': + return self.stb_query_sql_template + elif target == 'sub_table': + return self.tb_sub_sql_template + elif target == 'sub_stable': + return self.stb_sub_sql_template + else: + print(f'did not find {target}') + + # the folloing are the file generation functions + """defalut document: + generator functio for generating taosdemo json file + will assemble the dicts and dump the final json + + Args: + pathName (str): the directory wanting the json file to be + fileName (str): the name suffix of the json file + Returns: + str: [pathName]/[filetype]_[filName].json + """ + + def generate_insert_cfg(self, pathName, fileName): + cfgFileName = f'{pathName}/insert_{fileName}.json' + self.alter_insert_cfg('databases', None) + with open(cfgFileName, 'w') as file: + json.dump(self.insert_cfg, file) + return cfgFileName + + def generate_query_cfg(self, pathName, fileName): + cfgFileName = f'{pathName}/query_{fileName}.json' + self.alter_query_tb('sqls', None) + self.alter_query_stb('sqls', None) + self.alter_query_cfg('specified_table_query', None) + self.alter_query_cfg('super_table_query', None) + with open(cfgFileName, 'w') as file: + json.dump(self.query_cfg, file) + return cfgFileName + + def generate_subscribe_cfg(self, pathName, fileName): + cfgFileName = f'{pathName}/subscribe_{fileName}.json' + self.alter_sub_tb('sqls', None) + self.alter_sub_stb('sqls', None) + self.alter_sub_cfg('specified_table_query', None) + self.alter_sub_cfg('super_table_query', None) + with open(cfgFileName, 'w') as file: + json.dump(self.sub_cfg, file) + return cfgFileName + # file generation functions ends + + def drop_cfg_file(self, fileName): + os.remove(f'{fileName}') + + +taosdemoCfg = TDTaosdemoCfg() From f0091359b2e74418c78891ddaf7c26f0d4fd9263 Mon Sep 17 00:00:00 2001 From: cpwu Date: Wed, 23 Jun 2021 15:32:36 +0800 Subject: [PATCH 070/115] [TD-3825] to fix bug which make with asan caused --- src/client/src/tscServer.c | 4 +++- src/kit/taosdemo/taosdemo.c | 2 +- src/mnode/src/mnodeTable.c | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8c5e99474d..fa277cd9c3 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1495,7 +1495,9 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg = (char *)pSchema; pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen); - memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); + if (pAlterInfo->tagData.dataLen > 0) { + memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); + } pMsg += pAlterInfo->tagData.dataLen; msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 33ee2a9bc2..40168e5e97 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -90,7 +90,7 @@ enum TEST_MODE { #define OPT_ABORT 1 /* –abort */ #define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 -#define MAX_FILE_NAME_LEN 256 +#define MAX_FILE_NAME_LEN 128 #define MAX_SAMPLES_ONCE_FROM_FILE 10000 #define MAX_NUM_DATATYPE 10 diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 032c6ee94b..ba4de89cde 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1068,7 +1068,9 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pStable->info.tableId = strdup(pCreate->tableName); pStable->info.type = TSDB_SUPER_TABLE; pStable->createdTime = taosGetTimestampMs(); - pStable->uid = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + int64_t x = (us&0x000000FFFFFFFFFF); + x = x<<24; + pStable->uid = x + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); pStable->sversion = 0; pStable->tversion = 0; pStable->numOfColumns = numOfColumns; From e1505aceb160b174df0b31d1110b0a1b55a0f707 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 23 Jun 2021 15:51:47 +0800 Subject: [PATCH 071/115] [TD-4874]: reset customScore to 0 --- src/mnode/src/mnodeDnode.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 70a63517ca..ce21af49c2 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -101,6 +101,8 @@ static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) { pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; } + pDnode->customScore = 0; + dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort); mnodeUpdateDnodeEps(); From ce3c6d7a61ec0e4223caf2df59cfe53262cbac70 Mon Sep 17 00:00:00 2001 From: cpwu Date: Wed, 23 Jun 2021 16:58:01 +0800 Subject: [PATCH 072/115] [TD-3825] to fix bug --- tests/tsim/src/simMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 6a9d96bc3b..7d74c62c7d 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -35,7 +35,7 @@ int32_t main(int32_t argc, char *argv[]) { for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { - tstrncpy(configDir, argv[++i], MAX_FILE_NAME_LEN); + tstrncpy(configDir, argv[++i], 128); } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { strcpy(scriptFile, argv[++i]); } else if (strcmp(argv[i], "-a") == 0) { @@ -75,4 +75,4 @@ int32_t main(int32_t argc, char *argv[]) { simInfo("execute result %d", ret); return ret; -} \ No newline at end of file +} From c2734a60dc829d50c122614386ac4cc950b1787f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 23 Jun 2021 20:03:36 +0800 Subject: [PATCH 073/115] [TD-4874]: reset customScore to 0 --- src/mnode/src/mnodeDnode.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index a087b076a5..e3feea7d3a 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -101,6 +101,8 @@ static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) { pDnode->offlineReason = TAOS_DN_OFF_STATUS_NOT_RECEIVED; } + pDnode->customScore = 0; + dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort); mnodeUpdateDnodeEps(); @@ -1296,4 +1298,4 @@ int32_t mnodeCompactDnodes() { mInfo("end to compact dnodes table..."); return 0; -} \ No newline at end of file +} From cbc60231c549a2e40ff231110791f599c1fe4e8f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 23 Jun 2021 21:48:47 +0800 Subject: [PATCH 074/115] Hotfix/sangshuduo/td 3973 use jemalloc for master (#6598) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. Co-authored-by: Shuduo Sang --- .gitmodules | 3 + cmake/define.inc | 5 + deps/CMakeLists.txt | 13 ++ deps/jemalloc | 1 + packaging/deb/makedeb.sh | 50 +++++- packaging/release.sh | 44 +++-- packaging/rpm/makerpm.sh | 6 +- packaging/rpm/tdengine.spec | 81 +++++++-- packaging/tools/install.sh | 309 ++++++++++++++++++-------------- packaging/tools/make_install.sh | 113 ++++++++---- packaging/tools/makepkg.sh | 51 +++++- src/dnode/CMakeLists.txt | 9 +- src/kit/shell/CMakeLists.txt | 11 +- src/kit/taosdemo/CMakeLists.txt | 13 +- src/os/inc/osMemory.h | 4 + 15 files changed, 488 insertions(+), 225 deletions(-) create mode 160000 deps/jemalloc diff --git a/.gitmodules b/.gitmodules index 0e65b02221..346f5c0069 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "tests/examples/rust"] path = tests/examples/rust url = https://github.com/songtianyi/tdengine-rust-bindings.git +[submodule "deps/jemalloc"] + path = deps/jemalloc + url = https://github.com/jemalloc/jemalloc diff --git a/cmake/define.inc b/cmake/define.inc index da80baabbb..69807a8a05 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -59,6 +59,11 @@ IF (TD_LINUX_64) MESSAGE(STATUS "linux64 is defined") SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) + + IF (JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + ENDIF () + ENDIF () IF (TD_LINUX_32) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index cfc17442f5..99152c6ce3 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -18,3 +18,16 @@ ENDIF () IF (TD_DARWIN AND TD_MQTT) ADD_SUBDIRECTORY(MQTT-C) ENDIF () + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + MESSAGE("setup dpes/jemalloc, current source dir:" ${CMAKE_CURRENT_SOURCE_DIR}) + MESSAGE("binary dir:" ${CMAKE_BINARY_DIR}) + include(ExternalProject) + ExternalProject_Add(jemalloc + PREFIX "jemalloc" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure --prefix=${CMAKE_BINARY_DIR}/build/ + BUILD_COMMAND ${MAKE} + ) +ENDIF () diff --git a/deps/jemalloc b/deps/jemalloc new file mode 160000 index 0000000000..ea6b3e973b --- /dev/null +++ b/deps/jemalloc @@ -0,0 +1 @@ +Subproject commit ea6b3e973b477b8061e0076bb257dbd7f3faa756 diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 28be037e6c..e6ddb6d742 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -24,14 +24,14 @@ echo "compile_dir: ${compile_dir}" echo "pkg_dir: ${pkg_dir}" if [ -d ${pkg_dir} ]; then - rm -rf ${pkg_dir} + rm -rf ${pkg_dir} fi mkdir -p ${pkg_dir} cd ${pkg_dir} libfile="libtaos.so.${tdengine_ver}" -# create install dir +# create install dir install_home_path="/usr/local/taos" mkdir -p ${pkg_dir}${install_home_path} mkdir -p ${pkg_dir}${install_home_path}/bin @@ -42,7 +42,7 @@ mkdir -p ${pkg_dir}${install_home_path}/examples mkdir -p ${pkg_dir}${install_home_path}/include mkdir -p ${pkg_dir}${install_home_path}/init.d mkdir -p ${pkg_dir}${install_home_path}/script - + cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script @@ -54,7 +54,7 @@ cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_pat cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin -cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver +cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver cp ${compile_dir}/../src/inc/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../src/inc/taoserror.h ${pkg_dir}${install_home_path}/include cp -r ${top_dir}/tests/examples/* ${pkg_dir}${install_home_path}/examples @@ -67,7 +67,41 @@ fi cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector -cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector ||: +cp ${compile_dir}/build/lib/taos-jdbcdriver*.* ${pkg_dir}${install_home_path}/connector ||: + +if [ -f ${compile_dir}/build/bin/jemalloc-config ]; then + install_user_local_path="/usr/local" + mkdir -p ${pkg_dir}${install_user_local_path}/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${compile_dir}/build/bin/jemalloc-config ${pkg_dir}${install_user_local_path}/bin/ + if [ -f ${compile_dir}/build/bin/jemalloc.sh ]; then + cp ${compile_dir}/build/bin/jemalloc.sh ${pkg_dir}${install_user_local_path}/bin/ + fi + if [ -f ${compile_dir}/build/bin/jeprof ]; then + cp ${compile_dir}/build/bin/jeprof ${pkg_dir}${install_user_local_path}/bin/ + fi + if [ -f ${compile_dir}/build/include/jemalloc/jemalloc.h ]; then + cp ${compile_dir}/build/include/jemalloc/jemalloc.h ${pkg_dir}${install_user_local_path}/include/jemalloc/ + fi + if [ -f ${compile_dir}/build/lib/libjemalloc.so.2 ]; then + cp ${compile_dir}/build/lib/libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/ + ln -sf libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/libjemalloc.so + fi + if [ -f ${compile_dir}/build/lib/libjemalloc.a ]; then + cp ${compile_dir}/build/lib/libjemalloc.a ${pkg_dir}${install_user_local_path}/lib/ + fi + if [ -f ${compile_dir}/build/lib/libjemalloc_pic.a ]; then + cp ${compile_dir}/build/lib/libjemalloc_pic.a ${pkg_dir}${install_user_local_path}/lib/ + fi + if [ -f ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ]; then + cp ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ${pkg_dir}${install_user_local_path}/lib/pkgconfig/ + fi + if [ -f ${compile_dir}/build/share/doc/jemalloc/jemalloc.html ]; then + cp ${compile_dir}/build/share/doc/jemalloc/jemalloc.html ${pkg_dir}${install_user_local_path}/share/doc/jemalloc/ + fi + if [ -f ${compile_dir}/build/share/man/man3/jemalloc.3 ]; then + cp ${compile_dir}/build/share/man/man3/jemalloc.3 ${pkg_dir}${install_user_local_path}/share/man/man3/ + fi +fi cp -r ${compile_dir}/../packaging/deb/DEBIAN ${pkg_dir}/ chmod 755 ${pkg_dir}/DEBIAN/* @@ -75,7 +109,7 @@ chmod 755 ${pkg_dir}/DEBIAN/* # modify version of control debver="Version: "$tdengine_ver sed -i "2c$debver" ${pkg_dir}/DEBIAN/control - + #get taos version, then set deb name @@ -90,7 +124,7 @@ fi if [ "$verType" == "beta" ]; then debname=${debname}-${verType}".deb" -elif [ "$verType" == "stable" ]; then +elif [ "$verType" == "stable" ]; then debname=${debname}".deb" else echo "unknow verType, nor stabel or beta" @@ -101,7 +135,7 @@ fi dpkg -b ${pkg_dir} $debname echo "make deb package success!" -cp ${pkg_dir}/*.deb ${output_dir} +cp ${pkg_dir}/*.deb ${output_dir} # clean tmep dir rm -rf ${pkg_dir} diff --git a/packaging/release.sh b/packaging/release.sh index 68f947ccab..f5027246a6 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -5,8 +5,8 @@ set -e #set -x -# releash.sh -v [cluster | edge] -# -c [aarch32 | aarch64 | x64 | x86 | mips64 ...] +# release.sh -v [cluster | edge] +# -c [aarch32 | aarch64 | x64 | x86 | mips64 ...] # -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] # -V [stable | beta] # -l [full | lite] @@ -22,11 +22,12 @@ cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...] osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] pagMode=full # [full | lite] soMode=dynamic # [static | dynamic] +allocator=glibc # [glibc | jemalloc] dbName=taos # [taos | power] verNumber="" verNumberComp="2.0.0.0" -while getopts "hv:V:c:o:l:s:d:n:m:" arg +while getopts "hv:V:c:o:l:s:d:a:n:m:" arg do case $arg in v) @@ -53,6 +54,10 @@ do #echo "dbName=$OPTARG" dbName=$(echo $OPTARG) ;; + a) + #echo "allocator=$OPTARG" + allocator=$(echo $OPTARG) + ;; n) #echo "verNumber=$OPTARG" verNumber=$(echo $OPTARG) @@ -71,20 +76,21 @@ do echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] " echo " -V [stable | beta] " echo " -l [full | lite] " + echo " -a [glibc | jemalloc] " echo " -s [static | dynamic] " echo " -d [taos | power] " echo " -n [version number] " echo " -m [compatible version number] " exit 0 ;; - ?) #unknow option + ?) #unknow option echo "unkonw argument" exit 1 ;; esac done -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} dbName=${dbName} verNumber=${verNumber} verNumberComp=${verNumberComp}" +echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} dbName=${dbName} allocator=${allocator} verNumber=${verNumber} verNumberComp=${verNumberComp}" curr_dir=$(pwd) @@ -118,7 +124,7 @@ function vercomp () { echo 0 exit 0 fi - + local IFS=. local i ver1=($1) ver2=($2) @@ -164,7 +170,7 @@ if [[ "$verMode" == "cluster" ]]; then else gitinfoOfInternal=NULL fi - + cd ${curr_dir} # 2. cmake executable file @@ -180,12 +186,18 @@ else fi cd ${compile_dir} +if [[ "$allocator" == "jemalloc" ]]; then + allocator_macro="-DJEMALLOC_ENABLED=true" +else + allocator_macro="" +fi + # check support cpu type if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]] ; then if [ "$verMode" != "cluster" ]; then - cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} + cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} ${allocator_macro} else - cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} + cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} ${allocator_macro} fi else echo "input cpuType=${cpuType} error!!!" @@ -199,9 +211,9 @@ cd ${curr_dir} # 3. Call the corresponding script for packaging if [ "$osType" != "Darwin" ]; then if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then - ret='0' + ret='0' command -v dpkg >/dev/null 2>&1 || { ret='1'; } - if [ "$ret" -eq 0 ]; then + if [ "$ret" -eq 0 ]; then echo "====do deb package for the ubuntu system====" output_dir="${top_dir}/debs" if [ -d ${output_dir} ]; then @@ -214,9 +226,9 @@ if [ "$osType" != "Darwin" ]; then echo "==========dpkg command not exist, so not release deb package!!!" fi - ret='0' + ret='0' command -v rpmbuild >/dev/null 2>&1 || { ret='1'; } - if [ "$ret" -eq 0 ]; then + if [ "$ret" -eq 0 ]; then echo "====do rpm package for the centos system====" output_dir="${top_dir}/rpms" if [ -d ${output_dir} ]; then @@ -229,11 +241,11 @@ if [ "$osType" != "Darwin" ]; then echo "==========rpmbuild command not exist, so not release rpm package!!!" fi fi - + echo "====do tar.gz package for all systems====" cd ${script_dir}/tools - - if [[ "$dbName" == "taos" ]]; then + + if [[ "$dbName" == "taos" ]]; then ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${csudo} ./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh index 678e75c500..7c3272f8d0 100755 --- a/packaging/rpm/makerpm.sh +++ b/packaging/rpm/makerpm.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Generate rpm package for centos +# Generate rpm package for centos set -e # set -x @@ -60,7 +60,7 @@ ${csudo} rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_di # copy rpm package to output_dir, and modify package name, then clean temp dir #${csudo} cp -rf RPMS/* ${output_dir} -cp_rpm_package ${pkg_dir}/RPMS +cp_rpm_package ${pkg_dir}/RPMS if [ "$verMode" == "cluster" ]; then @@ -74,7 +74,7 @@ fi if [ "$verType" == "beta" ]; then rpmname=${rpmname}-${verType}".rpm" -elif [ "$verType" == "stable" ]; then +elif [ "$verType" == "stable" ]; then rpmname=${rpmname}".rpm" else echo "unknow verType, nor stabel or beta" diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 9910e20bfe..8a870286ab 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -1,4 +1,5 @@ %define homepath /usr/local/taos +%define userlocalpath /usr/local %define cfg_install_dir /etc/taos %define __strip /bin/true @@ -12,22 +13,22 @@ URL: www.taosdata.com AutoReqProv: no #BuildRoot: %_topdir/BUILDROOT -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root #Prefix: /usr/local/taos -#BuildRequires: -#Requires: +#BuildRequires: +#Requires: %description Big Data Platform Designed and Optimized for IoT -#"prep" Nothing needs to be done +#"prep" Nothing needs to be done #%prep #%setup -q -#%setup -T +#%setup -T -#"build" Nothing needs to be done +#"build" Nothing needs to be done #%build #%configure #make %{?_smp_mflags} @@ -75,9 +76,53 @@ fi cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector -cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector ||: +cp %{_compiledir}/build/lib/taos-jdbcdriver*.* %{buildroot}%{homepath}/connector ||: cp -r %{_compiledir}/../tests/examples/* %{buildroot}%{homepath}/examples + +if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then + mkdir -p %{buildroot}%{userlocalpath}/bin + mkdir -p %{buildroot}%{userlocalpath}/lib + mkdir -p %{buildroot}%{userlocalpath}/lib/pkgconfig + mkdir -p %{buildroot}%{userlocalpath}/include + mkdir -p %{buildroot}%{userlocalpath}/include/jemalloc + mkdir -p %{buildroot}%{userlocalpath}/share + mkdir -p %{buildroot}%{userlocalpath}/share/doc + mkdir -p %{buildroot}%{userlocalpath}/share/doc/jemalloc + mkdir -p %{buildroot}%{userlocalpath}/share/man + mkdir -p %{buildroot}%{userlocalpath}/share/man/man3 + + cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{userlocalpath}/bin/ + if [ -f %{_compiledir}/build/bin/jemalloc.sh ]; then + cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{userlocalpath}/bin/ + fi + if [ -f %{_compiledir}/build/bin/jeprof ]; then + cp %{_compiledir}/build/bin/jeprof %{buildroot}%{userlocalpath}/bin/ + fi + if [ -f %{_compiledir}/build/include/jemalloc/jemalloc.h ]; then + cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{userlocalpath}/include/jemalloc/ + fi + if [ -f %{_compiledir}/build/lib/libjemalloc.so.2 ]; then + cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/ + ln -sf libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/libjemalloc.so + fi + if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then + cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{userlocalpath}/lib/ + fi + if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then + cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{userlocalpath}/lib/ + fi + if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then + cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{userlocalpath}/lib/pkgconfig/ + fi + if [ -f %{_compiledir}/build/share/doc/jemalloc/jemalloc.html ]; then + cp %{_compiledir}/build/share/doc/jemalloc/jemalloc.html %{buildroot}%{userlocalpath}/share/doc/jemalloc/ + fi + if [ -f %{_compiledir}/build/share/man/man3/jemalloc.3 ]; then + cp %{_compiledir}/build/share/man/man3/jemalloc.3 %{buildroot}%{userlocalpath}/share/man/man3/ + fi +fi + #Scripts executed before installation %pre csudo="" @@ -103,7 +148,7 @@ fi # if taos.cfg already softlink, remove it if [ -f %{cfg_install_dir}/taos.cfg ]; then ${csudo} rm -f %{homepath}/cfg/taos.cfg || : -fi +fi # there can not libtaos.so*, otherwise ln -s error ${csudo} rm -f %{homepath}/driver/libtaos* || : @@ -116,18 +161,18 @@ if command -v sudo > /dev/null; then fi cd %{homepath}/script ${csudo} ./post.sh - + # Scripts executed before uninstall %preun csudo="" if command -v sudo > /dev/null; then csudo="sudo" fi -# only remove package to call preun.sh, not but update(2) +# only remove package to call preun.sh, not but update(2) if [ $1 -eq 0 ];then #cd %{homepath}/script #${csudo} ./preun.sh - + if [ -f %{homepath}/script/preun.sh ]; then cd %{homepath}/script ${csudo} ./preun.sh @@ -135,7 +180,7 @@ if [ $1 -eq 0 ];then bin_link_dir="/usr/bin" lib_link_dir="/usr/lib" inc_link_dir="/usr/include" - + data_link_dir="/usr/local/taos/data" log_link_dir="/usr/local/taos/log" cfg_link_dir="/usr/local/taos/cfg" @@ -149,20 +194,20 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : ${csudo} rm -f ${lib_link_dir}/libtaos.* || : - + ${csudo} rm -f ${log_link_dir} || : ${csudo} rm -f ${data_link_dir} || : - + pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo} kill -9 $pid || : - fi - fi + fi + fi fi - + # Scripts executed after uninstall %postun - + # clean build dir %clean csudo="" diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 178a248cfe..325ac81053 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -59,11 +59,11 @@ initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then service_mod=0 -elif $(which service &> /dev/null); then +elif $(which service &> /dev/null); then service_mod=1 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" if $(which chkconfig &> /dev/null); then - initd_mod=1 + initd_mod=1 elif $(which insserv &> /dev/null); then initd_mod=2 elif $(which update-rc.d &> /dev/null); then @@ -71,7 +71,7 @@ elif $(which service &> /dev/null); then else service_mod=2 fi -else +else service_mod=2 fi @@ -103,7 +103,7 @@ elif echo $osinfo | grep -qwi "fedora" ; then os_type=2 else echo " osinfo: ${osinfo}" - echo " This is an officially unverified linux system," + echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " echo " please feel free to contact taosdata.com for support." os_type=1 @@ -138,7 +138,7 @@ do echo "Usage: `basename $0` -v [server | client] -e [yes | no]" exit 0 ;; - ?) #unknow option + ?) #unknow option echo "unkonw argument" exit 1 ;; @@ -157,9 +157,9 @@ function kill_process() { function install_main_path() { #create install main dir and all sub dir ${csudo} rm -rf ${install_main_dir} || : - ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir} ${csudo} mkdir -p ${install_main_dir}/cfg - ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/bin ${csudo} mkdir -p ${install_main_dir}/connector ${csudo} mkdir -p ${install_main_dir}/driver ${csudo} mkdir -p ${install_main_dir}/examples @@ -168,10 +168,10 @@ function install_main_path() { if [ "$verMode" == "cluster" ]; then ${csudo} mkdir -p ${nginx_dir} fi - + if [[ -e ${script_dir}/email ]]; then - ${csudo} cp ${script_dir}/email ${install_main_dir}/ ||: - fi + ${csudo} cp ${script_dir}/email ${install_main_dir}/ ||: + fi } function install_bin() { @@ -207,29 +207,75 @@ function install_lib() { ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : #${csudo} rm -rf ${v15_java_app_dir} || : - ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : fi - - #if [ "$verMode" == "cluster" ]; then + + #if [ "$verMode" == "cluster" ]; then # # Compatible with version 1.5 # ${csudo} mkdir -p ${v15_java_app_dir} # ${csudo} ln -s ${install_main_dir}/connector/taos-jdbcdriver-1.0.2-dist.jar ${v15_java_app_dir}/JDBCDriver-1.0.2-dist.jar # ${csudo} chmod 777 ${v15_java_app_dir} || : #fi - + ${csudo} ldconfig } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function install_header() { ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } @@ -246,13 +292,13 @@ function add_newHostname_to_hosts() { if [[ "$s" == "$localIp" ]]; then return fi - done + done ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: } function set_hostname() { echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" - read newHostname + read newHostname while true; do if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then break @@ -266,25 +312,25 @@ function set_hostname() { if [[ $retval != 0 ]]; then echo echo "set hostname fail!" - return + return fi #echo -e -n "$(hostnamectl status --static)" #echo -e -n "$(hostnamectl status --transient)" #echo -e -n "$(hostnamectl status --pretty)" - + #ubuntu/centos /etc/hostname if [[ -e /etc/hostname ]]; then ${csudo} echo $newHostname > /etc/hostname ||: fi - + #debian: #HOSTNAME=yourname if [[ -e /etc/sysconfig/network ]]; then ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: fi ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg - serverFqdn=$newHostname - + serverFqdn=$newHostname + if [[ -e /etc/hosts ]]; then add_newHostname_to_hosts $newHostname fi @@ -302,7 +348,7 @@ function is_correct_ipaddr() { return 0 fi done - + return 1 } @@ -316,13 +362,13 @@ function set_ipAsFqdn() { echo echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg serverFqdn=$localFqdn echo return - fi - + fi + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" echo echo -e -n "${GREEN}$iplist${NC}" @@ -331,15 +377,15 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" read localFqdn while true; do - if [ ! -z "$localFqdn" ]; then + if [ ! -z "$localFqdn" ]; then # Check if correct ip address is_correct_ipaddr $localFqdn retval=`echo $?` if [[ $retval != 0 ]]; then read -p "Please choose an IP from local IP list:" localFqdn else - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg serverFqdn=$localFqdn break fi @@ -354,59 +400,59 @@ function local_fqdn_check() { echo echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" echo - if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" echo - + while true do - read -r -p "Set hostname now? [Y/n] " input - if [ ! -n "$input" ]; then - set_hostname - break - else - case $input in - [yY][eE][sS]|[yY]) - set_hostname - break - ;; - - [nN][oO]|[nN]) - set_ipAsFqdn - break - ;; - - *) - echo "Invalid input..." - ;; - esac - fi + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi done fi } function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* - fi - + fi + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client - + if ((${update_flag}==1)); then return 0 fi - + if [ "$interactiveFqdn" == "no" ]; then return 0 - fi - + fi + local_fqdn_check #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" @@ -424,8 +470,8 @@ function install_config() { if [ ! -z "$firstEp" ]; then # check the format of the firstEp #if [[ $firstEp == $FQDN_PATTERN ]]; then - # Write the first FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg break #else # read -p "Please enter the correct FQDN:port: " firstEp @@ -433,9 +479,9 @@ function install_config() { else break fi - done + done - # user email + # user email #EMAIL_PATTERN='^[A-Za-z0-9\u4e00-\u9fa5]+@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)+$' #EMAIL_PATTERN='^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$' #EMAIL_PATTERN="^[\w-]+(\.[\w-]+)*@[\w-]+(\.[\w-]+)+$" @@ -446,31 +492,31 @@ function install_config() { if [ ! -z "$emailAddr" ]; then # check the format of the emailAddr #if [[ "$emailAddr" =~ $EMAIL_PATTERN ]]; then - # Write the email address to temp file - email_file="${install_main_dir}/email" + # Write the email address to temp file + email_file="${install_main_dir}/email" ${csudo} bash -c "echo $emailAddr > ${email_file}" - break + break #else - # read -p "Please enter the correct email address: " emailAddr + # read -p "Please enter the correct email address: " emailAddr #fi else break fi - done + done } function install_log() { ${csudo} rm -rf ${log_dir} || : ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} - + ${csudo} ln -s ${log_dir} ${install_main_dir}/log } function install_data() { ${csudo} mkdir -p ${data_dir} - - ${csudo} ln -s ${data_dir} ${install_main_dir}/data + + ${csudo} ln -s ${data_dir} ${install_main_dir}/data } function install_connector() { @@ -485,26 +531,26 @@ function install_examples() { function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof taosd &> /dev/null; then ${csudo} service taosd stop || : fi - + if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi if ((${initd_mod}==1)); then - if [ -e ${service_config_dir}/taosd ]; then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} chkconfig --del taosd || : fi - if [ -e ${service_config_dir}/tarbitratord ]; then + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : fi elif ((${initd_mod}==2)); then - if [ -e ${service_config_dir}/taosd ]; then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} insserv -r taosd || : fi if [ -e ${service_config_dir}/tarbitratord ]; then @@ -518,10 +564,10 @@ function clean_service_on_sysvinit() { ${csudo} update-rc.d -f tarbitratord remove || : fi fi - + ${csudo} rm -f ${service_config_dir}/taosd || : ${csudo} rm -f ${service_config_dir}/tarbitratord || : - + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -544,10 +590,10 @@ function install_service_on_sysvinit() { ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord fi - + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - + if ((${initd_mod}==1)); then ${csudo} chkconfig --add taosd || : ${csudo} chkconfig --level 2345 taosd on || : @@ -572,7 +618,7 @@ function clean_service_on_systemd() { fi ${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null ${csudo} rm -f ${taosd_service_config} - + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" if systemctl is-active --quiet tarbitratord; then echo "tarbitrator is running, stopping it..." @@ -580,7 +626,7 @@ function clean_service_on_systemd() { fi ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null ${csudo} rm -f ${tarbitratord_service_config} - + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" if systemctl is-active --quiet nginxd; then @@ -588,8 +634,8 @@ function clean_service_on_systemd() { ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - fi + ${csudo} rm -f ${nginx_service_config} + fi } # taos:2345:respawn:/etc/init.d/taosd start @@ -621,7 +667,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" ${csudo} systemctl enable taosd - + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" @@ -643,9 +689,9 @@ function install_service_on_systemd() { ${csudo} bash -c "echo >> ${tarbitratord_service_config}" ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - #${csudo} systemctl enable tarbitratord - - if [ "$verMode" == "cluster" ]; then + #${csudo} systemctl enable tarbitratord + + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" ${csudo} bash -c "echo 'Description=Nginx For TDengine Service' >> ${nginx_service_config}" @@ -674,7 +720,7 @@ function install_service_on_systemd() { ${csudo} systemctl enable nginxd fi ${csudo} systemctl start nginxd - fi + fi } function install_service() { @@ -757,7 +803,7 @@ function update_TDengine() { fi sleep 1 fi - + if [ "$verMode" == "cluster" ]; then if pidof nginx &> /dev/null; then if ((${service_mod}==0)); then @@ -770,12 +816,13 @@ function update_TDengine() { sleep 1 fi fi - + install_main_path install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -783,10 +830,10 @@ function update_TDengine() { if [ -z $1 ]; then install_bin install_service - install_config - + install_config + openresty_work=false - if [ "$verMode" == "cluster" ]; then + if [ "$verMode" == "cluster" ]; then # Check if openresty is installed # Check if nginx is installed successfully if type curl &> /dev/null; then @@ -797,7 +844,7 @@ function update_TDengine() { echo -e "\033[44;31;5mNginx for TDengine does not work! Please try again!\033[0m" fi fi - fi + fi #echo #echo -e "\033[44;32;1mTDengine is updated successfully!${NC}" @@ -816,7 +863,7 @@ function update_TDengine() { else echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos -h $serverFqdn${NC} in shell${NC}" fi - + echo echo -e "\033[44;32;1mTDengine is updated successfully!${NC}" else @@ -839,14 +886,14 @@ function install_TDengine() { tar -zxf taos.tar.gz echo -e "${GREEN}Start to install TDengine...${NC}" - - install_main_path - + + install_main_path + if [ -z $1 ]; then install_data - fi - - install_log + fi + + install_log install_header install_lib if [ "$pagMode" != "lite" ]; then @@ -871,8 +918,8 @@ function install_TDengine() { fi fi fi - - install_config + + install_config # Ask if to start the service #echo @@ -885,36 +932,36 @@ function install_TDengine() { echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}" else echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}" - fi + fi #if [ ${openresty_work} = 'true' ]; then # echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" #else # echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" #fi - + if [ ! -z "$firstEp" ]; then - tmpFqdn=${firstEp%%:*} - substr=":" - if [[ $firstEp =~ $substr ]];then - tmpPort=${firstEp#*:} - else - tmpPort="" - fi - if [[ "$tmpPort" != "" ]];then - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" - else - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" - fi - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" - echo + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" + echo fi - + echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" - echo + echo else # Only install client install_bin install_config @@ -945,6 +992,6 @@ elif [ "$verType" == "client" ]; then else install_TDengine client fi -else - echo "please input correct verType" +else + echo "please input correct verType" fi diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index d6ace0a063..0c755d9f72 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -1,12 +1,12 @@ #!/bin/bash # -# This file is used to install TAOS time-series database on linux systems. The operating system +# This file is used to install TAOS time-series database on linux systems. The operating system # is required to use systemd to manage services at boot set -e # set -x -# -----------------------Variables definition--------------------- +# -----------------------Variables definition source_dir=$1 binary_dir=$2 osType=$3 @@ -71,9 +71,9 @@ if [ "$osType" != "Darwin" ]; then service_mod=0 elif $(which service &> /dev/null); then service_mod=1 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" if $(which chkconfig &> /dev/null); then - initd_mod=1 + initd_mod=1 elif $(which insserv &> /dev/null); then initd_mod=2 elif $(which update-rc.d &> /dev/null); then @@ -123,9 +123,9 @@ function kill_taosd() { function install_main_path() { #create install main dir and all sub dir ${csudo} rm -rf ${install_main_dir} || : - ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir} ${csudo} mkdir -p ${install_main_dir}/cfg - ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/bin ${csudo} mkdir -p ${install_main_dir}/connector ${csudo} mkdir -p ${install_main_dir}/driver ${csudo} mkdir -p ${install_main_dir}/examples @@ -176,6 +176,49 @@ function install_bin() { [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || : fi } +function install_jemalloc() { + if [ "$osType" != "Darwin" ]; then + /usr/bin/install -c -d /usr/local/bin + + if [ -f ${binary_dir}/build/bin/jemalloc-config ]; then + /usr/bin/install -c -m 755 ${binary_dir}/build/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${binary_dir}/build/bin/jemalloc.sh ]; then + /usr/bin/install -c -m 755 ${binary_dir}/build/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${binary_dir}/build/bin/jeprof ]; then + /usr/bin/install -c -m 755 ${binary_dir}/build/bin/jeprof /usr/local/bin + fi + if [ -f ${binary_dir}/build/include/jemalloc/jemalloc.h ]; then + /usr/bin/install -c -d /usr/local/include/jemalloc + /usr/bin/install -c -m 644 ${binary_dir}/build/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${binary_dir}/build/lib/libjemalloc.so.2 ]; then + /usr/bin/install -c -d /usr/local/lib + /usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.so.2 /usr/local/lib + ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + /usr/bin/install -c -d /usr/local/lib + if [ -f ${binary_dir}/build/lib/libjemalloc.a ]; then + /usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${binary_dir}/build/lib/libjemalloc_pic.a ]; then + /usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${binary_dir}/build/lib/pkgconfig/jemalloc.pc ]; then + /usr/bin/install -c -d /usr/local/lib/pkgconfig + /usr/bin/install -c -m 644 ${binary_dir}/build/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${binary_dir}/build/share/doc/jemalloc/jemalloc.html ]; then + /usr/bin/install -c -d /usr/local/share/doc/jemalloc + /usr/bin/install -c -m 644 ${binary_dir}/build/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${binary_dir}/build/share/man/man3/jemalloc.3 ]; then + /usr/bin/install -c -d /usr/local/share/man/man3 + /usr/bin/install -c -m 644 ${binary_dir}/build/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} function install_lib() { # Remove links @@ -183,12 +226,12 @@ function install_lib() { if [ "$osType" != "Darwin" ]; then ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : fi - + if [ "$osType" != "Darwin" ]; then ${csudo} cp ${binary_dir}/build/lib/libtaos.so.${verNumber} ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - + if [ -d "${lib64_link_dir}" ]; then ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 ${csudo} ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so @@ -198,7 +241,9 @@ function install_lib() { ${csudo} ln -sf ${install_main_dir}/driver/libtaos.1.dylib ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - + + install_jemalloc + if [ "$osType" != "Darwin" ]; then ${csudo} ldconfig fi @@ -206,26 +251,26 @@ function install_lib() { function install_header() { - ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - - if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} [ -f ${script_dir}/../cfg/taos.cfg ] && ${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* - fi - + fi + ${csudo} cp -f ${script_dir}/../cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg } -function install_log() { +function install_log() { ${csudo} rm -rf ${log_dir} || : if [ "$osType" != "Darwin" ]; then @@ -239,7 +284,7 @@ function install_log() { function install_data() { ${csudo} mkdir -p ${data_dir} - ${csudo} ln -s ${data_dir} ${install_main_dir}/data + ${csudo} ln -s ${data_dir} ${install_main_dir}/data } function install_connector() { @@ -254,8 +299,8 @@ function install_connector() { echo "WARNING: go connector not found, please check if want to use it!" fi ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector - - ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null + + ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null } function install_examples() { @@ -264,8 +309,8 @@ function install_examples() { function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof taosd &> /dev/null; then ${csudo} service taosd stop || : fi @@ -277,9 +322,9 @@ function clean_service_on_sysvinit() { elif ((${initd_mod}==3)); then ${csudo} update-rc.d -f taosd remove || : fi - + ${csudo} rm -f ${service_config_dir}/taosd || : - + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -298,10 +343,10 @@ function install_service_on_sysvinit() { ${csudo} cp -f ${script_dir}/../rpm/taosd ${install_main_dir}/init.d ${csudo} cp ${script_dir}/../rpm/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd fi - + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - + if ((${initd_mod}==1)); then ${csudo} chkconfig --add taosd || : ${csudo} chkconfig --level 2345 taosd on || : @@ -323,7 +368,7 @@ function clean_service_on_systemd() { ${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null ${csudo} rm -f ${taosd_service_config} -} +} # taos:2345:respawn:/etc/init.d/taosd start @@ -383,7 +428,7 @@ function update_TDengine() { sleep 1 fi fi - + install_main_path install_log @@ -431,16 +476,16 @@ function install_TDengine() { # Start to install if [ "$osType" != "Darwin" ]; then echo -e "${GREEN}Start to install TDEngine...${NC}" - else - echo -e "${GREEN}Start to install TDEngine Client ...${NC}" + else + echo -e "${GREEN}Start to install TDEngine Client ...${NC}" fi - install_main_path + install_main_path - if [ "$osType" != "Darwin" ]; then + if [ "$osType" != "Darwin" ]; then install_data fi - install_log + install_log install_header install_lib install_connector @@ -452,7 +497,7 @@ function install_TDengine() { install_service fi - install_config + install_config if [ "$osType" != "Darwin" ]; then # Ask if to start the service diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index e4d2d71b01..624f72278a 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -30,12 +30,12 @@ else install_dir="${release_dir}/TDengine-server-${version}" fi -# Directories and files. +# Directories and files if [ "$pagMode" == "lite" ]; then - strip ${build_dir}/bin/taosd + strip ${build_dir}/bin/taosd strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" -else +else bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator\ ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" fi @@ -73,10 +73,43 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/taos mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : +if [ -f ${build_dir}/bin/jemalloc-config ]; then + mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} + cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin + if [ -f ${build_dir}/bin/jemalloc.sh ]; then + cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/bin/jeprof ]; then + cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin + fi + if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then + cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc + fi + if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then + cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib + ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so + fi + if [ -f ${build_dir}/lib/libjemalloc.a ]; then + cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then + cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib + fi + if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then + cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig + fi + if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then + cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc + fi + if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then + cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3 + fi +fi + if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >> remove_temp.sh mv remove_temp.sh ${install_dir}/bin/remove.sh - + mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png rm -rf ${install_dir}/nginxd/png @@ -132,7 +165,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ -d ${examples_dir}/JDBC/taosdemo/target ]; then rm -rf ${examples_dir}/JDBC/taosdemo/target fi - + cp -r ${examples_dir}/JDBC ${install_dir}/examples cp -r ${examples_dir}/matlab ${install_dir}/examples cp -r ${examples_dir}/python ${install_dir}/examples @@ -142,7 +175,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then cp -r ${examples_dir}/C# ${install_dir}/examples fi # Copy driver -mkdir -p ${install_dir}/driver +mkdir -p ${install_dir}/driver cp ${lib_files} ${install_dir}/driver # Copy connector @@ -168,7 +201,7 @@ fi # exit 1 -cd ${release_dir} +cd ${release_dir} if [ "$verMode" == "cluster" ]; then pkg_name=${install_dir}-${osType}-${cpuType} @@ -185,8 +218,8 @@ fi if [ "$verType" == "beta" ]; then pkg_name=${pkg_name}-${verType} -elif [ "$verType" == "stable" ]; then - pkg_name=${pkg_name} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} else echo "unknow verType, nor stabel or beta" exit 1 diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index dd18f00920..f8d8f88438 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -10,8 +10,15 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") +ELSE () + SET(LINK_JEMALLOC "") +ENDIF () + ADD_EXECUTABLE(taosd ${SRC}) -TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) +TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync ${LINK_JEMALLOC}) IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(taosd taos_static) diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index d36c1e3fcc..d904945435 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -11,10 +11,17 @@ IF (TD_LINUX) LIST(REMOVE_ITEM SRC ./src/shellDarwin.c) ADD_EXECUTABLE(shell ${SRC}) +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") +ELSE () + SET(LINK_JEMALLOC "") +ENDIF () + IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(shell taos_static) + TARGET_LINK_LIBRARIES(shell taos_static ${LINK_JEMALLOC}) ELSE () - TARGET_LINK_LIBRARIES(shell taos) + TARGET_LINK_LIBRARIES(shell taos ${LINK_JEMALLOC}) ENDIF () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 5f75be0e19..091eecfe27 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -55,14 +55,21 @@ ENDIF () MESSAGE("TD_VERSION_NUMBER is:" ${TD_VERSION_NUMBER}) ADD_DEFINITIONS(-DTD_VERNUMBER="${TD_VERSION_NUMBER}") +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") +ELSE () + SET(LINK_JEMALLOC "") +ENDIF () + IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosdemo taos_static cJson) + TARGET_LINK_LIBRARIES(taosdemo taos_static cJson ${LINK_JEMALLOC}) ELSE () - TARGET_LINK_LIBRARIES(taosdemo taos cJson) + TARGET_LINK_LIBRARIES(taosdemo taos cJson ${LINK_JEMALLOC}) ENDIF () ELSEIF (TD_WINDOWS) AUX_SOURCE_DIRECTORY(. SRC) @@ -71,7 +78,7 @@ ELSEIF (TD_WINDOWS) IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(taosdemo taos_static cJson) ELSE () - TARGET_LINK_LIBRARIES(taosdemo taos cJson}) + TARGET_LINK_LIBRARIES(taosdemo taos cJson) ENDIF () ELSEIF (TD_DARWIN) # missing a few dependencies, such as diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h index 2cf7e14d2f..12d5c5ad78 100644 --- a/src/os/inc/osMemory.h +++ b/src/os/inc/osMemory.h @@ -22,6 +22,10 @@ extern "C" { #endif +#ifdef TD_JEMALLOC_ENABLED +#include +#endif + typedef enum { TAOS_ALLOC_MODE_DEFAULT = 0, TAOS_ALLOC_MODE_RANDOM_FAIL = 1, From 592278a65613ae73e85906275b0aced70f0943c5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 23 Jun 2021 23:38:45 +0800 Subject: [PATCH 075/115] fix compile error in test --- cmake/env.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/env.inc b/cmake/env.inc index 356bd61442..fa15ec6aee 100755 --- a/cmake/env.inc +++ b/cmake/env.inc @@ -39,7 +39,7 @@ SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_C_FLAGS} ${DEBUG_FLAGS} SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_C_FLAGS} ${RELEASE_FLAGS}") # Set c++ compiler options -SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11") +SET(COMMON_CXX_FLAGS "${COMMON_FLAGS} -std=c++11 -Wno-unused-function") SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_CXX_FLAGS} ${DEBUG_FLAGS}") SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") From 5cadc8242fd74222d6814e4a0c99dd391e09986f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 24 Jun 2021 10:03:46 +0800 Subject: [PATCH 076/115] fix compile errors in CI --- src/query/tests/astTest.cpp | 2 +- src/query/tests/histogramTest.cpp | 4 +++- src/query/tests/patternMatchTest.cpp | 3 ++- src/query/tests/percentileTest.cpp | 3 ++- src/query/tests/resultBufferTest.cpp | 3 ++- src/query/tests/tsBufTest.cpp | 4 +++- src/query/tests/unitTest.cpp | 9 ++++++--- 7 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index 7bd1c0bf8e..1143d00e8d 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -10,7 +10,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Werror=unused-function" +#pragma GCC diagnostic ignored "-Wunused-function" typedef struct ResultObj { int32_t numOfResult; diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp index 44a31f4241..0266ecffc1 100644 --- a/src/query/tests/histogramTest.cpp +++ b/src/query/tests/histogramTest.cpp @@ -6,7 +6,9 @@ #include "taos.h" #include "qHistogram.h" -#pragma GCC diagnostic ignored "-Werror=unused-function" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" + namespace { void doHistogramAddTest() { SHistogramInfo* pHisto = NULL; diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index cd242afc84..091604c65c 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -6,7 +6,8 @@ #include "qAggMain.h" #include "tcompare.h" -#pragma GCC diagnostic ignored "-Werror=unused-function" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" TEST(testCase, patternMatchTest) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; diff --git a/src/query/tests/percentileTest.cpp b/src/query/tests/percentileTest.cpp index 952129c8e7..1b6951201a 100644 --- a/src/query/tests/percentileTest.cpp +++ b/src/query/tests/percentileTest.cpp @@ -7,7 +7,8 @@ #include "qPercentile.h" -#pragma GCC diagnostic ignored "-Werror=unused-function" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" namespace { tMemBucket *createBigIntDataBucket(int32_t start, int32_t end) { diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 052e654066..54ac0bf4e5 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -6,7 +6,8 @@ #include "taos.h" #include "tsdb.h" -#pragma GCC diagnostic ignored "-Werror=unused-function" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" namespace { // simple test diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index 30faed1bc2..04c5a15252 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -9,7 +9,9 @@ #include "ttoken.h" #include "tutil.h" -#pragma GCC diagnostic ignored "-Werror=unused-function" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" namespace { /** diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index 75153a76ae..fcfed49140 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -6,14 +6,17 @@ #include "taos.h" #include "tsdb.h" +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + #include "../../client/inc/tscUtil.h" #include "tutil.h" #include "tvariant.h" #include "ttokendef.h" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Werror=unused-function" namespace { int32_t testValidateName(char* name) { SStrToken token = {0}; From 15014c2e19e51353e1a2f49a5d1bf6d3a9729733 Mon Sep 17 00:00:00 2001 From: wpan Date: Thu, 24 Jun 2021 10:50:10 +0800 Subject: [PATCH 077/115] fix bug --- src/client/src/tscSubquery.c | 2 +- src/query/inc/qExecutor.h | 1 + src/query/src/qExecutor.c | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c3df4773e1..4d97fef52f 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -3604,10 +3604,10 @@ void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGr // todo refactor: filter should not be applied here. createFilterInfo(pQueryAttr, 0); - pQueryAttr->numOfFilterCols = 0; SArray* pa = NULL; if (stage == MASTER_SCAN) { + pQueryAttr->createFilterOperator = false; // no need for parent query pa = createExecOperatorPlan(pQueryAttr); } else { pa = createGlobalMergePlan(pQueryAttr); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 9cd1c5b033..dbee1a7812 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -192,6 +192,7 @@ typedef struct SQueryAttr { bool needReverseScan; // need reverse scan bool distinctTag; // distinct tag query bool stateWindow; // window State on sub/normal table + bool createFilterOperator; // if filter operator is needed int32_t interBufSize; // intermediate buffer sizse int32_t havingNum; // having expr number diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 97a6cf807c..63e0025550 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -7045,6 +7045,8 @@ int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, &pQueryAttr->pFilterInfo, qId); + pQueryAttr->createFilterOperator = true; + return TSDB_CODE_SUCCESS; } From 069169e7c1d2209f543f838ea54631929aac9f1c Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Thu, 24 Jun 2021 15:04:02 +0800 Subject: [PATCH 078/115] WIP: [TD-4872]: fix buffer overflow in -O3 build (#6593) * [TD-4872]: fix buffer overflow in -O3 build * [TD-4872]: fix tasodemo buffer overflow with -O3 * [TD-4872]: fix tasodump buffer overflow with -O3 --- src/kit/taosdemo/taosdemo.c | 2 +- src/kit/taosdump/taosdump.c | 8 +++++--- src/tfs/src/tfs.c | 4 +++- src/util/src/tconfig.c | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 40168e5e97..a6ba7e9493 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5103,7 +5103,7 @@ static int32_t generateStbDataTail( } else { retLen = getRowDataFromSample( data, - remainderBufLen, + remainderBufLen < MAX_DATA_SIZE ? remainderBufLen : MAX_DATA_SIZE, startTime + superTblInfo->timeStampStep * k, superTblInfo, pSamplePos); diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 165bbdf990..05c6b1efbb 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -29,6 +29,9 @@ #define COMMAND_SIZE 65536 //#define DEFAULT_DUMP_FILE "taosdump.sql" +// for strncpy buffer overflow +#define min(a, b) (((a) < (b)) ? (a) : (b)) + int converStringToReadable(char *str, int size, char *buf, int bufsize); int convertNCharToReadable(char *str, int size, char *buf, int bufsize); void taosDumpCharset(FILE *fp); @@ -1119,12 +1122,11 @@ int taosGetTableDes( TAOS_FIELD *fields = taos_fetch_fields(res); tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); - while ((row = taos_fetch_row(res)) != NULL) { strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + min(16, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); @@ -1575,7 +1577,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); + min(TSDB_TABLE_NAME_LEN, fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes)); taosWrite(fd, &tableRecord, sizeof(STableRecord)); diff --git a/src/tfs/src/tfs.c b/src/tfs/src/tfs.c index f78535b8ed..9dc68dcdfd 100644 --- a/src/tfs/src/tfs.c +++ b/src/tfs/src/tfs.c @@ -480,11 +480,13 @@ static int tfsFormatDir(char *idir, char *odir) { return -1; } - if (realpath(wep.we_wordv[0], odir) == NULL) { + char tmp[PATH_MAX] = {0}; + if (realpath(wep.we_wordv[0], tmp) == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); wordfree(&wep); return -1; } + strcpy(odir, tmp); wordfree(&wep); return 0; diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index c4bd577602..442e83bb4f 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -151,7 +151,7 @@ static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { wordfree(&full_path); - char tmp[1025] = {0}; + char tmp[PATH_MAX] = {0}; if (realpath(option, tmp) != NULL) { strcpy(option, tmp); } From eed57b8a5b46aeaed9231973fcd214eef03603ff Mon Sep 17 00:00:00 2001 From: wpan Date: Thu, 24 Jun 2021 15:28:26 +0800 Subject: [PATCH 079/115] fix bug --- src/query/src/qPlan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index ee587a515d..d2eb9afb22 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -623,7 +623,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); } } else { // diff/add/multiply/subtract/division - if (pQueryAttr->numOfFilterCols > 0 && pQueryAttr->vgId == 0) { // todo refactor + if (pQueryAttr->numOfFilterCols > 0 && pQueryAttr->createFilterOperator && pQueryAttr->vgId == 0) { // todo refactor op = OP_Filter; taosArrayPush(plan, &op); } else { From 28a4d4c0308c8d84684955d979ca579c2473d8bd Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 24 Jun 2021 16:16:33 +0800 Subject: [PATCH 080/115] [TD-1568]fix tdMergeDataCols compare key bug --- src/common/inc/tdataformat.h | 5 +++++ src/common/src/tdataformat.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 8ee7329156..e3989a1deb 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -289,6 +289,11 @@ static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { } } +static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int row) { + ASSERT(row < pCols->numOfRows); + return dataColsKeyAt(pCols, row); +} + static FORCE_INLINE TSKEY dataColsKeyFirst(SDataCols *pCols) { if (pCols->numOfRows) { return dataColsKeyAt(pCols, 0); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 7ae34d532c..94c429cfc0 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -452,7 +452,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * SDataCols *pTarget = NULL; - if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyFirst(source))) { // No overlap + if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) { // No overlap ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); for (int i = 0; i < rowsToMerge; i++) { for (int j = 0; j < source->numOfCols; j++) { From b945a7fd333dbabb6bf0f8216e5a9cea03c0e5c2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Jun 2021 16:33:57 +0800 Subject: [PATCH 081/115] [td-4791] check sql function in outer query. --- src/client/src/tscSQLParser.c | 58 ++++++++++-------- src/query/inc/qExecutor.h | 8 +-- src/query/src/qAggMain.c | 1 - src/query/src/qExecutor.c | 54 ++++++++--------- src/query/src/qPlan.c | 14 ++--- tests/script/general/parser/nestquery.sim | 72 +++++++++++++++++++++-- 6 files changed, 138 insertions(+), 69 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e94f77d254..815442a038 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2069,33 +2069,29 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { const char* msg1 = "not support column types"; - int16_t type = 0; - int16_t bytes = 0; - int32_t functionID = cvtFunc.execFuncId; - - if (functionID == TSDB_FUNC_SPREAD) { + int32_t f = cvtFunc.execFuncId; + if (f == TSDB_FUNC_SPREAD) { int32_t t1 = pSchema->type; - if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) { + if (IS_VAR_DATA_TYPE(t1) || t1 == TSDB_DATA_TYPE_BOOL) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return -1; - } else { - type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypes[type].bytes; } - } else { - type = pSchema->type; - bytes = pSchema->bytes; } - SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, getNewResColId(pCmd), bytes, false); + int16_t resType = 0; + int16_t resBytes = 0; + int32_t interBufSize = 0; + + getResultDataInfo(pSchema->type, pSchema->bytes, f, 0, &resType, &resBytes, &interBufSize, 0, false); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, f, pColIndex, resType, resBytes, getNewResColId(pCmd), interBufSize, false); tstrncpy(pExpr->base.aliasName, name, tListLen(pExpr->base.aliasName)); - if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) { + if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != f) { pExpr->base.colInfo.flag |= TSDB_COL_NULL; } // set reverse order scan data blocks for last query - if (functionID == TSDB_FUNC_LAST) { + if (f == TSDB_FUNC_LAST) { pExpr->base.numOfParams = 1; pExpr->base.param[0].i64 = TSDB_ORDER_DESC; pExpr->base.param[0].nType = TSDB_DATA_TYPE_INT; @@ -2108,7 +2104,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS // if it is not in the final result, do not add it SColumnList ids = createColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); if (finalResult) { - insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, pExpr->base.aliasName, pExpr); + insertResultField(pQueryInfo, resColIdx, &ids, pSchema->bytes, (int8_t)pSchema->type, pExpr->base.aliasName, pExpr); } else { tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); } @@ -2557,8 +2553,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col tVariant* pVariant = &pParamElem[1].pNode->value; - int8_t resultType = pSchema->type; - int16_t resultSize = pSchema->bytes; + int16_t resultType = pSchema->type; + int16_t resultSize = pSchema->bytes; + int32_t interResult = 0; char val[8] = {0}; @@ -2571,8 +2568,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } - resultSize = sizeof(double); - resultType = TSDB_DATA_TYPE_DOUBLE; + getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, &interResult, 0, false); /* * sql function transformation @@ -2582,7 +2578,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); colIndex += 1; // the first column is ts - pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); + pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), interResult, false); tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT, true); @@ -7784,8 +7780,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf const char* msg3 = "start(end) time of query range required or time range too large"; const char* msg4 = "interval query not supported, since the result of sub query not include valid timestamp column"; const char* msg5 = "only tag query not compatible with normal column filter"; - const char* msg6 = "not support stddev/percentile in outer query yet"; - const char* msg7 = "drivative requires timestamp column exists in subquery"; + const char* msg6 = "not support stddev/percentile/interp in the outer query yet"; + const char* msg7 = "derivative/twa/irate requires timestamp column exists in subquery"; int32_t code = TSDB_CODE_SUCCESS; @@ -7828,15 +7824,17 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, timeWindowQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } + // parse the window_state if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } + // todo NOT support yet for(int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int32_t f = pExpr->base.functionId; - if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT) { + if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT || f == TSDB_FUNC_INTERP) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -7851,9 +7849,17 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, 0); - if (tscNumOfExprs(pQueryInfo) > 1) { + int32_t numOfExprs = tscNumOfExprs(pQueryInfo); + if (numOfExprs == 1) { + SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); + int32_t f = pExpr->base.functionId; + if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + } + } else { SExprInfo* pExpr = tscExprGet(pQueryInfo, 1); - if (pExpr->base.functionId == TSDB_FUNC_DERIVATIVE && pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { + int32_t f = pExpr->base.functionId; + if ((f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) && pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 9cd1c5b033..1f1f9f3e60 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -285,7 +285,7 @@ enum OPERATOR_TYPE_E { OP_TagScan = 4, OP_TableBlockInfoScan= 5, OP_Aggregate = 6, - OP_Arithmetic = 7, + OP_Project = 7, OP_Groupby = 8, OP_Limit = 9, OP_SLimit = 10, @@ -413,13 +413,13 @@ typedef struct SAggOperatorInfo { uint32_t seed; } SAggOperatorInfo; -typedef struct SArithOperatorInfo { +typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; int32_t bufCapacity; uint32_t seed; SSDataBlock *existDataBlock; -} SArithOperatorInfo; +} SProjectOperatorInfo; typedef struct SLimitOperatorInfo { int64_t limit; @@ -513,7 +513,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 8efc4aad4c..676e5b6ce6 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -74,7 +74,6 @@ } while (0); void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} -void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {} void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 97a6cf807c..49ab0af680 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -184,7 +184,7 @@ static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr); static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); -static void destroyArithOperatorInfo(void* param, int32_t numOfOutput); +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); @@ -912,7 +912,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlo doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } } else { - if (/*pCtx[0].pInput == NULL && */pBlock->pDataBlock != NULL) { + if (pBlock->pDataBlock != NULL) { doSetInputDataBlock(pOperator, pCtx, pBlock, order); } else { doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); @@ -978,7 +978,7 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction } } -static void arithmeticApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t k = 0; k < numOfOutput; ++k) { @@ -1806,17 +1806,17 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf break; } - case OP_Arithmetic: { // TODO refactor to remove arith operator. + case OP_Project: { // TODO refactor to remove arith operator. SOperatorInfo* prev = pRuntimeEnv->proot; if (i == 0) { - pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); } } else { prev = pRuntimeEnv->proot; assert(pQueryAttr->pExpr2 != NULL); - pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); + pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); } break; } @@ -4578,8 +4578,8 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo; pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_Arithmetic) { - SArithOperatorInfo *pInfo = pDownstream->info; + } else if (pDownstream->operatorType == OP_Project) { + SProjectOperatorInfo *pInfo = pDownstream->info; pTableScanInfo->pCtx = pInfo->binfo.pCtx; pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; @@ -4934,23 +4934,23 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { return pInfo->pRes; } -static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { +static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; - SArithOperatorInfo* pArithInfo = pOperator->info; + SProjectOperatorInfo* pProjectInfo = pOperator->info; SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - SOptrBasicInfo *pInfo = &pArithInfo->binfo; + SOptrBasicInfo *pInfo = &pProjectInfo->binfo; SSDataBlock* pRes = pInfo->pRes; int32_t order = pRuntimeEnv->pQueryAttr->order.order; pRes->info.rows = 0; - if (pArithInfo->existDataBlock) { // TODO refactor + if (pProjectInfo->existDataBlock) { // TODO refactor STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - SSDataBlock* pBlock = pArithInfo->existDataBlock; - pArithInfo->existDataBlock = NULL; + SSDataBlock* pBlock = pProjectInfo->existDataBlock; + pProjectInfo->existDataBlock = NULL; *newgroup = true; // todo dynamic set tags @@ -4960,9 +4960,9 @@ static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pArithInfo->binfo, &pArithInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); - arithmeticApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); if (pTableQueryInfo != NULL) { updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); } @@ -4990,7 +4990,7 @@ static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { // Return result of the previous group in the firstly. if (*newgroup) { if (pRes->info.rows > 0) { - pArithInfo->existDataBlock = pBlock; + pProjectInfo->existDataBlock = pBlock; clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); return pInfo->pRes; } else { // init output buffer for a new group data @@ -5010,9 +5010,9 @@ static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pArithInfo->binfo, &pArithInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); - arithmeticApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); if (pTableQueryInfo != NULL) { updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); } @@ -5649,8 +5649,8 @@ static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { tfree(pInfo->prevData); } -static void destroyArithOperatorInfo(void* param, int32_t numOfOutput) { - SArithOperatorInfo* pInfo = (SArithOperatorInfo*) param; +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { + SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } @@ -5696,8 +5696,8 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SO return pOperator; } -SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SArithOperatorInfo* pInfo = calloc(1, sizeof(SArithOperatorInfo)); +SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); pInfo->seed = rand(); pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; @@ -5710,8 +5710,8 @@ SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MASTER_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "ArithmeticOperator"; - pOperator->operatorType = OP_Arithmetic; + pOperator->name = "ProjectOperator"; + pOperator->operatorType = OP_Project; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -5719,8 +5719,8 @@ SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doArithmeticOperation; - pOperator->cleanup = destroyArithOperatorInfo; + pOperator->exec = doProjectOperation; + pOperator->cleanup = destroyProjectOperatorInfo; appendUpstream(pOperator, upstream); return pOperator; diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index ee587a515d..214ef32cd4 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -565,7 +565,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } @@ -585,7 +585,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { } if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } else if (pQueryAttr->sw.gap > 0) { @@ -593,7 +593,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } else if (pQueryAttr->stateWindow) { @@ -601,7 +601,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } else if (pQueryAttr->simpleAgg) { @@ -619,7 +619,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { } if (pQueryAttr->pExpr2 != NULL && !pQueryAttr->stableQuery) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } else { // diff/add/multiply/subtract/division @@ -627,7 +627,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { op = OP_Filter; taosArrayPush(plan, &op); } else { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } @@ -665,7 +665,7 @@ SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr) { } if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index 8249d9197f..fd56a91dd6 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -180,20 +180,82 @@ if $data21 != 49.500000000 then endi #define TSDB_FUNC_APERCT 7 -#define TSDB_FUNC_LAST_ROW 10 #define TSDB_FUNC_TWA 14 #define TSDB_FUNC_LEASTSQR 15 -#define TSDB_FUNC_ARITHM 23 #define TSDB_FUNC_DIFF 24 #define TSDB_FUNC_INTERP 28 -#define TSDB_FUNC_RATE 29 #define TSDB_FUNC_IRATE 30 #define TSDB_FUNC_DERIVATIVE 32 sql_error select stddev(c1) from (select c1 from nest_tb0); sql_error select percentile(c1, 20) from (select * from nest_tb0); +sql_error select interp(c1) from (select * from nest_tb0); +sql_error select derivative(val, 1s, 0) from (select c1 val from nest_tb0); +sql_error select twa(c1) from (select c1 from nest_tb0); +sql_error select irate(c1) from (select c1 from nest_tb0); +sql_error select diff(c1), twa(c1) from (select * from nest_tb0); +sql_error select irate(c1), interp(c1), twa(c1) from (select * from nest_tb0); + +sql select apercentile(c1, 50) from (select * from nest_tb0) interval(1d) +sql select twa(c1) from (select * from nest_tb0); +sql select leastsquares(c1, 1, 1) from (select * from nest_tb0); +sql select irate(c1) from (select * from nest_tb0); sql select avg(c1),sum(c2), max(c3), min(c4), count(*), first(c7), last(c7),spread(c6) from (select * from nest_tb0) interval(1d); +if $rows != 7 then + return -1 +endi + +if $data00 != @20-09-15 00:00:00.000@ then + return -1 +endi + +if $data01 != 48.666666667 then + print expect 48.666666667, actual: $data01 + return -1 +endi + +if $data02 != 70080.000000000 then + return -1 +endi + +if $data03 != 99 then + return -1 +endi + +if $data04 != 0 then + return -1 +endi + +if $data05 != 1440 then + return -1 +endi + +if $data06 != 0 then + print $data06 + return -1 +endi + +if $data07 != 1 then + return -1 +endi + +if $data08 != 99.000000000 then + print expect 99.000000000, actual: $data08 + return -1 +endi + +if $data10 != @20-09-16 00:00:00.000@ then + return -1 +endi + +if $data11 != 49.777777778 then + return -1 +endi + +if $data12 != 71680.000000000 then + return -1 +endi sql select top(x, 20) from (select c1 x from nest_tb0); @@ -207,6 +269,9 @@ print ===================> group by + having +print =========================> ascending order/descending order + + print =========================> nest query join @@ -273,7 +338,6 @@ if $data03 != @20-09-15 00:00:00.000@ then return -1 endi -sql_error select derivative(val, 1s, 0) from (select c1 val from nest_tb0); sql select diff(val) from (select c1 val from nest_tb0); if $rows != 9999 then return -1 From e143c34c7b60335e13b03cb318c9fe3b528b1db1 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 24 Jun 2021 17:59:13 +0800 Subject: [PATCH 082/115] comment "nestedQuery/queryInterval.py" --- tests/pytest/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 37fb3acdfa..c66ccc5477 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -235,7 +235,7 @@ python3 ./test.py -f query/queryTscomputWithNow.py python3 ./test.py -f query/computeErrorinWhere.py python3 ./test.py -f query/queryTsisNull.py python3 ./test.py -f query/subqueryFilter.py -python3 ./test.py -f query/nestedQuery/queryInterval.py +# python3 ./test.py -f query/nestedQuery/queryInterval.py python3 ./test.py -f query/queryStateWindow.py From eb0f1a39e7372e736cd5521247e0f27c0826a78b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 24 Jun 2021 20:49:01 +0800 Subject: [PATCH 083/115] Hotfix/sangshuduo/td 4892 taosdemo sub fetch for develop (#6609) * [TD-4892]: taosdemo subscribe fetch result. for develop branch * fix stbname length. * restrict prefix length. * submit empty * fix minor code. --- src/kit/taosdemo/taosdemo.c | 127 ++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 71 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index a6ba7e9493..751f6bc07a 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -79,10 +79,9 @@ enum TEST_MODE { #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) -#define COND_BUF_LEN BUFFER_SIZE - 30 +#define COND_BUF_LEN (BUFFER_SIZE - 30) #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 -#define MAX_DB_NAME_SIZE 64 #define MAX_HOSTNAME_SIZE 64 #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*1024)+20 // max record len: 16*1024, timestamp string and ,('') need extra space @@ -90,7 +89,7 @@ enum TEST_MODE { #define OPT_ABORT 1 /* –abort */ #define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 -#define MAX_FILE_NAME_LEN 128 +#define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. #define MAX_SAMPLES_ONCE_FROM_FILE 10000 #define MAX_NUM_DATATYPE 10 @@ -195,13 +194,6 @@ enum _describe_table_index { TSDB_MAX_DESCRIBE_METRIC }; -typedef struct { - char field[TSDB_COL_NAME_LEN + 1]; - char type[16]; - int length; - char note[128]; -} SColDes; - /* Used by main to communicate with parse_opt. */ static char *g_dupstr = NULL; @@ -247,16 +239,16 @@ typedef struct SArguments_S { } SArguments; typedef struct SColumn_S { - char field[TSDB_COL_NAME_LEN + 1]; - char dataType[MAX_TB_NAME_SIZE]; + char field[TSDB_COL_NAME_LEN]; + char dataType[16]; uint32_t dataLen; char note[128]; } StrColumn; typedef struct SSuperTable_S { - char sTblName[MAX_TB_NAME_SIZE+1]; - char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample - char childTblPrefix[MAX_TB_NAME_SIZE]; + char sTblName[TSDB_TABLE_NAME_LEN]; + char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest uint16_t childTblExists; int64_t childTblCount; @@ -277,8 +269,8 @@ typedef struct SSuperTable_S { int64_t timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json - char sampleFile[MAX_FILE_NAME_LEN+1]; - char tagsFile[MAX_FILE_NAME_LEN+1]; + char sampleFile[MAX_FILE_NAME_LEN]; + char tagsFile[MAX_FILE_NAME_LEN]; uint32_t columnCount; StrColumn columns[MAX_COLUMN_COUNT]; @@ -305,7 +297,7 @@ typedef struct SSuperTable_S { } SSuperTable; typedef struct { - char name[TSDB_DB_NAME_LEN + 1]; + char name[TSDB_DB_NAME_LEN]; char create_time[32]; int64_t ntables; int32_t vgroups; @@ -341,11 +333,11 @@ typedef struct SDbCfg_S { int cache; int blocks; int quorum; - char precision[MAX_TB_NAME_SIZE]; + char precision[8]; } SDbCfg; typedef struct SDataBase_S { - char dbName[MAX_DB_NAME_SIZE]; + char dbName[TSDB_DB_NAME_LEN]; bool drop; // 0: use exists, 1: if exists, drop then new create SDbCfg dbCfg; uint64_t superTblCount; @@ -353,14 +345,14 @@ typedef struct SDataBase_S { } SDataBase; typedef struct SDbs_S { - char cfgDir[MAX_FILE_NAME_LEN+1]; + char cfgDir[MAX_FILE_NAME_LEN]; char host[MAX_HOSTNAME_SIZE]; struct sockaddr_in serv_addr; uint16_t port; char user[MAX_USERNAME_SIZE]; char password[MAX_PASSWORD_SIZE]; - char resultFile[MAX_FILE_NAME_LEN+1]; + char resultFile[MAX_FILE_NAME_LEN]; bool use_metric; bool insert_only; bool do_aggreFunc; @@ -387,7 +379,7 @@ typedef struct SpecifiedQueryInfo_S { bool subscribeRestart; int subscribeKeepProgress; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; int resubAfterConsume[MAX_QUERY_SQL_COUNT]; int endAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; @@ -398,7 +390,7 @@ typedef struct SpecifiedQueryInfo_S { } SpecifiedQueryInfo; typedef struct SuperQueryInfo_S { - char sTblName[MAX_TB_NAME_SIZE+1]; + char sTblName[TSDB_TABLE_NAME_LEN]; uint64_t queryInterval; // 0: unlimit > 0 loop/s uint32_t threadCnt; uint32_t asyncMode; // 0: sync, 1: async @@ -407,10 +399,10 @@ typedef struct SuperQueryInfo_S { int subscribeKeepProgress; uint64_t queryTimes; int64_t childTblCount; - char childTblPrefix[MAX_TB_NAME_SIZE]; + char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq int sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; int resubAfterConsume; int endAfterConsume; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; @@ -420,13 +412,13 @@ typedef struct SuperQueryInfo_S { } SuperQueryInfo; typedef struct SQueryMetaInfo_S { - char cfgDir[MAX_FILE_NAME_LEN+1]; + char cfgDir[MAX_FILE_NAME_LEN]; char host[MAX_HOSTNAME_SIZE]; uint16_t port; struct sockaddr_in serv_addr; char user[MAX_USERNAME_SIZE]; char password[MAX_PASSWORD_SIZE]; - char dbName[MAX_DB_NAME_SIZE+1]; + char dbName[TSDB_DB_NAME_LEN]; char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest SpecifiedQueryInfo specifiedQueryInfo; @@ -438,11 +430,11 @@ typedef struct SThreadInfo_S { TAOS * taos; TAOS_STMT *stmt; int threadID; - char db_name[MAX_DB_NAME_SIZE+1]; + char db_name[TSDB_DB_NAME_LEN]; uint32_t time_precision; char filePath[4096]; FILE *fp; - char tb_prefix[MAX_TB_NAME_SIZE]; + char tb_prefix[TSDB_TABLE_NAME_LEN]; uint64_t start_table_from; uint64_t end_table_to; int64_t ntables; @@ -608,7 +600,7 @@ SArguments g_args = { 1, // query_times 0, // interlace_rows; 30000, // num_of_RPR - (1024*1024), // max_sql_len + (1024*1024), // max_sql_len 10000, // num_of_tables 10000, // num_of_DPT 0, // abort @@ -3035,7 +3027,7 @@ static int startMultiThreadCreateChildTable( for (int64_t i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); pThreadInfo->superTblInfo = superTblInfo; verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); pThreadInfo->taos = taos_connect( @@ -3326,7 +3318,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( goto PARSE_OVER; } //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); - tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); cJSON* dataLen = cJSON_GetObjectItem(column, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3341,7 +3333,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( for (int n = 0; n < count; ++n) { tstrncpy(superTbls->columns[index].dataType, - columnCase.dataType, MAX_TB_NAME_SIZE); + columnCase.dataType, strlen(columnCase.dataType) + 1); superTbls->columns[index].dataLen = columnCase.dataLen; index++; } @@ -3397,7 +3389,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( __func__, __LINE__); goto PARSE_OVER; } - tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3412,7 +3404,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( for (int n = 0; n < count; ++n) { tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, - MAX_TB_NAME_SIZE); + strlen(columnCase.dataType) + 1); superTbls->tags[index].dataLen = columnCase.dataLen; index++; } @@ -3635,7 +3627,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, db name not found\n"); goto PARSE_OVER; } - tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, TSDB_DB_NAME_LEN); cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { @@ -3656,10 +3648,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (precision && precision->type == cJSON_String && precision->valuestring != NULL) { tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, - MAX_DB_NAME_SIZE); + 8); } else if (!precision) { - //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); - memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE); + memset(g_Dbs.db[i].dbCfg.precision, 0, 8); } else { printf("ERROR: failed to read json, precision not found\n"); goto PARSE_OVER; @@ -3836,7 +3827,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, - MAX_TB_NAME_SIZE); + TSDB_TABLE_NAME_LEN); cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { @@ -3844,7 +3835,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, - MAX_DB_NAME_SIZE); + TSDB_TABLE_NAME_LEN - 20); cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); if (autoCreateTbl @@ -3912,9 +3903,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (dataSource && dataSource->type == cJSON_String && dataSource->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, - dataSource->valuestring, MAX_DB_NAME_SIZE); + dataSource->valuestring, TSDB_DB_NAME_LEN); } else if (!dataSource) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN); } else { errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__); @@ -3972,10 +3963,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - ts->valuestring, MAX_DB_NAME_SIZE); + ts->valuestring, TSDB_DB_NAME_LEN); } else if (!ts) { tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, - "now", MAX_DB_NAME_SIZE); + "now", TSDB_DB_NAME_LEN); } else { printf("ERROR: failed to read json, start_timestamp not found\n"); goto PARSE_OVER; @@ -3995,9 +3986,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, - sampleFormat->valuestring, MAX_DB_NAME_SIZE); + sampleFormat->valuestring, TSDB_DB_NAME_LEN); } else if (!sampleFormat) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN); } else { printf("ERROR: failed to read json, sample_format not found\n"); goto PARSE_OVER; @@ -4242,7 +4233,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { - tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); + tstrncpy(g_queryInfo.dbName, dbs->valuestring, TSDB_DB_NAME_LEN); } else if (!dbs) { printf("ERROR: failed to read json, databases not found\n"); goto PARSE_OVER; @@ -4492,7 +4483,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, - MAX_TB_NAME_SIZE); + TSDB_TABLE_NAME_LEN); } else { errorPrint("%s() LN%d, failed to read json, super table name input error\n", __func__, __LINE__); @@ -6417,7 +6408,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, for (int i = 0; i < threads; i++) { threadInfo *pThreadInfo = infos + i; pThreadInfo->threadID = i; - tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN); pThreadInfo->time_precision = timePrec; pThreadInfo->superTblInfo = superTblInfo; @@ -6861,7 +6852,7 @@ static void *specifiedTableQuery(void *sarg) { } } - char sqlStr[MAX_DB_NAME_SIZE + 5]; + char sqlStr[TSDB_DB_NAME_LEN + 5]; sprintf(sqlStr, "use %s", g_queryInfo.dbName); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); @@ -7337,12 +7328,6 @@ static void *superSubscribe(void *sarg) { performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st)); if (res) { - if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->filePath, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - fetchResult(res, pThreadInfo); - } if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], @@ -7449,10 +7434,10 @@ static void *specifiedSubscribe(void *sarg) { sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - fetchResult( - g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], - pThreadInfo); } + fetchResult( + g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], + pThreadInfo); g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++; if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1) @@ -7689,9 +7674,9 @@ static void setParaFromArg(){ g_Dbs.dbCount = 1; g_Dbs.db[0].drop = true; - tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN); g_Dbs.db[0].dbCfg.replica = g_args.replica; - tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8); tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); @@ -7713,7 +7698,7 @@ static void setParaFromArg(){ if (g_args.use_metric) { g_Dbs.db[0].superTblCount = 1; - tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN); g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; @@ -7724,7 +7709,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, - g_args.tb_prefix, MAX_TB_NAME_SIZE); + g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].iface = g_args.iface; tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, @@ -7741,7 +7726,7 @@ static void setParaFromArg(){ } tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - data_type[i], MAX_TB_NAME_SIZE); + data_type[i], strlen(data_type[i]) + 1); g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].columnCount++; } @@ -7752,18 +7737,18 @@ static void setParaFromArg(){ for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - "INT", MAX_TB_NAME_SIZE); + "INT", strlen("INT") + 1); g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; g_Dbs.db[0].superTbls[0].columnCount++; } } tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, - "INT", MAX_TB_NAME_SIZE); + "INT", strlen("INT") + 1); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, - "BINARY", MAX_TB_NAME_SIZE); + "BINARY", strlen("BINARY") + 1); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { @@ -7899,11 +7884,11 @@ static void queryResult() { pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; tstrncpy(pThreadInfo->tb_prefix, - g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].childTblPrefix, TSDB_TABLE_NAME_LEN - 20); } else { pThreadInfo->ntables = g_args.num_of_tables; pThreadInfo->end_table_to = g_args.num_of_tables -1; - tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, TSDB_TABLE_NAME_LEN); } pThreadInfo->taos = taos_connect( From 8fc2b3d79d6e8e07871bef2380470f23b504c4ff Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 24 Jun 2021 22:04:29 +0800 Subject: [PATCH 084/115] Hotfix/sangshuduo/td 4823 taosdemo gettablename for develop (#6605) * [TD-4823]: taosdemo getTableName return empty. * fix typo. * check table name is empty in early stage. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 751f6bc07a..38f0ecb219 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2493,6 +2493,13 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, char* pTblName = childTblName; while((row = taos_fetch_row(res)) != NULL) { int32_t* len = taos_fetch_lengths(res); + + if (0 == strlen((char *)row[0])) { + errorPrint("%s() LN%d, No.%"PRId64" table return empty name\n", + __func__, __LINE__, count); + exit(-1); + } + tstrncpy(pTblName, (char *)row[0], len[0]+1); //printf("==== sub table name: %s\n", pTblName); count++; @@ -6293,16 +6300,6 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } - // read sample data from file first - if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, - "sample", strlen("sample")))) { - if (0 != prepareSampleDataForSTable(superTblInfo)) { - errorPrint("%s() LN%d, prepare sample data for stable failed!\n", - __func__, __LINE__); - exit(-1); - } - } - TAOS* taos0 = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); From 53f12701f7cc55df65ea038cc0a67ad1149907cf Mon Sep 17 00:00:00 2001 From: zhaoyanggh Date: Fri, 25 Jun 2021 09:07:12 +0800 Subject: [PATCH 085/115] add test case to verify taosdump with binary and nchar data --- tests/pytest/tools/taosdumpTest.py | 2 + tests/pytest/tools/taosdumpTest2.py | 74 +++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/pytest/tools/taosdumpTest2.py diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index 534a477b34..8746f4ecdf 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -45,7 +45,9 @@ class TDTestCase: for i in range(100): sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) tdSql.execute(sql) + + os.system("rm /tmp/*.sql") os.system("taosdump --databases db -o /tmp") tdSql.execute("drop database db") diff --git a/tests/pytest/tools/taosdumpTest2.py b/tests/pytest/tools/taosdumpTest2.py new file mode 100644 index 0000000000..51a73555a8 --- /dev/null +++ b/tests/pytest/tools/taosdumpTest2.py @@ -0,0 +1,74 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1601481600000 + self.numberOfTables = 1 + self.numberOfRecords = 15000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, c1 timestamp, c2 int, c3 bigint, c4 float, c5 double, c6 binary(8), c7 smallint, c8 tinyint, c9 bool, c10 nchar(8)) tags(t1 int)") + tdSql.execute("create table t1 using st tags(0)") + currts = self.ts + finish = 0 + while(finish < self.numberOfRecords): + sql = "insert into t1 values" + for i in range(finish, self.numberOfRecords): + sql += "(%d, 1019774612, 29931, 1442173978, 165092.468750, 1128.643179, 'MOCq1pTu', 18405, 82, 0, 'g0A6S0Fu')" % (currts + i) + finish = i + 1 + if (1048576 - len(sql)) < 16384: + break + tdSql.execute(sql) + + os.system("rm /tmp/*.sql") + os.system("taosdump --databases db -o /tmp -B 32766 -L 1048576") + + tdSql.execute("drop database db") + tdSql.query("show databases") + tdSql.checkRows(0) + + os.system("taosdump -i /tmp") + + tdSql.query("show databases") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'db') + + tdSql.execute("use db") + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'st') + + tdSql.query("select count(*) from t1") + tdSql.checkData(0, 0, self.numberOfRecords) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 795bd8c07676bac3d0a163b862be7ed2d2f4c1e6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 25 Jun 2021 10:39:17 +0800 Subject: [PATCH 086/115] [TD-4098] unsigned date type not found --- src/client/src/tscSQLParser.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e94f77d254..5107884422 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -165,6 +165,7 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, } else { tbufWriteUint32(&bw, colType); } + tbufWriteInt32(&bw, (int32_t)(pList->size)); for (int32_t i = 0; i < (int32_t)pList->size; i++) { @@ -181,10 +182,11 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, } tbufWriteInt64(&bw, var->i64); } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) { - if (IS_SIGNED_NUMERIC_TYPE(var->nType) && IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { + if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { + tbufWriteUint64(&bw, var->u64); + } else { break; - } - tbufWriteUint64(&bw, var->u64); + } } else if (colType == TSDB_DATA_TYPE_DOUBLE || colType == TSDB_DATA_TYPE_FLOAT) { if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { tbufWriteDouble(&bw, (double)(var->i64)); From 757642507f751a080a06d93c59ded7cf396e6eb4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 25 Jun 2021 12:11:17 +0800 Subject: [PATCH 087/115] Update README.md (#6615) --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 0e1adcd97c..89e35f6e63 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,12 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` +You can use Jemalloc as memory allocator instead of glibc: +``` +apt install autoconf +cmake .. -DJEMALLOC_ENABLED=true +``` + TDengine build script can detect the host machine's architecture on X86-64, X86, arm64, arm32 and mips64 platform. You can also specify CPUTYPE option like aarch64 or aarch32 too if the detection result is not correct: From 2b879fb158745ddac6b44a1fa97b1df57f906276 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Fri, 25 Jun 2021 13:49:07 +0800 Subject: [PATCH 088/115] [TD4824] --- tests/pytest/insert/in_function.py | 580 ++++++++++++++++++++++++----- 1 file changed, 484 insertions(+), 96 deletions(-) diff --git a/tests/pytest/insert/in_function.py b/tests/pytest/insert/in_function.py index 263c8a78aa..3f2e1a03ca 100644 --- a/tests/pytest/insert/in_function.py +++ b/tests/pytest/insert/in_function.py @@ -18,7 +18,6 @@ from util.log import * from util.cases import * from util.sql import * - class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) @@ -27,6 +26,7 @@ class TDTestCase: def run(self): tdSql.prepare() # test case for https://jira.taosdata.com:18080/browse/TD-4568 + # test case for https://jira.taosdata.com:18080/browse/TD-4824 tdLog.info("=============== step1,check bool and tinyint data type") @@ -137,8 +137,28 @@ class TDTestCase: tdSql.checkData(0,1,'True') tdSql.checkData(0,2,'0') + tdLog.info("=============== step1.3,multiple column and multiple tag check in function") + cmd1 = '''select * from in_stable_1 + where in_bool in (true,false) and in_tinyint in (0,127,-127) + and tin_bool in (true,false) and tin_tinyint in (0,127,-127) + order by ts desc ;''' + tdLog.info(cmd1) + tdSql.query(cmd1) + tdSql.checkData(0,1,'True') + tdSql.checkData(0,2,'0') + tdSql.checkData(0,3,'False') + tdSql.checkData(0,4,'0') + tdSql.checkData(1,1,'False') + tdSql.checkData(1,2,'127') + tdSql.checkData(1,3,'False') + tdSql.checkData(1,4,'-127') + tdSql.checkData(2,1,'True') + tdSql.checkData(2,2,'-127') + tdSql.checkData(2,3,'True') + tdSql.checkData(2,4,'127') - tdLog.info("=============== step1.3,drop normal table && create table") + + tdLog.info("=============== step1.4,drop normal table && create table") cmd1 = 'drop table if exists normal_in_bool_tinyint_1 ;' cmd2 = 'create table normal_in_bool_tinyint_1 (ts timestamp,in_bool bool,in_tinyint tinyint) ; ' tdLog.info(cmd1) @@ -147,7 +167,7 @@ class TDTestCase: tdSql.execute(cmd2) - tdLog.info("=============== step1.4,insert normal table right data and check in function") + tdLog.info("=============== step1.5,insert normal table right data and check in function") cmd1 = 'insert into normal_in_bool_tinyint_1 values(now,\'true\',\'-127\') ;' tdLog.info(cmd1) tdSql.execute(cmd1) @@ -175,6 +195,17 @@ class TDTestCase: tdSql.checkData(0,1,'True') tdSql.checkData(0,2,'0') + cmd4 = '''select * from normal_in_bool_tinyint_1 + where in_bool in (true,false) and in_tinyint in (0,127,-127) + order by ts desc ;''' + tdLog.info(cmd4) + tdSql.query(cmd4) + tdSql.checkData(0,1,'True') + tdSql.checkData(0,2,'0') + tdSql.checkData(1,1,'False') + tdSql.checkData(1,2,'127') + tdSql.checkData(2,1,'True') + tdSql.checkData(2,2,'-127') tdLog.info("=============== step2,check int、smallint and bigint data type") @@ -378,10 +409,39 @@ class TDTestCase: tdSql.query('select * from in_int_smallint_bigint_3 where in_big in (-9223372036854775807) order by ts desc') tdSql.checkData(0,1,'0') tdSql.checkData(0,2,'32767') - tdSql.checkData(0,3,'-9223372036854775807') + tdSql.checkData(0,3,'-9223372036854775807') - tdLog.info("=============== step2.3,drop normal table && create table") + tdLog.info("=============== step2.3,multiple column and multiple tag check in function") + cmd1 = '''select * from in_stable_2 + where in_int in (0,2147483647,-2147483647) and in_small in (0,32767,-32767) + and in_big in (0,9223372036854775807,-9223372036854775807) + and tin_int in (0,2147483647,-2147483647) and tin_small in (0,32767,-32767) + and tin_big in (0,9223372036854775807,-9223372036854775807) + order by ts desc ;''' + tdLog.info(cmd1) + tdSql.query(cmd1) + tdSql.checkData(0,1,'0') + tdSql.checkData(0,2,'32767') + tdSql.checkData(0,3,'-9223372036854775807') + tdSql.checkData(0,4,'0') + tdSql.checkData(0,5,'32767') + tdSql.checkData(0,6,'-9223372036854775807') + tdSql.checkData(1,1,'-2147483647') + tdSql.checkData(1,2,'0') + tdSql.checkData(1,3,'9223372036854775807') + tdSql.checkData(1,4,'-2147483647') + tdSql.checkData(1,5,'0') + tdSql.checkData(1,6,'9223372036854775807') + tdSql.checkData(2,1,'2147483647') + tdSql.checkData(2,2,'-32767') + tdSql.checkData(2,3,'0') + tdSql.checkData(2,4,'2147483647') + tdSql.checkData(2,5,'-32767') + tdSql.checkData(2,6,'0') + + + tdLog.info("=============== step2.4,drop normal table && create table") cmd1 = 'drop table if exists normal_int_smallint_bigint_1 ;' cmd2 = 'create table normal_int_smallint_bigint_1 (ts timestamp,in_int int,in_small smallint , in_big bigint) ; ' tdLog.info(cmd1) @@ -390,7 +450,7 @@ class TDTestCase: tdSql.execute(cmd2) - tdLog.info("=============== step2.4,insert normal table right data and check in function") + tdLog.info("=============== step2.5,insert normal table right data and check in function") cmd1 = 'insert into normal_int_smallint_bigint_1 values(now,\'2147483647\',\'-32767\',\'0\') ;' tdLog.info(cmd1) tdSql.execute(cmd1) @@ -437,7 +497,23 @@ class TDTestCase: tdSql.query('select * from normal_int_smallint_bigint_1 where in_big in (-9223372036854775807) order by ts desc') tdSql.checkData(0,1,'0') tdSql.checkData(0,2,'32767') - tdSql.checkData(0,3,'-9223372036854775807') + tdSql.checkData(0,3,'-9223372036854775807') + + cmd4 = '''select * from normal_int_smallint_bigint_1 + where in_int in (0,2147483647,-2147483647) and in_small in (0,32767,-32767) + and in_big in (0,9223372036854775807,-9223372036854775807) + order by ts desc ;''' + tdLog.info(cmd4) + tdSql.query(cmd4) + tdSql.checkData(0,1,'0') + tdSql.checkData(0,2,'32767') + tdSql.checkData(0,3,'-9223372036854775807') + tdSql.checkData(1,1,'-2147483647') + tdSql.checkData(1,2,'0') + tdSql.checkData(1,3,'9223372036854775807') + tdSql.checkData(2,1,'2147483647') + tdSql.checkData(2,2,'-32767') + tdSql.checkData(2,3,'0') tdLog.info("=============== step3,check binary and nchar data type") @@ -560,7 +636,30 @@ class TDTestCase: tdSql.checkData(0,2,'北京涛思数据科技有限公司') - tdLog.info("=============== step3.3,drop normal table && create table") + tdLog.info("=============== step3.3,multiple column and multiple tag check in function") + cmd1 = '''select * from in_stable_3 + where in_binary in (\'0\',\'TDengine\',\'TAOS\') + and in_nchar in (\'0\',\'北京涛思数据科技有限公司\',\'涛思数据TAOSdata\') + and tin_binary in (\'0\',\'TDengine\',\'taosdataTDengine\') + and tin_nchar in (\'0\',\'北京涛思数据科技有限公司\',\'北京涛思数据科技有限公司TDengine\') + order by ts desc ;''' + tdLog.info(cmd1) + tdSql.query(cmd1) + tdSql.checkData(0,1,'TDengine') + tdSql.checkData(0,2,'北京涛思数据科技有限公司') + tdSql.checkData(0,3,'taosdataTDengine') + tdSql.checkData(0,4,'北京涛思数据科技有限公司TDengine') + tdSql.checkData(1,1,'TAOS') + tdSql.checkData(1,2,'涛思数据TAOSdata') + tdSql.checkData(1,3,'TDengine') + tdSql.checkData(1,4,'北京涛思数据科技有限公司') + tdSql.checkData(2,1,'0') + tdSql.checkData(2,2,'0') + tdSql.checkData(2,3,'0') + tdSql.checkData(2,4,'0') + + + tdLog.info("=============== step3.4,drop normal table && create table") cmd1 = 'drop table if exists normal_in_binary_nchar_1 ;' cmd2 = 'create table normal_in_binary_nchar_1 (ts timestamp,in_binary binary(8),in_nchar nchar(12)) ; ' tdLog.info(cmd1) @@ -569,7 +668,7 @@ class TDTestCase: tdSql.execute(cmd2) - tdLog.info("=============== step3.4,insert normal table right data and check in function") + tdLog.info("=============== step3.5,insert normal table right data and check in function") cmd1 = 'insert into normal_in_binary_nchar_1 values(now,\'0\',\'0\') ;' tdLog.info(cmd1) tdSql.execute(cmd1) @@ -598,124 +697,413 @@ class TDTestCase: tdSql.checkData(0,2,'北京涛思数据科技有限公司') tdSql.query('select * from normal_in_binary_nchar_1 where in_nchar in (\'北京涛思数据科技有限公司\') order by ts desc') tdSql.checkData(0,1,'TDengine') - tdSql.checkData(0,2,'北京涛思数据科技有限公司') + tdSql.checkData(0,2,'北京涛思数据科技有限公司') - tdLog.info("=============== step4,check float and double data type,not support") + cmd4 = '''select * from normal_in_binary_nchar_1 + where in_binary in (\'0\',\'TDengine\',\'TAOS\') + and in_nchar in (\'0\',\'北京涛思数据科技有限公司\',\'涛思数据TAOSdata\') + order by ts desc ;''' + tdLog.info(cmd4) + tdSql.query(cmd4) + tdSql.checkData(0,1,'TDengine') + tdSql.checkData(0,2,'北京涛思数据科技有限公司') + tdSql.checkData(1,1,'TAOS') + tdSql.checkData(1,2,'涛思数据TAOSdata') + tdSql.checkData(2,1,'0') + tdSql.checkData(2,2,'0') + + + tdLog.info("=============== step4,check float and double data type") tdLog.info("=============== step4.1,drop table && create table") - cmd1 = 'drop table if exists in_float_double_1 ;' + cmd1 = 'drop table if exists in_ts_float_double_1 ;' + cmd2 = 'drop table if exists in_ts_float_double_2 ;' + cmd3 = 'drop table if exists in_ts_float_double_3 ;' cmd10 = 'drop table if exists in_stable_4 ;' - cmd11 = 'create stable in_stable_4(ts timestamp,in_float float,in_double double) tags (tin_float float,tin_double double) ;' - cmd12 = 'create table in_float_double_1 using in_stable_4 tags(\'666\',\'88888\') ; ' + cmd11 = 'create stable in_stable_4(ts timestamp,in_ts timestamp,in_float float,in_double double) tags (tin_ts timestamp,tin_float float,tin_double double) ;' + cmd12 = 'create table in_ts_float_double_1 using in_stable_4 tags(\'0\',\'0\',\'0\') ; ' + cmd13 = 'create table in_ts_float_double_2 using in_stable_4 tags(\'2020-01-01 08:00:00.001\',\'666\',\'-88888\') ; ' + cmd14 = 'create table in_ts_float_double_3 using in_stable_4 tags(\'2021-01-01 08:00:00.001\',\'-888.00000\',\'66666.000000000\') ; ' tdLog.info(cmd1) tdSql.execute(cmd1) + tdLog.info(cmd2) + tdSql.execute(cmd2) + tdLog.info(cmd3) + tdSql.execute(cmd3) tdLog.info(cmd10) tdSql.execute(cmd10) tdLog.info(cmd11) tdSql.execute(cmd11) tdLog.info(cmd12) tdSql.execute(cmd12) + tdLog.info(cmd13) + tdSql.execute(cmd13) + tdLog.info(cmd14) + tdSql.execute(cmd14) tdLog.info("=============== step4.2,insert stable right data and check in function") - cmd1 = 'insert into in_float_double_1 values(now,\'888\',\'66666\') ;' + cmd1 = 'insert into in_ts_float_double_1 values(now,\'0\',\'0\',\'0\') ;' tdLog.info(cmd1) - tdSql.execute(cmd1) + tdSql.execute(cmd1) + + tdSql.query('select * from in_stable_4 where in_ts in (\'0\') order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,'1970-01-01 08:00:00.000') + tdSql.checkData(0,5,0.00000) + tdSql.checkData(0,6,0.000000000) + tdSql.query('select * from in_stable_4 where in_ts in (\'1970-01-01 08:00:00.000\') order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,'1970-01-01 08:00:00.000') + tdSql.checkData(0,5,0.00000) + tdSql.checkData(0,6,0.000000000) + tdSql.query('select * from in_stable_4 where in_float in (0.00000) order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,'1970-01-01 08:00:00.000') + tdSql.checkData(0,5,0.00000) + tdSql.checkData(0,6,0.000000000) + tdSql.query('select * from in_stable_4 where in_double in (0.000000000) order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,'1970-01-01 08:00:00.000') + tdSql.checkData(0,5,0.00000) + tdSql.checkData(0,6,0.000000000) + + tdSql.query('select * from in_stable_4 where tin_ts in (\'0\') order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,'1970-01-01 08:00:00.000') + tdSql.checkData(0,5,0.00000) + tdSql.checkData(0,6,0.000000000) + tdSql.query('select * from in_stable_4 where tin_ts in (\'1970-01-01 08:00:00.000\') order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,'1970-01-01 08:00:00.000') + tdSql.checkData(0,5,0.00000) + tdSql.checkData(0,6,0.000000000) + tdSql.query('select * from in_stable_4 where tin_float in (0.00000) order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,'1970-01-01 08:00:00.000') + tdSql.checkData(0,5,0.00000) + tdSql.checkData(0,6,0.000000000) + tdSql.query('select * from in_stable_4 where tin_double in (0.000000000) order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,'1970-01-01 08:00:00.000') + tdSql.checkData(0,5,0.00000) + tdSql.checkData(0,6,0.000000000) + + tdSql.query('select * from in_ts_float_double_1 where in_ts in (\'0\') order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.query('select * from in_ts_float_double_1 where in_ts in (\'1970-01-01 08:00:00.000\') order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.query('select * from in_ts_float_double_1 where in_float in (0.00000) order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.query('select * from in_ts_float_double_1 where in_double in (0.000000000) order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) - cmd2 = 'select * from in_stable_4 where in_float in (\'888\');' + cmd2 = 'insert into in_ts_float_double_2 values(now,\'2020-01-01 08:00:00.001\',\'666\',\'-88888\') ;' tdLog.info(cmd2) - tdSql.error(cmd2) - try: - tdSql.execute(cmd2) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") + tdSql.execute(cmd2) + + tdSql.query('select * from in_stable_4 where in_ts in (\'2020-01-01 08:00:00.001\') order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001000') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.checkData(0,4,'2020-01-01 08:00:00.001') + tdSql.checkData(0,5,666.00000) + tdSql.checkData(0,6,-88888.000000000) + tdSql.query('select * from in_stable_4 where in_ts in (\'1577836800001\') order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001000') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.checkData(0,4,'2020-01-01 08:00:00.001') + tdSql.checkData(0,5,666.00000) + tdSql.checkData(0,6,-88888.000000000) + tdSql.query('select * from in_stable_4 where in_float in (666.00000) order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001000') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.checkData(0,4,'2020-01-01 08:00:00.001') + tdSql.checkData(0,5,666.00000) + tdSql.checkData(0,6,-88888.000000000) + tdSql.query('select * from in_stable_4 where in_double in (-88888.000000000) order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001000') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.checkData(0,4,'2020-01-01 08:00:00.001') + tdSql.checkData(0,5,666.00000) + tdSql.checkData(0,6,-88888.000000000) + + tdSql.query('select * from in_stable_4 where tin_ts in (\'2020-01-01 08:00:00.001000\') order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001000') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.checkData(0,4,'2020-01-01 08:00:00.001') + tdSql.checkData(0,5,666.00000) + tdSql.checkData(0,6,-88888.000000000) + tdSql.query('select * from in_stable_4 where tin_ts in (\'1577836800001\') order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001000') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.checkData(0,4,'2020-01-01 08:00:00.001') + tdSql.checkData(0,5,666.00000) + tdSql.checkData(0,6,-88888.000000000) + tdSql.query('select * from in_stable_4 where tin_float in (666.00000) order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001000') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.checkData(0,4,'2020-01-01 08:00:00.001') + tdSql.checkData(0,5,666.00000) + tdSql.checkData(0,6,-88888.000000000) + tdSql.query('select * from in_stable_4 where tin_double in (-88888.000000000) order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001000') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.checkData(0,4,'2020-01-01 08:00:00.001') + tdSql.checkData(0,5,666.00000) + tdSql.checkData(0,6,-88888.000000000) + + tdSql.query('select * from in_ts_float_double_2 where in_ts in (\'1577836800001\') order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.query('select * from in_ts_float_double_2 where in_ts in (\'2020-01-01 08:00:00.001\') order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.query('select * from in_ts_float_double_2 where in_float in (666.00000) order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.query('select * from in_ts_float_double_2 where in_double in (-88888.000000000) order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) - cmd3 = 'select * from in_stable_4 where in_double in (\'66666\');' + cmd3 = 'insert into in_ts_float_double_3 values(now,\'2021-01-01 08:00:00.001\',\'-888.00000\',\'66666.000000000\') ;' tdLog.info(cmd3) - tdSql.error(cmd3) - try: - tdSql.execute(cmd3) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") + tdSql.execute(cmd3) - cmd4 = 'select * from in_stable_4 where tin_float in (\'666\');' - tdLog.info(cmd4) - tdSql.error(cmd4) - try: - tdSql.execute(cmd4) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") + tdSql.query('select * from in_stable_4 where in_ts in (\'2021-01-01 08:00:00.001\') order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001000') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(0,4,'2021-01-01 08:00:00.001') + tdSql.checkData(0,5,-888.00000) + tdSql.checkData(0,6,66666.000000000) + tdSql.query('select * from in_stable_4 where in_ts in (\'1609459200001\') order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001000') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(0,4,'2021-01-01 08:00:00.001') + tdSql.checkData(0,5,-888.00000) + tdSql.checkData(0,6,66666.000000000) + tdSql.query('select * from in_stable_4 where in_float in (-888.00000) order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001000') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(0,4,'2021-01-01 08:00:00.001') + tdSql.checkData(0,5,-888.00000) + tdSql.checkData(0,6,66666.000000000) + tdSql.query('select * from in_stable_4 where in_double in (66666.000000000) order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001000') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(0,4,'2021-01-01 08:00:00.001') + tdSql.checkData(0,5,-888.00000) + tdSql.checkData(0,6,66666.000000000) + + tdSql.query('select * from in_stable_4 where tin_ts in (\'2021-01-01 08:00:00.001000\') order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001000') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(0,4,'2021-01-01 08:00:00.001') + tdSql.checkData(0,5,-888.00000) + tdSql.checkData(0,6,66666.000000000) + tdSql.query('select * from in_stable_4 where tin_ts in (\'1609459200001\') order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001000') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(0,4,'2021-01-01 08:00:00.001') + tdSql.checkData(0,5,-888.00000) + tdSql.checkData(0,6,66666.000000000) + tdSql.query('select * from in_stable_4 where tin_float in (-888.00000) order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001000') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(0,4,'2021-01-01 08:00:00.001') + tdSql.checkData(0,5,-888.00000) + tdSql.checkData(0,6,66666.000000000) + tdSql.query('select * from in_stable_4 where tin_double in (66666.000000000) order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001000') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(0,4,'2021-01-01 08:00:00.001') + tdSql.checkData(0,5,-888.00000) + tdSql.checkData(0,6,66666.000000000) + + tdSql.query('select * from in_ts_float_double_3 where in_ts in (\'1609459200001\') order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.query('select * from in_ts_float_double_3 where in_ts in (\'2021-01-01 08:00:00.001\') order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.query('select * from in_ts_float_double_3 where in_float in (-888.00000) order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.query('select * from in_ts_float_double_3 where in_double in (66666.000000000) order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + + + tdLog.info("=============== step4.3,multiple column and multiple tag check in function") + cmd1 = '''select * from in_stable_4 + where in_ts in (\'1609459200001\',\'2021-01-01 08:00:00.001\',\'1577836800001\',\'2020-01-01 08:00:00.001000\',\'0\',\'1970-01-01 08:00:00.000\') + and in_float in (0.00000,666.00000,-888.00000) + and in_double in (0.000000000,66666.000000000,-88888.000000000) + and tin_ts in (\'1609459200001\',\'2021-01-01 08:00:00.001\',\'1577836800001\',\'2020-01-01 08:00:00.001000\',\'0\',\'1970-01-01 08:00:00.000\') + and tin_float in (0.00000,666.00000,-888.00000) + and tin_double in (0.000000000,66666.000000000,-88888.000000000) + order by ts desc ;''' + tdLog.info(cmd1) + tdSql.query(cmd1) + tdSql.checkData(0,1,'2021-01-01 08:00:00.001000') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(0,4,'2021-01-01 08:00:00.001') + tdSql.checkData(0,5,-888.00000) + tdSql.checkData(0,6,66666.000000000) + tdSql.checkData(1,1,'2020-01-01 08:00:00.001000') + tdSql.checkData(1,2,666.00000) + tdSql.checkData(1,3,-88888.000000000) + tdSql.checkData(1,4,'2020-01-01 08:00:00.001') + tdSql.checkData(1,5,666.00000) + tdSql.checkData(1,6,-88888.000000000) + tdSql.checkData(2,1,'1970-01-01 08:00:00.000') + tdSql.checkData(2,2,0.00000) + tdSql.checkData(2,3,0.000000000) + tdSql.checkData(2,4,'1970-01-01 08:00:00.000') + tdSql.checkData(2,5,0.00000) + tdSql.checkData(2,6,0.000000000) + - cmd5 = 'select * from in_stable_4 where tin_double in (\'88888\');' - tdLog.info(cmd5) - tdSql.error(cmd5) - try: - tdSql.execute(cmd5) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") - - cmd6 = 'select * from in_float_double_1 where in_float in (\'888\');' - tdLog.info(cmd6) - tdSql.error(cmd6) - try: - tdSql.execute(cmd6) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") - - cmd7 = 'select * from in_float_double_1 where in_double in (\'66666\');' - tdLog.info(cmd7) - tdSql.error(cmd7) - try: - tdSql.execute(cmd7) - tdLog.exit("invalid operation: not supported filter condition") - except Exception as e: - tdLog.info(repr(e)) - tdLog.info("invalid operation: not supported filter condition") - - tdLog.info("=============== step4.3,drop normal table && create table") - cmd1 = 'drop table if exists normal_in_float_double_1 ;' - cmd2 = 'create table normal_in_float_double_1 (ts timestamp,in_float float,in_double double) ; ' + tdLog.info("=============== step4.4,drop normal table && create table") + cmd1 = 'drop table if exists normal_in_ts_float_double_1 ;' + cmd2 = 'create table normal_in_ts_float_double_1 (ts timestamp,in_ts timestamp,in_float float,in_double double) ; ' tdLog.info(cmd1) tdSql.execute(cmd1) tdLog.info(cmd2) tdSql.execute(cmd2) - tdLog.info("=============== step4.4,insert normal table right data and check in function") - cmd1 = 'insert into normal_in_float_double_1 values(now,\'888\',\'666666\') ;' + tdLog.info("=============== step4.5,insert normal table right data and check in function") + cmd1 = 'insert into normal_in_ts_float_double_1 values(now,\'0\',\'0\',\'0\') ;' tdLog.info(cmd1) - tdSql.execute(cmd1) + tdSql.execute(cmd1) - cmd2 = 'select * from normal_in_float_double_1 where in_float in (\'888\');' - #tdLog.info(cmd2) - #tdSql.error(cmd2) - #try: - # tdSql.execute(cmd2) - # tdLog.exit("invalid operation: not supported filter condition") - #except Exception as e: - # tdLog.info(repr(e)) - # tdLog.info("invalid operation: not supported filter condition") - # - #cmd3 = 'select * from normal_in_float_double_1 where in_double in (\'66666\');' - #tdLog.info(cmd3) - #tdSql.error(cmd3) - #try: - # tdSql.execute(cmd3) - # tdLog.exit("invalid operation: not supported filter condition") - #except Exception as e: - # tdLog.info(repr(e)) - # tdLog.info("invalid operation: not supported filter condition") + tdSql.query('select * from normal_in_ts_float_double_1 where in_ts in (\'0\') order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.query('select * from normal_in_ts_float_double_1 where in_ts in (\'1970-01-01 08:00:00.000\') order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.query('select * from normal_in_ts_float_double_1 where in_float in (0.00000) order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + tdSql.query('select * from normal_in_ts_float_double_1 where in_double in (0.000000000) order by ts desc') + tdSql.checkData(0,1,'1970-01-01 08:00:00.000') + tdSql.checkData(0,2,0.00000) + tdSql.checkData(0,3,0.000000000) + + cmd2 = 'insert into normal_in_ts_float_double_1 values(now,\'2020-01-01 08:00:00.001\',\'666\',\'-88888\') ;' + tdLog.info(cmd2) + tdSql.execute(cmd2) + + tdSql.query('select * from normal_in_ts_float_double_1 where in_ts in (\'1577836800001\') order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.query('select * from normal_in_ts_float_double_1 where in_ts in (\'2020-01-01 08:00:00.001\') order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.query('select * from normal_in_ts_float_double_1 where in_float in (666.00000) order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + tdSql.query('select * from normal_in_ts_float_double_1 where in_double in (-88888.000000000) order by ts desc') + tdSql.checkData(0,1,'2020-01-01 08:00:00.001') + tdSql.checkData(0,2,666.00000) + tdSql.checkData(0,3,-88888.000000000) + + cmd3 = 'insert into normal_in_ts_float_double_1 values(now,\'2021-01-01 08:00:00.001\',\'-888.00000\',\'66666.000000000\') ;' + tdLog.info(cmd3) + tdSql.execute(cmd3) + + tdSql.query('select * from normal_in_ts_float_double_1 where in_ts in (\'1609459200001\') order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.query('select * from normal_in_ts_float_double_1 where in_ts in (\'2021-01-01 08:00:00.001\') order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.query('select * from normal_in_ts_float_double_1 where in_float in (-888.00000) order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.query('select * from normal_in_ts_float_double_1 where in_double in (66666.000000000) order by ts desc') + tdSql.checkData(0,1,'2021-01-01 08:00:00.001') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + + cmd4 = '''select * from normal_in_ts_float_double_1 + where in_ts in (\'1609459200001\',\'2021-01-01 08:00:00.001\',\'1577836800001\',\'2020-01-01 08:00:00.001000\',\'0\',\'1970-01-01 08:00:00.000\') + and in_double in (0.000000000,66666.000000000,-88888.000000000) + and in_float in (0.00000,666.00000,-888.00000) + order by ts desc ;''' + tdLog.info(cmd4) + tdSql.query(cmd4) + tdSql.checkData(0,1,'2021-01-01 08:00:00.001') + tdSql.checkData(0,2,-888.00000) + tdSql.checkData(0,3,66666.000000000) + tdSql.checkData(1,1,'2020-01-01 08:00:00.001') + tdSql.checkData(1,2,666.00000) + tdSql.checkData(1,3,-88888.000000000) + tdSql.checkData(2,1,'1970-01-01 08:00:00.000') + tdSql.checkData(2,2,0.00000) + tdSql.checkData(2,3,0.000000000) + + def stop(self): tdSql.close() From ee974c556393ea675bc84ca55145b9f7418e9b39 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 25 Jun 2021 14:12:37 +0800 Subject: [PATCH 089/115] Update README-CN.md (#6621) --- README-CN.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README-CN.md b/README-CN.md index afb242d621..d5586c78b7 100644 --- a/README-CN.md +++ b/README-CN.md @@ -23,7 +23,7 @@ TDengine是涛思数据专为物联网、车联网、工业互联网、IT运维 TDengine是一个高效的存储、查询、分析时序大数据的平台，专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它，但建议您在使用前仔细阅读一遍下面的文档，特别是 [数据模型](https://www.taosdata.com/cn/documentation/architecture) 与 [数据建模](https://www.taosdata.com/cn/documentation/model)。除本文档之外，欢迎 [下载产品白皮书](https://www.taosdata.com/downloads/TDengine%20White%20Paper.pdf)。 -# 生成 +# 构建 TDengine目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、macOS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。本快速指南仅适用于通过源码安装。 @@ -107,7 +107,7 @@ Go 连接器和 Grafana 插件在其他独立仓库，如果安装它们的话 git submodule update --init --recursive ``` -## 生成 TDengine +## 构建 TDengine ### Linux 系统 @@ -116,6 +116,12 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` +您可以选择使用 Jemalloc 作为内存分配器，替代默认的 glibc： +```bash +apt install autoconf +cmake .. -DJEMALLOC_ENABLED=true +``` + 在X86-64、X86、arm64、arm32 和 mips64 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 aarch64： From 360d59428c4b74069c9760feedbff1f16048ba66 Mon Sep 17 00:00:00 2001 From: SunShine Chan Date: Fri, 25 Jun 2021 14:20:39 +0800 Subject: [PATCH 090/115] [TD-4902] FIX -m parameter of taosdemo with letter-initial string (#6620) * [TD-4902] FIX with 1 char * Fix prompt of -m @ taosdemo --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 38f0ecb219..0eac1518a7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -982,9 +982,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->len_of_binary = atoi(argv[++i]); } else if (strcmp(argv[i], "-m") == 0) { if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { + (isStringNumber(argv[i+1]))) { printHelp(); - errorPrint("%s", "\n\t-m need a number following!\n"); + errorPrint("%s", "\n\t-m need a letter-initial string following!\n"); exit(EXIT_FAILURE); } arguments->tb_prefix = argv[++i]; From 2555665a8e33a8a8774b41acbb181f6fd265f036 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 25 Jun 2021 15:22:22 +0800 Subject: [PATCH 091/115] [TD-4897] : clients should process result set ASAP after calling "taos_consume()". --- documentation20/cn/08.connector/docs.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 2d76c866d1..aa5fa50b66 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -427,12 +427,15 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 * res：查询结果集，注意结果集中可能没有记录 * param：调用 `taos_subscribe`时客户程序提供的附加参数 * code：错误码 + **注意**：在这个回调函数里不可以做耗时过长的处理，尤其是对于返回的结果集中数据较多的情况，否则有可能导致客户端阻塞等异常状态。如果必须进行复杂计算，则建议在另外的线程中进行处理。 * `TAOS_RES *taos_consume(TAOS_SUB *tsub)` 同步模式下，该函数用来获取订阅的结果。 用户应用程序将其置于一个循环之中。 如两次调用`taos_consume`的间隔小于订阅的轮询周期，API将会阻塞，直到时间间隔超过此周期。 如果数据库有新记录到达，该API将返回该最新的记录，否则返回一个没有记录的空结果集。 如果返回值为 `NULL`，说明系统出错。 异步模式下，用户程序不应调用此API。 + **注意**：在调用 `taos_consume()` 之后，用户应用应确保尽快调用 `taos_fetch_row()` 或 `taos_fetch_block()` 来处理订阅结果，否则服务端会持续缓存查询结果数据等待客户端读取，极端情况下会导致服务端内存消耗殆尽，影响服务稳定性。 + * `void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress)` 取消订阅。 如参数 `keepProgress` 不为0，API会保留订阅的进度信息，后续调用 `taos_subscribe` 时可以基于此进度继续；否则将删除进度信息，后续只能重新开始读取数据。 From d8570135179a1c237997dbad40100106b9ec6111 Mon Sep 17 00:00:00 2001 From: wpan Date: Fri, 25 Jun 2021 17:05:37 +0800 Subject: [PATCH 092/115] fix double free issue --- src/client/src/tscSQLParser.c | 7 ++++++- src/client/src/tscUtil.c | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e94f77d254..b388c21599 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7786,7 +7786,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf const char* msg5 = "only tag query not compatible with normal column filter"; const char* msg6 = "not support stddev/percentile in outer query yet"; const char* msg7 = "drivative requires timestamp column exists in subquery"; - + const char* msg8 = "condition missing for join query"; + int32_t code = TSDB_CODE_SUCCESS; SSqlCmd* pCmd = &pSql->cmd; @@ -7863,6 +7864,10 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } + } else { + if (pQueryInfo->numOfTables > 1) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); + } } // validate the interval info diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 74dbe42eeb..83be33ffef 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -4077,7 +4077,10 @@ SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo) { size_t size = sizeof(SVgroupInfo) * pVgroupsInfo->numOfVgroups + sizeof(SVgroupsInfo); SVgroupsInfo* pInfo = calloc(1, size); - memcpy(pInfo, pVgroupsInfo, size); + pInfo->numOfVgroups = pVgroupsInfo->numOfVgroups; + for (int32_t m = 0; m < pVgroupsInfo->numOfVgroups; ++m) { + tscSVgroupInfoCopy(&pInfo->vgroups[m], &pVgroupsInfo->vgroups[m]); + } return pInfo; } From 0ecaba7e9ea4050d2ab813e35469164d6c0efb8a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 21 Jun 2021 11:18:17 +0800 Subject: [PATCH 093/115] [TD-4756]: nanosecond precison on gcBuildQueryJson in http plugin --- src/plugins/http/src/httpGcJson.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 397791706d..f33a994465 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -228,13 +228,11 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, case TSDB_DATA_TYPE_NCHAR: httpJsonStringForTransMean(jsonBuf, (char *)row[i], fields[i].bytes); break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (precision == TSDB_TIME_PRECISION_MILLI) { // ms - httpJsonInt64(jsonBuf, *((int64_t *)row[i])); - } else { - httpJsonInt64(jsonBuf, *((int64_t *)row[i]) / 1000); - } + case TSDB_DATA_TYPE_TIMESTAMP: { + int64_t ts = convertTimePrecision(*((int64_t *)row[i]), precision, TSDB_TIME_PRECISION_MILLI); + httpJsonInt64(jsonBuf, ts); break; + } default: httpJsonString(jsonBuf, "-", 1); break; From 2361040efe22c4abeeb17f5f2ceef6e60e6c387b Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 25 Jun 2021 18:09:37 +0800 Subject: [PATCH 094/115] [TD-4793] : describe SQL function IRATE(). --- documentation20/cn/12.taos-sql/docs.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 5904abbbaa..5ab055bcda 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -854,7 +854,23 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 适用于：**表**。 + 适用于：**表、（超级表）**。 + + 说明：从 2.1.3.0 版本开始，TWA 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 + +- **IRATE** + ```mysql + SELECT IRATE(field_name) FROM tb_name WHERE clause; + ``` + 功能说明：计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率；如果这两个值呈递减关系，那么只取最后一个数用于计算，而不是使用二者差值。 + + 返回结果数据类型：双精度浮点数Double。 + + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + + 适用于：**表、（超级表）**。 + + 说明：从 2.1.3.0 版本开始新增此函数。IRATE 可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 - **SUM** ```mysql From 41e1d2dd9689cd93fc1a65c88ef5f8d3f9d54953 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 25 Jun 2021 18:55:01 +0800 Subject: [PATCH 095/115] [TD-4555] : describe SQL function DERIVATIVE(). --- documentation20/cn/12.taos-sql/docs.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 5ab055bcda..844865f6db 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -870,7 +870,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 适用于：**表、（超级表）**。 - 说明：从 2.1.3.0 版本开始新增此函数。IRATE 可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 + 说明：（从 2.1.3.0 版本开始新增此函数）IRATE 可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 - **SUM** ```mysql @@ -1219,13 +1219,14 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ``` ### 计算函数 + - **DIFF** ```mysql SELECT DIFF(field_name) FROM tb_name [WHERE clause]; ``` 功能说明：统计表中某列的值与前一行对应值的差。 - 返回结果数据类型： 同应用字段。 + 返回结果数据类型：同应用字段。 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 @@ -1243,13 +1244,27 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 2 row(s) in set (0.001162s) ``` +- **DERIVATIVE** + ```mysql + SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHERE clause]; + ``` + 功能说明：统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定，最小可以是 1 秒（1s）；ignore_negative 参数的值可以是 0 或 1，为 1 时表示忽略负值。 + + 返回结果数据类型：双精度浮点数。 + + 应用字段：不能应用在 timestamp、binary、nchar、bool 类型字段。 + + 适用于：**表、（超级表）**。 + + 说明：（从 2.1.3.0 版本开始新增此函数）输出结果行数是范围内总行数减一，第一行没有结果输出。DERIVATIVE 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表（也即 GROUP BY tbname）。 + - **SPREAD** ```mysql SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` 功能说明：统计表/超级表中某列的最大值和最小值之差。 - 返回结果数据类型： 双精度浮点数。 + 返回结果数据类型：双精度浮点数。 应用字段：不能应用在binary、nchar、bool类型字段。 From 11761d016d01159a0c5bb56c1b646bd9ffa996a9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Jun 2021 23:10:10 +0800 Subject: [PATCH 096/115] [td-4894]:fix bug in group by normal columns. --- src/query/src/qExecutor.c | 18 ++++------- tests/script/general/parser/groupby.sim | 40 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0742d3eb81..e7f63a027d 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1282,11 +1282,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn return; } - int64_t* tsList = NULL; SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); - if (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - tsList = (int64_t*) pFirstColData->pData; - } + int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; STimeWindow w = TSWINDOW_INITIALIZER; @@ -1319,12 +1316,10 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, - bytes); + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, - item->groupIndex); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } @@ -1340,17 +1335,16 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn memcpy(pInfo->prevData, val, bytes); if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, - bytes); + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, - item->groupIndex); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + tfree(pInfo->prevData); } } diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 507431f536..6ae5d420d8 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -741,4 +741,44 @@ if $data14 != 2 then return -1 endi +sql create table m1 (ts timestamp, k int, f1 int) tags(a int); +sql create table tm0 using m1 tags(0); +sql create table tm1 using m1 tags(1); + +sql insert into tm0 values('2020-1-1 1:1:1', 1, 10); +sql insert into tm0 values('2020-1-1 1:1:2', 1, 20); +sql insert into tm1 values('2020-2-1 1:1:1', 2, 10); +sql insert into tm1 values('2020-2-1 1:1:2', 2, 20); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 100 +system sh/exec.sh -n dnode1 -s start +sleep 100 + +sql connect +sleep 100 +sql use group_db0; + +print =========================>TD-4894 +sql select count(*),k from m1 group by k; +if $rows != 2 then + return -1 +endi + +if $data00 != 2 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data10 != 2 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 782e498d1fac32243305ea646f45da8ba2935e8c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Jun 2021 23:22:22 +0800 Subject: [PATCH 097/115] [td-225]fix compiler error. --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 815442a038..805fd9dfe4 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2613,7 +2613,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); if (finalResult) { - insertResultField(pQueryInfo, colIndex, &ids, resultSize, resultType, pExpr->base.aliasName, pExpr); + insertResultField(pQueryInfo, colIndex, &ids, resultSize, (int8_t)resultType, pExpr->base.aliasName, pExpr); } else { assert(ids.num == 1); tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); From 9094bf8849d50721dc1996d2798ad938864b97ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Jun 2021 23:42:15 +0800 Subject: [PATCH 098/115] [td-225]fix compiler error. --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 805fd9dfe4..da02d63f1e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7849,7 +7849,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, 0); - int32_t numOfExprs = tscNumOfExprs(pQueryInfo); + int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); if (numOfExprs == 1) { SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); int32_t f = pExpr->base.functionId; From 2f37ebf686889b600398987d628f84b375e3c4e7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 26 Jun 2021 09:08:59 +0800 Subject: [PATCH 099/115] [TD-4825] order by col after group-by --- src/client/src/tscSQLParser.c | 40 +++++++++++++++++---- src/query/src/qExecutor.c | 67 +++++++++++++++++++++++++++++++++-- src/util/inc/tarray.h | 13 +++++++ src/util/src/tarray.c | 52 ++++++++++++++++++++++++++- 4 files changed, 163 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b2b188661d..76a56a79d0 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5110,7 +5110,7 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, SSchema* pSchema) { const char* msg0 = "only support order by primary timestamp"; const char* msg1 = "invalid column name"; - const char* msg2 = "order by primary timestamp or first tag in groupby clause allowed"; + const char* msg2 = "order by primary timestamp, first tag or groupby column in groupby clause allowed"; const char* msg3 = "invalid column in order by clause, only primary timestamp or first tag in groupby clause allowed"; setDefaultOrderInfo(pQueryInfo); @@ -5163,6 +5163,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq bool orderByTags = false; bool orderByTS = false; + bool orderByGroupbyCol = false; if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { int32_t relTagIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); @@ -5182,11 +5183,18 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq if (PRIMARYKEY_TIMESTAMP_COL_INDEX == index.columnIndex) { orderByTS = true; } - - if (!(orderByTags || orderByTS) && !isTopBottomQuery(pQueryInfo)) { + + SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; + if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { + SColIndex* pColIndex = taosArrayGet(columnInfo, 0); + if (pColIndex->colIndex == index.columnIndex) { + orderByGroupbyCol = true; + } + } + if (!(orderByTags || orderByTS || orderByGroupbyCol) && !isTopBottomQuery(pQueryInfo)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } else { // order by top/bottom result value column is not supported in case of interval query. - assert(!(orderByTags && orderByTS)); + assert(!(orderByTags && orderByTS && orderByGroupbyCol)); } size_t s = taosArrayGetSize(pSortorder); @@ -5196,6 +5204,11 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->groupbyExpr.orderType = p1->sortOrder; + } else if (orderByGroupbyCol) { + tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); + + pQueryInfo->groupbyExpr.orderType = p1->sortOrder; + pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; } else if (isTopBottomQuery(pQueryInfo)) { /* order of top/bottom query in interval is not valid */ SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); @@ -5228,6 +5241,9 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq if (orderByTags) { pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); pQueryInfo->groupbyExpr.orderType = pItem->sortOrder; + } else if (orderByGroupbyCol){ + pQueryInfo->order.order = pItem->sortOrder; + pQueryInfo->order.orderColId = index.columnIndex; } else { pQueryInfo->order.order = pItem->sortOrder; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; @@ -5253,9 +5269,20 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq if (getColumnIndexByName(pCmd, &columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX && !isTopBottomQuery(pQueryInfo)) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + bool validOrder = false; + SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; + if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { + SColIndex* pColIndex = taosArrayGet(columnInfo, 0); + validOrder = (pColIndex->colIndex == index.columnIndex); + } + if (!validOrder) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); + pQueryInfo->groupbyExpr.orderIndex = pSchema[index.columnIndex].colId; + pQueryInfo->groupbyExpr.orderType = p1->sortOrder; + } if (isTopBottomQuery(pQueryInfo)) { @@ -5276,6 +5303,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; + pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; } return TSDB_CODE_SUCCESS; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 69b17504d3..c0fe7b128a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -16,6 +16,7 @@ #include "qFill.h" #include "taosmsg.h" #include "tglobal.h" +#include "talgo.h" #include "exception.h" #include "hash.h" @@ -26,6 +27,7 @@ #include "queryLog.h" #include "tlosertree.h" #include "ttype.h" +#include "tcompare.h" #include "tscompression.h" #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) @@ -207,7 +209,65 @@ static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowIn SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t groupIndex); -// setup the output buffer for each operator +SArray* getOrderCheckColumns(SQueryAttr* pQuery); + + +typedef struct SRowCompSupporter { + SQueryRuntimeEnv *pRuntimeEnv; + int16_t dataOffset; + __compar_fn_t comFunc; +} SRowCompSupporter; + +static int compareRowData(const void *a, const void *b, const void *userData) { + const SResultRow *pRow1 = (const SResultRow *)a; + const SResultRow *pRow2 = (const SResultRow *)b; + + SRowCompSupporter *supporter = (SRowCompSupporter *)userData; + SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; + + tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId); + tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); + + int16_t offset = supporter->dataOffset; + char *in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); + char *in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset); + + return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; +} + +static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) { + SArray *columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); + if (taosArrayGetSize(columnOrderList) <= 0) { + return; + } + int32_t orderId = pRuntimeEnv->pQueryAttr->order.orderColId; + if (orderId <= 0) { + return; + } + bool found = false; + int16_t dataOffset = 0; + + //SColIndex *index = taosArrayGet(columnOrderList, 0); + for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock->pDataBlock, j); + if (orderId == j) { + found = true; + break; + } + dataOffset += pColInfoData->info.bytes; + } + + if (found == false) { + return; + } + int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resType; + + SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)}; + + return taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support); + +} +//setup the output buffer for each operator SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { const static int32_t minSize = 8; @@ -5416,8 +5476,11 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { } initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo); + if (!pRuntimeEnv->pQueryAttr->stableQuery) { + sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + } toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index fc7b6b8584..63cadf39a3 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -197,8 +197,21 @@ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t compa */ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags); + +/** + * sort the pointer data in the array + * @param pArray + * @param compar + * @param param + * @return + */ + +void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param); + #ifdef __cplusplus } #endif + + #endif // TDENGINE_TAOSARRAY_H diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 5e7d9d14da..a59e36d29f 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -15,6 +15,7 @@ #include "os.h" #include "tarray.h" +#include "talgo.h" void* taosArrayInit(size_t size, size_t elemSize) { assert(elemSize > 0); @@ -249,4 +250,53 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t return NULL; } return *(char**)p; -} \ No newline at end of file +} + +static int taosArrayPartition(SArray *pArray, int i, int j, __ext_compar_fn_t fn, const void *userData) { + void* key = taosArrayGetP(pArray, i); + while (i < j) { + while (i < j && fn(taosArrayGetP(pArray, j), key, userData) >= 0) { j--; } + if (i < j) { + void *a = taosArrayGetP(pArray, j); + taosArraySet(pArray, i, &a); + } + while (i < j && fn(taosArrayGetP(pArray, i), key, userData) <= 0) { i++;} + if (i < j) { + void *a = taosArrayGetP(pArray, i); + taosArraySet(pArray, j, &a); + } + } + taosArraySet(pArray, i, &key); + return i; +} + +static void taosArrayQuicksortHelper(SArray *pArray, int low, int high, __ext_compar_fn_t fn, const void *param) { + if (low < high) { + int idx = taosArrayPartition(pArray, low, high, fn, param); + taosArrayQuicksortHelper(pArray, low, idx - 1, fn, param); + taosArrayQuicksortHelper(pArray, idx + 1, high, fn, param); + } +} + +static void taosArrayQuickSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { + return taosArrayQuicksortHelper(pArray, 0, taosArrayGetSize(pArray) - 1, fn, param); +} +static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { + for (int i = 1; i <= pArray->size - 1; ++i) { + for (int j = i; j > 0; --j) { + if (fn(taosArrayGetP(pArray, j), taosArrayGetP(pArray, j - 1), param) == -1) { + void *a = taosArrayGetP(pArray, j); + void *b = taosArrayGetP(pArray, j - 1); + taosArraySet(pArray, j - 1, &a); + taosArraySet(pArray, j, &b); + } else { + break; + } + } + } + +} +void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param) { + return taosArrayGetSize(pArray) > 8 ? + taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); +} From e6273a38dbd9a1dc1971cf3644488a213891fe01 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 26 Jun 2021 13:43:48 +0800 Subject: [PATCH 100/115] [TD-4915] fix show create database except --- src/client/src/tscLocal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 2c9717306f..668a9e9406 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -485,6 +485,7 @@ static int32_t tscGetDBInfo(SCreateBuilder *builder, char *result) { char buf[TSDB_DB_NAME_LEN + 64] = {0}; do { + memset(buf, 0, sizeof(buf)); int32_t* lengths = taos_fetch_lengths(pSql); int32_t ret = tscGetNthFieldResult(row, fields, lengths, 0, buf); if (0 == ret && STR_NOCASE_EQUAL(buf, strlen(buf), builder->buf, strlen(builder->buf))) { From 14d9473b5a56200e43a4c86c8349b6a7bb3cb115 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 26 Jun 2021 15:43:08 +0800 Subject: [PATCH 101/115] [td-225]fix the bug found by regression test. --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index da02d63f1e..cf72b7abd9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2104,7 +2104,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS // if it is not in the final result, do not add it SColumnList ids = createColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); if (finalResult) { - insertResultField(pQueryInfo, resColIdx, &ids, pSchema->bytes, (int8_t)pSchema->type, pExpr->base.aliasName, pExpr); + insertResultField(pQueryInfo, resColIdx, &ids, resBytes, (int8_t)resType, pExpr->base.aliasName, pExpr); } else { tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); } From 09ae4a83f0209c4809fbbf56fd96044d40c655ec Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 26 Jun 2021 22:34:50 +0800 Subject: [PATCH 102/115] Hotfix/sangshuduo/td 4892 taosdemo sub fetch for develop (#6635) * fix crash if no result file. * make taosdemo same as master branch. --- src/kit/taosdemo/taosdemo.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0eac1518a7..b2ac11b2ca 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1204,23 +1204,24 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { return ; } - int totalLen = 0; + int64_t totalLen = 0; // fetch the records row by row while((row = taos_fetch_row(res))) { - if ((strlen(pThreadInfo->filePath) > 0) - && (totalLen >= 100*1024*1024 - 32000)) { - appendResultBufToFile(databuf, pThreadInfo); + if (totalLen >= 100*1024*1024 - 32000) { + if (strlen(pThreadInfo->filePath) > 0) + appendResultBufToFile(databuf, pThreadInfo); totalLen = 0; memset(databuf, 0, 100*1024*1024); } num_rows++; - char temp[16000] = {0}; + char temp[16000] = {0}; int len = taos_print_row(temp, row, fields, num_fields); len += sprintf(temp + len, "\n"); //printf("query result:%s\n", temp); memcpy(databuf + totalLen, temp, len); totalLen += len; + debugPrint("totalLen: %"PRId64"\n", totalLen); } verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", From 7145bc5fc2bdb31bc6ab628263829afa3473aebd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 27 Jun 2021 03:53:30 +0800 Subject: [PATCH 103/115] [TD-4825] order by col after group-by --- src/util/src/tarray.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index a59e36d29f..15fda4ab51 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -279,9 +279,15 @@ static void taosArrayQuicksortHelper(SArray *pArray, int low, int high, __ext_co } static void taosArrayQuickSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { - return taosArrayQuicksortHelper(pArray, 0, taosArrayGetSize(pArray) - 1, fn, param); + if (pArray->size <= 0) { + return; + } + taosArrayQuicksortHelper(pArray, 0, taosArrayGetSize(pArray) - 1, fn, param); } static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { + if (pArray->size <= 0) { + return; + } for (int i = 1; i <= pArray->size - 1; ++i) { for (int j = i; j > 0; --j) { if (fn(taosArrayGetP(pArray, j), taosArrayGetP(pArray, j - 1), param) == -1) { @@ -294,9 +300,12 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } } } + return; } +// order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param) { - return taosArrayGetSize(pArray) > 8 ? + taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); } +//TODO(yihaoDeng) order array From a2edc05ba0ee80762dadbd00736c840f28f15829 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 27 Jun 2021 04:46:51 +0800 Subject: [PATCH 104/115] [TD-4825] order by col after group-by --- src/util/src/tarray.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 15fda4ab51..fe529edaac 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -279,13 +279,13 @@ static void taosArrayQuicksortHelper(SArray *pArray, int low, int high, __ext_co } static void taosArrayQuickSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { - if (pArray->size <= 0) { + if (pArray->size <= 1) { return; } - taosArrayQuicksortHelper(pArray, 0, taosArrayGetSize(pArray) - 1, fn, param); + taosArrayQuicksortHelper(pArray, 0, (int)(taosArrayGetSize(pArray) - 1), fn, param); } static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { - if (pArray->size <= 0) { + if (pArray->size <= 1) { return; } for (int i = 1; i <= pArray->size - 1; ++i) { @@ -303,9 +303,9 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void return; } -// order array +// order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param) { taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); } -//TODO(yihaoDeng) order array +//TODO(yihaoDeng) add order array From b2975b7c4d4dac617f270c3940e86c6f24af8e6e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 27 Jun 2021 05:04:33 +0800 Subject: [PATCH 105/115] [TD-4825] order by col after group-by --- src/query/src/qExecutor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c0fe7b128a..aa5ce4fa57 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -264,7 +264,8 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeE SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)}; - return taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support); + taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support); + return; } //setup the output buffer for each operator From ae972e2fb44e63bf56a08441dd2db9eac3cc5169 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 27 Jun 2021 09:12:14 +0800 Subject: [PATCH 106/115] [TD-4926]:fix core during operator time usage calculate --- src/query/src/qExecutor.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1cd50df46b..336925c797 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3792,7 +3792,9 @@ void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType e event.operatorType = operatorInfo->operatorType; SQInfo* qInfo = operatorInfo->pRuntimeEnv->qinfo; - taosArrayPush(qInfo->summary.queryProfEvents, &event); + if (qInfo->summary.queryProfEvents) { + taosArrayPush(qInfo->summary.queryProfEvents, &event); + } } void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code) { @@ -3801,7 +3803,9 @@ void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code) { event.eventTime = taosGetTimestampUs(); event.abortCode = code; - taosArrayPush(pQInfo->summary.queryProfEvents, &event); + if (pQInfo->summary.queryProfEvents) { + taosArrayPush(pQInfo->summary.queryProfEvents, &event); + } } typedef struct { @@ -3837,10 +3841,21 @@ static void doOperatorExecProfOnce(SOperatorStackItem* item, SQueryProfEvent* ev } void calculateOperatorProfResults(SQInfo* pQInfo) { + if (pQInfo->summary.queryProfEvents == NULL) { + qDebug("query prof events array is null"); + return; + } + + if (pQInfo->summary.operatorProfResults == NULL) { + qDebug("operator prof results hash is null"); + return; + } + SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem)); if (opStack == NULL) { return; } + size_t size = taosArrayGetSize(pQInfo->summary.queryProfEvents); SHashObj* profResults = pQInfo->summary.operatorProfResults; @@ -3897,11 +3912,13 @@ void queryCostStatis(SQInfo *pQInfo) { qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); - SOperatorProfResult* opRes = taosHashIterate(pSummary->operatorProfResults, NULL); - while (opRes != NULL) { - qDebug("QInfo:0x%"PRIx64" :cost summary: operator : %d, exec times: %"PRId64", self time: %"PRId64, pQInfo->qId, - opRes->operatorType, opRes->sumRunTimes, opRes->sumSelfTime ); - opRes = taosHashIterate(pSummary->operatorProfResults, opRes); + if (pSummary->operatorProfResults) { + SOperatorProfResult* opRes = taosHashIterate(pSummary->operatorProfResults, NULL); + while (opRes != NULL) { + qDebug("QInfo:0x%" PRIx64 " :cost summary: operator : %d, exec times: %" PRId64 ", self time: %" PRId64, + pQInfo->qId, opRes->operatorType, opRes->sumRunTimes, opRes->sumSelfTime); + opRes = taosHashIterate(pSummary->operatorProfResults, opRes); + } } } @@ -4305,8 +4322,14 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr int32_t numOfTables = (int32_t)pQueryAttr->tableGroupInfo.numOfTables; pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); pQInfo->summary.queryProfEvents = taosArrayInit(512, sizeof(SQueryProfEvent)); + if (pQInfo->summary.queryProfEvents == NULL) { + qDebug("failed to allocate query prof events array"); + } pQInfo->summary.operatorProfResults = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); + if (pQInfo->summary.operatorProfResults == NULL) { + qDebug("failed to allocate operator prof results hash"); + } code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQueryAttr->tableGroupInfo.numOfTables, pOperator, param); if (code != TSDB_CODE_SUCCESS) { From 3451e7fba9a7a4652232303cd38988cbbf4dfb0c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 27 Jun 2021 14:16:22 +0800 Subject: [PATCH 107/115] [TD-4926]:follow log format convention --- src/query/src/qExecutor.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 336925c797..4aca4d3534 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3842,12 +3842,12 @@ static void doOperatorExecProfOnce(SOperatorStackItem* item, SQueryProfEvent* ev void calculateOperatorProfResults(SQInfo* pQInfo) { if (pQInfo->summary.queryProfEvents == NULL) { - qDebug("query prof events array is null"); + qDebug("QInfo:0x%"PRIx64" query prof events array is null", pQInfo->qId); return; } if (pQInfo->summary.operatorProfResults == NULL) { - qDebug("operator prof results hash is null"); + qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId); return; } @@ -4323,12 +4323,12 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); pQInfo->summary.queryProfEvents = taosArrayInit(512, sizeof(SQueryProfEvent)); if (pQInfo->summary.queryProfEvents == NULL) { - qDebug("failed to allocate query prof events array"); + qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId); } pQInfo->summary.operatorProfResults = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); if (pQInfo->summary.operatorProfResults == NULL) { - qDebug("failed to allocate operator prof results hash"); + qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId); } code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQueryAttr->tableGroupInfo.numOfTables, pOperator, param); From 9a06248432a70b29ce76b4d77331e1056ba21726 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Sun, 27 Jun 2021 06:13:33 -0700 Subject: [PATCH 108/115] Rename file and add some comment/changes (#6587) 1. rename semphone to semaphore 2. add comment for tsched.h 3. change the function signature for taosSchedulerTask, changing from return int to void. We currently don't check any return code of the function. 4. add some error handlings. For fatal error, just exit the process because the program may run into a random state. --- src/os/inc/os.h | 2 +- src/os/inc/{osSemphone.h => osSemaphore.h} | 4 +- .../darwin/{dwSemphone.c => dwSemaphore.c} | 0 .../detail/{osSemphone.c => osSemaphore.c} | 0 .../src/windows/{wSemphone.c => wSemaphore.c} | 0 src/util/inc/tsched.h | 39 +++++++++-- src/util/src/tsched.c | 67 ++++++++++++------- 7 files changed, 79 insertions(+), 33 deletions(-) rename src/os/inc/{osSemphone.h => osSemaphore.h} (97%) rename src/os/src/darwin/{dwSemphone.c => dwSemaphore.c} (100%) rename src/os/src/detail/{osSemphone.c => osSemaphore.c} (100%) rename src/os/src/windows/{wSemphone.c => wSemaphore.c} (100%) diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 6731ca6d7d..903e80d5c7 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -29,7 +29,7 @@ extern "C" { #include "osMath.h" #include "osMemory.h" #include "osRand.h" -#include "osSemphone.h" +#include "osSemaphore.h" #include "osSignal.h" #include "osSleep.h" #include "osSocket.h" diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemaphore.h similarity index 97% rename from src/os/inc/osSemphone.h rename to src/os/inc/osSemaphore.h index fe59095205..10d14700e0 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemaphore.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_SEMPHONE_H -#define TDENGINE_OS_SEMPHONE_H +#ifndef TDENGINE_OS_SEMAPHORE_H +#define TDENGINE_OS_SEMAPHORE_H #ifdef __cplusplus extern "C" { diff --git a/src/os/src/darwin/dwSemphone.c b/src/os/src/darwin/dwSemaphore.c similarity index 100% rename from src/os/src/darwin/dwSemphone.c rename to src/os/src/darwin/dwSemaphore.c diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemaphore.c similarity index 100% rename from src/os/src/detail/osSemphone.c rename to src/os/src/detail/osSemaphore.c diff --git a/src/os/src/windows/wSemphone.c b/src/os/src/windows/wSemaphore.c similarity index 100% rename from src/os/src/windows/wSemphone.c rename to src/os/src/windows/wSemaphore.c diff --git a/src/util/inc/tsched.h b/src/util/inc/tsched.h index 3e481cbc32..a1591512c1 100644 --- a/src/util/inc/tsched.h +++ b/src/util/inc/tsched.h @@ -28,10 +28,41 @@ typedef struct SSchedMsg { void *thandle; } SSchedMsg; -void *taosInitScheduler(int queueSize, int numOfThreads, const char *label); -void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl); -int taosScheduleTask(void *qhandle, SSchedMsg *pMsg); -void taosCleanUpScheduler(void *param); +/** + * Create a thread-safe ring-buffer based task queue and return the instance. A thread + * pool will be created to consume the messages in the queue. + * @param capacity the queue capacity + * @param numOfThreads the number of threads for the thread pool + * @param label the label of the queue + * @return the created queue scheduler + */ +void *taosInitScheduler(int capacity, int numOfThreads, const char *label); + +/** + * Create a thread-safe ring-buffer based task queue and return the instance. + * Same as taosInitScheduler, and it also print the queue status every 1 minite. + * @param capacity the queue capacity + * @param numOfThreads the number of threads for the thread pool + * @param label the label of the queue + * @param tmrCtrl the timer controller, tmr_ctrl_t* + * @return the created queue scheduler + */ +void *taosInitSchedulerWithInfo(int capacity, int numOfThreads, const char *label, void *tmrCtrl); + +/** + * Clean up the queue scheduler instance and free the memory. + * @param queueScheduler the queue scheduler to free + */ +void taosCleanUpScheduler(void *queueScheduler); + +/** + * Schedule a new task to run, the task is described by pMsg. + * The function may be blocked if no thread is available to execute the task. + * That may happen when all threads are busy. + * @param queueScheduler the queue scheduler instance + * @param pMsg the message for the task + */ +void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg); #ifdef __cplusplus } diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index f014dd0fab..16142470c9 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -108,39 +108,47 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl) { SSchedQueue* pSched = taosInitScheduler(queueSize, numOfThreads, label); - + if (tmrCtrl != NULL && pSched != NULL) { pSched->pTmrCtrl = tmrCtrl; taosTmrReset(taosDumpSchedulerStatus, DUMP_SCHEDULER_TIME_WINDOW, pSched, pSched->pTmrCtrl, &pSched->pTimer); } - + return pSched; } -void *taosProcessSchedQueue(void *param) { +void *taosProcessSchedQueue(void *scheduler) { SSchedMsg msg; - SSchedQueue *pSched = (SSchedQueue *)param; + SSchedQueue *pSched = (SSchedQueue *)scheduler; + int ret = 0; while (1) { - if (tsem_wait(&pSched->fullSem) != 0) { - uError("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); + if ((ret = tsem_wait(&pSched->fullSem)) != 0) { + uFatal("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); + exit(ret); } if (pSched->stop) { break; } - if (pthread_mutex_lock(&pSched->queueMutex) != 0) - uError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } msg = pSched->queue[pSched->fullSlot]; memset(pSched->queue + pSched->fullSlot, 0, sizeof(SSchedMsg)); pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize; - if (pthread_mutex_unlock(&pSched->queueMutex) != 0) - uError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } - if (tsem_post(&pSched->emptySem) != 0) - uError("post %s emptySem failed(%s)", pSched->label, strerror(errno)); + if ((ret = tsem_post(&pSched->emptySem)) != 0) { + uFatal("post %s emptySem failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } if (msg.fp) (*(msg.fp))(&msg); @@ -151,30 +159,37 @@ void *taosProcessSchedQueue(void *param) { return NULL; } -int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) { - SSchedQueue *pSched = (SSchedQueue *)qhandle; +void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { + SSchedQueue *pSched = (SSchedQueue *)queueScheduler; + int ret = 0; + if (pSched == NULL) { uError("sched is not ready, msg:%p is dropped", pMsg); - return 0; + return; } - if (tsem_wait(&pSched->emptySem) != 0) { - uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); + if ((ret = tsem_wait(&pSched->emptySem)) != 0) { + uFatal("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); + exit(ret); } - if (pthread_mutex_lock(&pSched->queueMutex) != 0) - uError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } pSched->queue[pSched->emptySlot] = *pMsg; pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize; - if (pthread_mutex_unlock(&pSched->queueMutex) != 0) - uError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } - if (tsem_post(&pSched->fullSem) != 0) - uError("post %s fullSem failed(%s)", pSched->label, strerror(errno)); - - return 0; + if ((ret = tsem_post(&pSched->fullSem)) != 0) { + uFatal("post %s fullSem failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } } void taosCleanUpScheduler(void *param) { @@ -219,4 +234,4 @@ void taosDumpSchedulerStatus(void *qhandle, void *tmrId) { } taosTmrReset(taosDumpSchedulerStatus, DUMP_SCHEDULER_TIME_WINDOW, pSched, pSched->pTmrCtrl, &pSched->pTimer); -} +} \ No newline at end of file From f48783df4f7b20bf291f6ba9bb4e6a86b66d4040 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 27 Jun 2021 22:45:38 +0800 Subject: [PATCH 109/115] fix converity scan issue. (#6641) --- src/kit/taosdump/taosdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 05c6b1efbb..33118ce311 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -1126,7 +1126,7 @@ int taosGetTableDes( strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(16, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); From 3d1f8f8bcf9d12ca220512e3ba0b13e2d13c28a2 Mon Sep 17 00:00:00 2001 From: wpan Date: Mon, 28 Jun 2021 08:33:20 +0800 Subject: [PATCH 110/115] fix mem leak --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 83be33ffef..937fe1000a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1304,7 +1304,7 @@ void tscResetSqlCmd(SSqlCmd* pCmd, bool clearCachedMeta) { if (pCmd->pTableMetaMap != NULL) { STableMetaVgroupInfo* p = taosHashIterate(pCmd->pTableMetaMap, NULL); while (p) { - tfree(p->pVgroupInfo); + tscVgroupInfoClear(p->pVgroupInfo); tfree(p->pTableMeta); p = taosHashIterate(pCmd->pTableMetaMap, p); } From b62a1d8baee2d23dcfd2ce149490a8001082e706 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 28 Jun 2021 10:54:03 +0800 Subject: [PATCH 111/115] [TD-2639] : fix typo in doc index. --- documentation20/cn/00.index/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/cn/00.index/docs.md b/documentation20/cn/00.index/docs.md index c900cd373d..ed7156be17 100644 --- a/documentation20/cn/00.index/docs.md +++ b/documentation20/cn/00.index/docs.md @@ -63,7 +63,7 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 ## [高级功能](/advanced-features) * [连续查询(Continuous Query)](/advanced-features#continuous-query)：基于滑动窗口，定时自动的对数据流进行查询计算 -* [数据订阅(Publisher/Subscriber)](/advanced-features#subscribe)：象典型的消息队列，应用可订阅接收到的最新数据 +* [数据订阅(Publisher/Subscriber)](/advanced-features#subscribe)：类似典型的消息队列，应用可订阅接收到的最新数据 * [缓存(Cache)](/advanced-features#cache)：每个设备最新的数据都会缓存在内存中，可快速获取 * [报警监测](/advanced-features#alert)：根据配置规则，自动监测超限行为数据，并主动推送 From 1d5b8fa83c0457054156010bc657c7e3d70d0a56 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Mon, 28 Jun 2021 11:01:09 +0800 Subject: [PATCH 112/115] [TD-4928] : update version dependency about JDBC driver & TDengine server. --- documentation20/cn/08.connector/01.java/docs.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation20/cn/08.connector/01.java/docs.md b/documentation20/cn/08.connector/01.java/docs.md index fb47d79268..546e2dbcd3 100644 --- a/documentation20/cn/08.connector/01.java/docs.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -532,8 +532,9 @@ Query OK, 1 row(s) in set (0.000141s) | taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | | -------------------- | ----------------- | -------- | -| 2.0.22 | 2.0.18.0 及以上 | 1.8.x | -| 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.0 | 1.8.x | +| 2.0.31 | 2.1.3.0 及以上 | 1.8.x | +| 2.0.22 - 20.0.30 | 2.0.18.0 - 2.1.2.x | 1.8.x | +| 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.x | 1.8.x | | 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | | 1.0.3 | 1.6.1.x 及以上 | 1.8.x | | 1.0.2 | 1.6.1.x 及以上 | 1.8.x | From 839ad25feeb1ae8842ac2775cb95bd2c67799a05 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 28 Jun 2021 11:12:39 +0800 Subject: [PATCH 113/115] [td-4905]:fix import caused crash. --- src/client/src/tscParseInsert.c | 1 - src/client/src/tscUtil.c | 2 + tests/script/general/parser/gendata.sh | 2 + tests/script/general/parser/import_file.sim | 45 +++++++++++++++++++-- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index a158162dc5..137a7be7c7 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1580,7 +1580,6 @@ void tscImportDataFromFile(SSqlObj *pSql) { SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport)); SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); - pCmd->count = 1; FILE *fp = fopen(pCmd->payload, "rb"); if (fp == NULL) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 74dbe42eeb..52d1092107 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3278,6 +3278,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t pnCmd->insertParam.pTableNameList = NULL; pnCmd->insertParam.pTableBlockHashList = NULL; + memset(&pnCmd->insertParam.tagData, 0, sizeof(STagData)); + if (tscAddQueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; diff --git a/tests/script/general/parser/gendata.sh b/tests/script/general/parser/gendata.sh index f56fdc3468..b2074147ca 100755 --- a/tests/script/general/parser/gendata.sh +++ b/tests/script/general/parser/gendata.sh @@ -4,3 +4,5 @@ Cur_Dir=$(pwd) echo $Cur_Dir echo "'2020-1-1 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql +echo "'2020-1-2 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql +echo "'2020-1-3 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql diff --git a/tests/script/general/parser/import_file.sim b/tests/script/general/parser/import_file.sim index e9f0f1ed08..cf11194ba7 100644 --- a/tests/script/general/parser/import_file.sim +++ b/tests/script/general/parser/import_file.sim @@ -15,6 +15,8 @@ $inFileName = '~/data.csv' $numOfRows = 10000 system general/parser/gendata.sh +sql create table stbx (ts TIMESTAMP, collect_area NCHAR(12), device_id BINARY(16), imsi BINARY(16), imei BINARY(16), mdn BINARY(10), net_type BINARY(4), mno NCHAR(4), province NCHAR(10), city NCHAR(16), alarm BINARY(2)) tags(a int, b binary(12)); + sql create table tbx (ts TIMESTAMP, collect_area NCHAR(12), device_id BINARY(16), imsi BINARY(16), imei BINARY(16), mdn BINARY(10), net_type BINARY(4), mno NCHAR(4), province NCHAR(10), city NCHAR(16), alarm BINARY(2)) print ====== create tables success, starting import data @@ -23,13 +25,48 @@ sql import into tbx file '~/data.sql' sql select count(*) from tbx if $rows != 1 then + print expect 1, actual: $rows return -1 endi -#if $data00 != $numOfRows then -# print "expect: $numOfRows, act: $data00" -# return -1 -#endi +if $data00 != 3 then + return -1 +endi + +sql drop table tbx; + +sql insert into tbx using stbx tags(1,'abc') file '~/data.sql'; +sql insert into tbx using stbx tags(1,'abc') file '~/data.sql'; + +sql select count(*) from tbx +if $rows != 1 then + return -1 +endi + +if $data00 != 3 then + return -1 +endi + +sql drop table tbx; +sql insert into tbx using stbx(b) tags('abcf') file '~/data.sql'; + +sql select ts,a,b from tbx; +if $rows != 3 then + return -1 +endi + +if $data00 != @20-01-01 01:01:01.000@ then + print expect 20-01-01 01:01:01.000 , actual: $data00 + return -1 +endi + +if $data01 != NULL then + return -1 +endi + +if $data02 != @abcf@ then + return -1 +endi system rm -f ~/data.sql From 1e592fb52bdb607f3c592457b06e386dbd707e6b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 28 Jun 2021 12:23:26 +0800 Subject: [PATCH 114/115] [TD-4950]: fix undefined left bit shift --- src/mnode/src/mnodeTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 20edb02c38..6c96f1f0d2 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1068,7 +1068,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pStable->info.tableId = strdup(pCreate->tableName); pStable->info.type = TSDB_SUPER_TABLE; pStable->createdTime = taosGetTimestampMs(); - int64_t x = (us&0x000000FFFFFFFFFF); + uint64_t x = (us&0x000000FFFFFFFFFF); x = x<<24; pStable->uid = x + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); pStable->sversion = 0; From b33916be9f568ea7f9267d6da7ec3d8d8e331b73 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 28 Jun 2021 14:57:04 +0800 Subject: [PATCH 115/115] [TD-4825] order by col after group-by --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5884d18ee7..f4eb5a3c2e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5179,7 +5179,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq SArray *columnInfo = pQueryInfo->groupbyExpr.columnInfo; if (columnInfo != NULL && taosArrayGetSize(columnInfo) > 0) { SColIndex* pColIndex = taosArrayGet(columnInfo, 0); - if (pColIndex->colIndex == index.columnIndex) { + if (PRIMARYKEY_TIMESTAMP_COL_INDEX != index.columnIndex && pColIndex->colIndex == index.columnIndex) { orderByGroupbyCol = true; } }